Elasticsearch实现搜索推荐词
本篇介绍的是基于Elasticsearch实现搜索推荐词，其中需要用到Elasticsearch的pinyin插件以及ik分词插件，代码的实现这里提供了java跟C#的版本方便大家参考。
1.实现的结果
①当搜索【qiy】的时候，能匹配企业、祈愿等
②当搜索【qi业】的时候，只能匹配的到企业，如果没有企业，将使用模糊查询，匹配祈愿。
③当搜索【q业】的时候结果同②。
④当搜索【企y】或【企ye】的时候结果同②。
④当搜索【qy】的时候，能匹配企业、祈愿等。
2.实现的逻辑
中文匹配前缀==》全拼匹配前缀==》拼音首字母匹配前缀==》拼音模糊匹配前缀
优先级从左到右，当前面三个有结果的时候不建议用模糊匹配，这样结果更加精确。比如需要获取8个推荐词，先获取中文的，如果足够8个将不再获取之后的匹配结果。但是当模糊匹配之前已经存在匹配结果了，即使数量没有达到8个，也不再继续获取模糊匹配结果。
3.插件准备
ik分词插件安装相对简单，网上教程也多，这里不做介绍。这里讲解下pinyin插件，官方版本的拼音插件不支持中文，处理结果只有拼音的，这样会出现同音字匹配，结果不准确。
这里感谢小伙伴分享的拼音插件修改方法：https://www.cnblogs.com/danvid/p/10691547.html。
按照里面的操作处理后的插件将实现：
企业画报：{"qi"，"企"，"ye"，"业"，"hua"，"画"，"bao"，"报"}
拼音插件的各项具体属性参考：https://blog.csdn.net/a1148233614/article/details/80280024，里面有详细介绍。
4.Elasticsearch创建index
这里使用的ES版本为7.0.1，不再支持mapping，创建代码如下：
PUT /suggest_tset
{
"settings": {
"number_of_shards": 1,
"number_of_replicas": 0,
"analysis": {
"analyzer": {
"prefix_pinyin_analyzer": {
"tokenizer": "standard",
"filter": [
"lowercase",
"prefix_pinyin"
]
},
"full_pinyin_analyzer": {
"tokenizer": "standard",
"filter": [
"lowercase",
"full_pinyin"
]
},
"like_pinyin_analyzer": {
"tokenizer": "standard",
"filter": [
"lowercase",
"like_pinyin"
]
}
},
"filter": {
"_pattern": {
"type": "pattern_capture",
"preserve_original": true,
"patterns": [
"([0-9])",
"([a-z])"
]
},
"prefix_pinyin": {
"type": "pinyin",
"keep_first_letter": "true",
"keep_full_pinyin": "false",
"none_chinese_pinyin_tokenize": "false",
"keep_separate_chinese": "true",
"keep_original": "false"
},
"full_pinyin": {
"type": "pinyin",
"keep_first_letter": "false",
"keep_full_pinyin": "true",
"keep_original": "false",
"keep_separate_chinese": "true",
"keep_none_chinese_in_first_letter": "false"
},
"like_pinyin": {
"type": "pinyin",
"keep_first_letter": "true",
"keep_full_pinyin": "true",
"keep_joined_full_pinyin": "false",
"keep_original": "false",
"keep_separate_chinese": "false",
"keep_none_chinese_in_first_letter": "false"
}
}
}
},
"mappings": {
"dynamic": "false",
"properties": {
"kwsuggest": {
"fields": {
"suggestText": {
"type": "completion",
"analyzer": "standard",
"preserve_separators": "false",
"preserve_position_increments": "true",
"max_input_length": 50
},
"prefix_pinyin": {
"type": "completion",
"analyzer": "prefix_pinyin_analyzer",
"search_analyzer": "standard",
"preserve_separators": "false"
},
"full_pinyin": {
"type": "completion",
"analyzer": "full_pinyin_analyzer",
"search_analyzer": "standard",
"preserve_separators": "false"
},
"like_pinyin": {
"type": "completion",
"analyzer": "like_pinyin_analyzer",
"preserve_separators": "false"
}
},
"type": "text"
}
}
}
}
这里插入几条测试数据
POST _bulk/?refresh=true
{ "index" : { "_index" : "suggest_tset", "_type" : "_doc" } }
{ "kwsuggest": "企业规划"}
{ "index" : { "_index" : "suggest_tset", "_type" : "_doc" } }
{ "kwsuggest": "祈愿设计 完美无瑕"}
{ "index" : { "_index" : "suggest_tset", "_type" : "_doc" } }
{ "kwsuggest": "悬崖的图片 美景"}
{ "index" : { "_index" : "suggest_tset", "_type" : "_doc" } }
{ "kwsuggest": "县衙地址 那里呢"}
{ "index" : { "_index" : "suggest_tset", "_type" : "_doc" } }
{ "kwsuggest": "悬崖风景图"}
{ "index" : { "_index" : "suggest_tset", "_type" : "_doc" } }
{ "kwsuggest": "起夜的风光 真的美"}
{ "index" : { "_index" : "suggest_tset", "_type" : "_doc" } }
{ "kwsuggest": "起夜第二个词 测试使用"}
{ "index" : { "_index" : "suggest_tset", "_type" : "_doc" } }
{ "kwsuggest": "需要一半留下一半打一字谜"}
{ "index" : { "_index" : "suggest_tset", "_type" : "_doc" } }
{ "kwsuggest": "许亚为"}
{ "index" : { "_index" : "suggest_tset", "_type" : "_doc" } }
{ "kwsuggest": "许雅非测试"}
{ "index" : { "_index" : "suggest_tset", "_type" : "_doc" } }
{ "kwsuggest": "徐杨是谁"}
下面为测试的查询语句
GET /suggest_tset/_search
{
"suggest": {
"suggestText": {
"prefix": "qi业",
"completion": {
"field": "kwsuggest.suggestText",
"skip_duplicates": true
}
},
"full_pinyin": {
"prefix": "qi业",
"completion": {
"field": "kwsuggest.full_pinyin",
"skip_duplicates": true
}
},
"prefix_pinyin": {
"prefix": "qi业",
"completion": {
"field": "kwsuggest.prefix_pinyin",
"skip_duplicates": true
}
},
"like_pinyin": {
"prefix": "qi业",
"completion": {
"field": "kwsuggest.like_pinyin",
"skip_duplicates": true,
"fuzzy": {
"fuzziness": 1
}
}
}
}
}
当输入查询条件为【qiy】的时候，结果为：
{
"took" : 17,
"timed_out" : false,
"_shards" : {
"total" : 1,
"successful" : 1,
"skipped" : 0,
"failed" : 0
},
"hits" : {
"total" : {
"value" : 0,
"relation" : "eq"
},
"max_score" : null,
"hits" : [ ]
},
"suggest" : {
"full_pinyin" : [
{
"text" : "qiy",
"offset" : 0,
"length" : 3,
"options" : [
{
"text" : "起夜的风光 真的美",
"_index" : "suggest_tset",
"_type" : "_doc",
"_id" : "-jgnlHMBSEyTxFiDO4lU",
"_score" : 1.0,
"_source" : {
"kwsuggest" : "起夜的风光 真的美"
}
},
{
"text" : "起夜第二个词 测试使用",
"_index" : "suggest_tset",
"_type" : "_doc",
"_id" : "aDg3lHMBSEyTxFiDXprV",
"_score" : 1.0,
"_source" : {
"kwsuggest" : "起夜第二个词 测试使用"
}
}
]
}
],
"like_pinyin" : [
{
"text" : "qiy",
"offset" : 0,
"length" : 3,
"options" : [
{
"text" : "企业规划",
"_index" : "suggest_tset",
"_type" : "_doc",
"_id" : "9TgnlHMBSEyTxFiDO4lU",
"_score" : 2.0,
"_source" : {
"kwsuggest" : "企业规划"
}
},
{
"text" : "祈愿设计 这是啥呢",
"_index" : "suggest_tset",
"_type" : "_doc",
"_id" : "9jgnlHMBSEyTxFiDO4lU",
"_score" : 2.0,
"_source" : {
"kwsuggest" : "祈愿设计 这是啥呢"
}
},
{
"text" : "起夜的风光 真的美",
"_index" : "suggest_tset",
"_type" : "_doc",
"_id" : "-jgnlHMBSEyTxFiDO4lU",
"_score" : 2.0,
"_source" : {
"kwsuggest" : "起夜的风光 真的美"
}
},
{
"text" : "起夜第二个词 测试使用",
"_index" : "suggest_tset",
"_type" : "_doc",
"_id" : "aDg3lHMBSEyTxFiDXprV",
"_score" : 2.0,
"_source" : {
"kwsuggest" : "起夜第二个词 测试使用"
}
}
]
}
],
"prefix_pinyin" : [
{
"text" : "qiy",
"offset" : 0,
"length" : 3,
"options" : [ ]
}
],
"suggestText" : [
{
"text" : "qiy",
"offset" : 0,
"length" : 3,
"options" : [ ]
}
]
}
}
输入【qi业】的查询结果为
{
"took" : 2,
"timed_out" : false,
"_shards" : {
"total" : 1,
"successful" : 1,
"skipped" : 0,
"failed" : 0
},
"hits" : {
"total" : {
"value" : 0,
"relation" : "eq"
},
"max_score" : null,
"hits" : [ ]
},
"suggest" : {
"full_pinyin" : [
{
"text" : "qi业",
"offset" : 0,
"length" : 3,
"options" : [
{
"text" : "企业规划",
"_index" : "suggest_tset",
"_type" : "_doc",
"_id" : "9TgnlHMBSEyTxFiDO4lU",
"_score" : 1.0,
"_source" : {
"kwsuggest" : "企业规划"
}
}
]
}
],
"like_pinyin" : [
{
"text" : "qi业",
"offset" : 0,
"length" : 3,
"options" : [
{
"text" : "企业规划",
"_index" : "suggest_tset",
"_type" : "_doc",
"_id" : "9TgnlHMBSEyTxFiDO4lU",
"_score" : 2.0,
"_source" : {
"kwsuggest" : "企业规划"
}
},
{
"text" : "祈愿设计 这是啥呢",
"_index" : "suggest_tset",
"_type" : "_doc",
"_id" : "9jgnlHMBSEyTxFiDO4lU",
"_score" : 2.0,
"_source" : {
"kwsuggest" : "祈愿设计 这是啥呢"
}
},
{
"text" : "起夜的风光 真的美",
"_index" : "suggest_tset",
"_type" : "_doc",
"_id" : "-jgnlHMBSEyTxFiDO4lU",
"_score" : 2.0,
"_source" : {
"kwsuggest" : "起夜的风光 真的美"
}
},
{
"text" : "起夜第二个词 测试使用",
"_index" : "suggest_tset",
"_type" : "_doc",
"_id" : "aDg3lHMBSEyTxFiDXprV",
"_score" : 2.0,
"_source" : {
"kwsuggest" : "起夜第二个词 测试使用"
}
}
]
}
],
"prefix_pinyin" : [
{
"text" : "qi业",
"offset" : 0,
"length" : 3,
"options" : [ ]
}
],
"suggestText" : [
{
"text" : "qi业",
"offset" : 0,
"length" : 3,
"options" : [ ]
}
]
}
}
输入【qy】的结果为
{
"took" : 1,
"timed_out" : false,
"_shards" : {
"total" : 1,
"successful" : 1,
"skipped" : 0,
"failed" : 0
},
"hits" : {
"total" : {
"value" : 0,
"relation" : "eq"
},
"max_score" : null,
"hits" : [ ]
},
"suggest" : {
"full_pinyin" : [
{
"text" : "qy",
"offset" : 0,
"length" : 2,
"options" : [ ]
}
],
"like_pinyin" : [
{
"text" : "qy",
"offset" : 0,
"length" : 2,
"options" : [
{
"text" : "起夜的风光 真的美",
"_index" : "suggest_tset",
"_type" : "_doc",
"_id" : "-jgnlHMBSEyTxFiDO4lU",
"_score" : 2.0,
"_source" : {
"kwsuggest" : "起夜的风光 真的美"
}
},
{
"text" : "起夜第二个词 测试使用",
"_index" : "suggest_tset",
"_type" : "_doc",
"_id" : "aDg3lHMBSEyTxFiDXprV",
"_score" : 2.0,
"_source" : {
"kwsuggest" : "起夜第二个词 测试使用"
}
}
]
}
],
"prefix_pinyin" : [
{
"text" : "qy",
"offset" : 0,
"length" : 2,
"options" : [
{
"text" : "起夜的风光 真的美",
"_index" : "suggest_tset",
"_type" : "_doc",
"_id" : "-jgnlHMBSEyTxFiDO4lU",
"_score" : 1.0,
"_source" : {
"kwsuggest" : "起夜的风光 真的美"
}
},
{
"text" : "起夜第二个词 测试使用",
"_index" : "suggest_tset",
"_type" : "_doc",
"_id" : "aDg3lHMBSEyTxFiDXprV",
"_score" : 1.0,
"_source" : {
"kwsuggest" : "起夜第二个词 测试使用"
}
}
]
}
],
"suggestText" : [
{
"text" : "qy",
"offset" : 0,
"length" : 2,
"options" : [ ]
}
]
}
}
5.java版本代码
这里使用elasticsearch-rest-high-level-client
application.yml添加配置
# ES配置 elasticsearch: ipAddress: [127.0.0.1:9200]
添加配置类
@Component
@Configuration
@ConfigurationProperties(prefix = "elasticsearch")
@Data
public class ElasticsearchRestClientConfig {
private Logger logger = LoggerFactory.getLogger(getClass());
private static final int ADDRESS_LENGTH = 2;
private static final String HTTP_SCHEME = "http";
/**
* 使用冒号隔开ip和端口
*/
public String[] ipAddress;
@Bean
public RestClientBuilder restClientBuilder() {
HttpHost[] hosts = Arrays.stream(ipAddress)
.map(this::makeHttpHost)
.filter(Objects::nonNull)
.toArray(HttpHost[]::new);
logger.debug("hosts:{}", Arrays.toString(hosts));
return RestClient.builder(hosts);
}
@Bean(name = "highLevelClient")
public RestHighLevelClient highLevelClient(@Autowired RestClientBuilder restClientBuilder) {
return new RestHighLevelClient(restClientBuilder);
}
private HttpHost makeHttpHost(String s) {
assert StringUtils.isNotEmpty(s);
String[] address = s.split(":");
if (address.length == ADDRESS_LENGTH) {
String ip = address[0];
int port = Integer.parseInt(address[1]);
return new HttpHost(ip, port, HTTP_SCHEME);
} else {
return null;
}
}
}
实现的代码：
@Service
public class KwSuggestService implements IKwSuggest {
@Autowired
RestHighLevelClient highLevelClient;
@Override
public List<String> GetKwSuggestList(String kw){
SearchRequest searchRequest = new SearchRequest("suggest_tset");
SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
SuggestBuilder suggestBuilder=new SuggestBuilder();
suggestBuilder.addSuggestion("suggestText", SuggestBuilders.completionSuggestion("kwsuggest.suggestText").prefix(kw).skipDuplicates(true).size(5));
suggestBuilder.addSuggestion("full_pinyin", SuggestBuilders.completionSuggestion("kwsuggest.full_pinyin").prefix(kw).skipDuplicates(true).size(5));
suggestBuilder.addSuggestion("prefix_pinyin", SuggestBuilders.completionSuggestion("kwsuggest.prefix_pinyin").prefix(kw).skipDuplicates(true).size(5));
suggestBuilder.addSuggestion("like_pinyin", SuggestBuilders.completionSuggestion("kwsuggest.like_pinyin").prefix(kw, Fuzziness.fromEdits(1)).skipDuplicates(true).size(5));
sourceBuilder.suggest(suggestBuilder);
sourceBuilder.timeout(new TimeValue(10, TimeUnit.SECONDS));
searchRequest.source(sourceBuilder);
List<String> result = new ArrayList<>();
List<String> suggestionList= Arrays.asList("suggestText","full_pinyin","prefix_pinyin","like_pinyin");
try {
SearchResponse response = highLevelClient.search(searchRequest, RequestOptions.DEFAULT);
Suggest suggestions = response.getSuggest();
Integer index = 1;
for(String suggestionType : suggestionList){
CompletionSuggestion completionSuggestion = suggestions.getSuggestion(suggestionType);
for (CompletionSuggestion.Entry entry : completionSuggestion.getEntries()) {
for (CompletionSuggestion.Entry.Option option : entry) {
String suggestText = option.getHit().getSourceAsMap().get("kwsuggest").toString();
result.add(suggestText);
}
}
// 按照中文匹配、全拼匹配、拼音首字母匹配、模糊匹配的顺序，结果大于5的时候返回结果，根据自己业务需要判断这个返回的数量
if(result.size()>=5){
break;
}
// 中文匹配，全拼匹配以及拼音首字母匹配存在结果的，不需要模糊匹配
if(index==3 && result.size()>0){
break;
}
// 超过3个字模糊匹配不准确
if(kw.length()>3 && result.size()==0){
break;
}
}
return result;
} catch (IOException e) {
e.printStackTrace();
return new ArrayList<>();
}
}
}
6..c#代码实现
C#使用的是NEST
public partial class ElasticFactory
{
public ExternalServiceResponse<KeywordsSuggestResponseDataEntity> GetKeywordsSuggest(ElasticKeywordsSuggestRequest request)
{
var result = new ExternalServiceResponse<KeywordsSuggestResponseDataEntity>();
try
{
if (string.IsNullOrEmpty(request.q)) return result;
var nodes = new Uri[0];
nodes[0] = new Uri("http://127.0.0.1:9200");
var pool = new StaticConnectionPool(nodes);
var settings = new ConnectionSettings(pool).DefaultIndex("suggest_tset");
var client = new ElasticClient(settings);
string[] keys = new[] { "suggestText", "full_pinyin", "prefix_pinyin", "like_pinyin" };
SearchDescriptor<object> search = new SearchDescriptor<object>();
search
.Source(r => r
.Includes(f => f
.Fields("kw")
)
)
.Suggest(s => s.Completion(keys[0], c => c.Field("kwsuggest.suggestText").SkipDuplicates(true).Prefix(request.q).SkipDuplicates())
.Completion(keys[1], c => c.Field("kwsuggest.full_pinyin").SkipDuplicates(true).Prefix(request.q).SkipDuplicates())
.Completion(keys[2], c => c.Field("kwsuggest.prefix_pinyin").SkipDuplicates(true).Prefix(request.q).SkipDuplicates())
.Completion(keys[3], c => c.Field("kwsuggest.like_pinyin").SkipDuplicates(true).Prefix(request.q).SkipDuplicates().Fuzzy(m=>m.Fuzziness(Fuzziness.EditDistance(1)))))
;
var esResult = client.Search<dynamic>(s => search);
if (esResult != null)
{
result.code = 1;
result.data = new KeywordsSuggestResponseDataEntity();
//1.先获取中文全匹配
//2.上面不满5个，再匹配全拼
//3.上面不满5个，中文全拼匹配首字母
//4.上面都没有用模糊匹配
if (esResult.Suggest != null)
{
result.data.items = new List<KeywordsSuggestResponseItemEntity>();
int index = 1;
foreach (var key in keys)
{
AddSuggestItems(esResult.Suggest, key, result.data.items);
//1-3之间，够了5个就返回
if (index >= 1 && index <= 3 && result.data.items.Count >= 5)
{
result.data.items = result.data.items.Skip(0).Take(5).ToList();
break;
}
//到了第3步如果还没有满足5个，直接返回，模糊匹配不精确
if (index == 3 && result.data.items.Count > 0)
{
break;
}
//输入的字符数大于3个以上，前面没有关键词匹配，后面不做模糊处理，匹配度太差了
if (index == 3 && request.q.Length>3)
{
break;
}
index++;
}
result.data.num = result.data.items.Count;
}
else
{
result.data.num = 0;
}
}
else
{
result.code = 0;
result.msg = "查询失败";
}
}
catch (Exception ex)
{
result.code = 0;
result.msg = ex.Message;
}
return result;
}
private void AddSuggestItems(ISuggestDictionary<dynamic> suggest, string key, List<KeywordsSuggestResponseItemEntity> items)
{
var suggestFullPinyin = suggest[key];
if (suggestFullPinyin != null)
{
foreach (var hit in suggestFullPinyin[0].Options)
{
string kwSource = hit.Source["kwsuggest"];
//已经存在的不要重复添加
if (items.Any(m => m.kw == kwSource))
{
continue;
}
items.Add(new KeywordsSuggestResponseItemEntity() { kw = kwSource });
}
}
}
}
