[go-每日一库] golang olivere/elasticsearch/v6 的基本使用 -> 索引|文档常用操作及查询

目录

• 准备工作
• 1.索引的增删改查
• 2.文档的增删改查
• 3.数据的bool查询 && 聚合查询
  • 通过构造NewxxxQuery
  • 通过json或者map的source

准备工作

本文演示内容基于olivere/elasticsearch/v6
下载该库：
go get github.com/olivere/elastic/v6

初始化es

es初始化代码

func initES() *elastic.Client {
	sniffopt := elastic.SetSniff(false) // 非集群下，关闭嗅探机制
	urlopt := elastic.SetURL(URL) // url自行设置，比如我的 http://<user>:<passwd>@ip:9200
	//authOpt := elastic.SetBasicAuth(USER, PASSWORD) // 不知道为何，olivere的库这样设置认证参数有问题，故把auth参数放到url中
	checkOpt := elastic.SetHealthcheck(false)
	options := []elastic.ClientOptionFunc{urlopt, sniffopt, checkOpt}
 
	client, err := elastic.NewClient(options...)
	if err != nil {
		log.Fatal(err)
	}
	// check es conn
	_, statusCode, err := client.Ping(URL).Do(context.Background())
	if err != nil {
		log.Fatal(err)
	}
	if statusCode == 200 {
		log.Println("connect es success.")
	}
	return client
}

创建es结构体并创建new函数提供对外接口

es客户端结构体

type ElasticORM struct{
	client *elastic.Client
	index string
	ctx context.Context
}
 
func NewElasticORM() *ElasticORM {
	return &ElasticORM{
		client: initES(),
		ctx: context.Background(),
	}
}

1.索引的增删改查

查看索引是否存在

// check index exist or not
func (es *ElasticORM) IsIndexExists(index string) bool {
	exist, err := es.client.IndexExists(index).Do(es.ctx)
	if err != nil {
		log.Println(err)
	}
	return exist
}

获取索引mapping

// get index mapping
func (es *ElasticORM) GetIndexMappings(index string) interface{} {
	mapping, err := es.client.GetMapping().Index(index).Do(es.ctx)
	if err != nil {
		log.Println(err)
		return nil
	}
	return mapping
}

设置mapping

// put mapping
func (es *ElasticORM) PutIndexMapping(index string, mapping interface{}) bool {
	var putResp *elastic.PutMappingResponse
	var err error
	switch reflect.TypeOf(mapping).Kind() {
	case reflect.String:
		putResp, err = es.client.PutMapping().Index(index).
			Type("_doc").IgnoreUnavailable(true).
			BodyString(mapping.(string)).Do(es.ctx)
	case reflect.Map:
		putResp, err = es.client.PutMapping().Index(index).
			Type("_doc").IgnoreUnavailable(true).
			BodyJson(mapping.(map[string]interface{})).Do(es.ctx)
	}
	if err != nil {
		log.Println(err)
		return false
	}
	if putResp.Acknowledged {
		return true
	}
	return false
}

创建索引并设置settings

// create index, and put settings
func (es *ElasticORM) CreateIndex(index string, body interface{}) bool {
	resp, err := es.client.CreateIndex(index).BodyJson(body.(map[string]interface{})).Do(es.ctx)
	if err != nil {
		log.Println(err)
	}
	if resp.Acknowledged {
		return true
	}
	return false
}

删除索引

// delete index
func (es *ElasticORM) DeleteIndex(index string) bool {
	resp, err := es.client.DeleteIndex(index).Do(es.ctx)
	if err != nil {
		log.Println(err)
		return false
	}
	log.Println(resp.Acknowledged) // true
	return true
}

2.文档的增删改查

插入文档

// insert data -> created
func (es *ElasticORM) InsertDocToIndex(index string, doc interface{}) bool {
	resp, err := es.client.Index().Index(index).Type("_doc").
		BodyJson(doc.(map[string]interface{})).Do(es.ctx)
	if err != nil {
		log.Println(err)
	}
	if resp.Result == "created" || resp.Result == "updated" {
		log.Println(resp.Result)
	}
	return true
}

创建或更新文档
注意用update更新个别字段时，请用map结构，struct会更新全部的。

// create or update data, same doc, no operation
func (es *ElasticORM) CreateOrUpdateDoc(index string, id string, doc interface{}) bool {
	resp, err := es.client.Update().Index(index).Type("_doc").Id(id).
		Doc(doc).DocAsUpsert(true).Do(es.ctx)
	if err != nil {
		log.Println(err)
	}
	if resp.Result == "created" || resp.Result == "updated" || resp.Result == "noop" {
		log.Println(resp.Result)
	}
	return true
}

批量插入

// bulk insert docs
func (es *ElasticORM) BulkInsertDocs(index string, docs []interface{}, bulkNum int) bool {
	length := len(docs)
	// case-1, no need to care add, just add and do
	if length < bulkNum {
		bulkReq := es.client.Bulk()
		for i:=0; i<length; i++ {
			doc := docs[i]
			req := elastic.NewBulkIndexRequest().Index(index).Type("_doc").Doc(doc)
			bulkReq = bulkReq.Add(req)
		}
		_, err := bulkReq.Do(es.ctx)
		if err != nil {
			log.Println(err)
			return false
		}
		return true
	}
	// case-2, length > bulkNum
	idx := 0
	for {
		bulkReq := es.client.Bulk()
		for i:=0; i<bulkNum; i++ {
			doc := docs[i]
			req := elastic.NewBulkIndexRequest().Index(index).Type("_doc").Doc(doc)
			bulkReq = bulkReq.Add(req)
			idx++
			if idx >= length {
				break
			}
		}
		_, err := bulkReq.Do(es.ctx)
		if err != nil {
			log.Println(err)
			return false
		}
		if idx >= length {
			break
		}
	}
	return true
}

删除文档

// delete doc
func (es *ElasticORM) DeleteDoc(index, id string) bool {
	resp, err := es.client.Delete().Index(index).Id(id).
		Type("_doc").Do(es.ctx)
	if err != nil {
		log.Println(err)
		return false
	}
	if resp.Result == "deleted" {
		log.Println(resp.Result)
	}
	return true
}

获取单个文档

// get specific doc
func (es *ElasticORM) GetDoc(index string, docID string) interface{} {
	resp, err := es.client.Get().Index(index).Type("_doc").Id(docID).Do(es.ctx)
	if err != nil {
		log.Println(err)
	}
	return resp.Source // *json.RawMessage
}

3.数据的bool查询 && 聚合查询

通过构造NewxxxQuery

精确字段查询-个值或数组

// advanced query
// single must cond, default size=10000
func (es *ElasticORM) MustTermOrTermsQuery(index string, name string, val interface{}) []*elastic.SearchHit {
	query := elastic.NewBoolQuery()
	if reflect.TypeOf(val).Kind() == reflect.Slice {
		query.Must(elastic.NewTermsQuery(name, val))
	} else {
		query.Must(elastic.NewTermsQuery(name, val))
	}
	resp, err := es.client.Search().Index(index).Query(query).
		Type("_doc").Size(10000).Do(es.ctx)
	if err != nil {
		log.Println(err)
	}
	return resp.Hits.Hits
}

范围查询

// single must-range cond, deafult size=10000
func (es *ElasticORM) MustRangeQuery(index string, name string, lte interface{}, gte interface{}) []*elastic.SearchHit {
	query := elastic.NewBoolQuery()
	query.Must(elastic.NewRangeQuery(name).Gte(gte).Lte(lte)) // must_not/filter类似，故不再赘述
	resp, err := es.client.Search().Index(index).
		Query(query).Type("_doc").
		Size(10000).Do(es.ctx)
	if err != nil {
		log.Println(err)
	}
	return resp.Hits.Hits
}

指定数量返回

// specify size
func (es *ElasticORM) SizeQuery(index string, size int) []*elastic.SearchHit {
	resp, err := es.client.Search().Index(index).
		Type("_doc").Size(size).Do(es.ctx)
	if err != nil {
		log.Println(err)
	}
	return resp.Hits.Hits
}

指定返回字段

// specify source fields
func (es *ElasticORM) SourceQuery(index string, source interface{}) []*elastic.SearchHit {
	resp, err := es.client.Search().Index(index).
		Source(source).Type("_doc").
		Size(10000).Do(es.ctx)
	if err != nil {
		log.Println(err)
	}
	return resp.Hits.Hits
}

指定顺序查询

// specify sort order
func (es *ElasticORM) SortQuery(index string, field string, ascend bool) []*elastic.SearchHit {
	resp, err := es.client.Search().Index(index).Size(10000).
		Sort(field, ascend).Type("_doc").Do(es.ctx)
	if err != nil {
		log.Println(err)
	}
	return resp.Hits.Hits
}

复杂查询
所谓复杂，无非是多种条件加一起，通过接口提供的不断链式添加即可。

// some complicated query, e.g.
func (es *ElasticORM) ComplicatedQueryForEg(index string) []*elastic.SearchHit {
	query := elastic.NewBoolQuery()
	query.Must(
		elastic.NewTermQuery("name-1", "val"),
		elastic.NewTermQuery("name-2", 100),
		elastic.NewTermsQuery("name-3", []string{"val-1", "val-2"}),
		elastic.NewRangeQuery("name-4").Gte(10).Lte(1000),
		)
	query.Filter(
		elastic.NewTermQuery("name-5", "val"),
		elastic.NewTermQuery("name-6", 100),
		elastic.NewTermsQuery("name-7", []string{"val-1", "val-2"}),
		)
	// must_not/filter like above
	resp, err := es.client.Search().Index(index).
		Query(query).Type("_doc").Do(es.ctx)
	if err != nil {
		log.Println(err)
	}
	return resp.Hits.Hits
}

简单桶聚合

// aggregation query  -> terms bucket
func (es *ElasticORM) AggregationTerms(index string, name string, field string) interface{} {
	agg := elastic.NewTermsAggregation().Field(field).Size(10000)
	resp, err := es.client.Search().
		Index(index).
		Aggregation(name, agg).
		Type("_doc").
		Do(es.ctx)
	if err != nil {
		log.Println(err)
	}
	ret, ok := resp.Aggregations.Terms(name)
	if !ok {
		log.Println("agg results is nil")
	}
	bucketKeys := make([]string, len(ret.Buckets))
	for i, _ := range ret.Buckets {
		bucketKeys[i] = ret.Buckets[i].Key.(string)
	}
	return bucketKeys
}

多级聚合查询
由于多级聚合不像kibana或者python操作，可以在一级的buckets下再往下buckets，索引多查几次实现多级查找。或者在一级buckets中找Aggregations的terms，然后遍历获取具体的key。

// aggregation query -> subAggs terms
func (es *ElasticORM) AggregationTermsWithSubAgg(index string, name1 string, field1 string, name2 string, field2 string) interface{} {
	agg := elastic.NewTermsAggregation().Field(field1).Size(10000)
	resp, err := es.client.Search().
		Index(index).
		Type("_doc").
		Size(0).
		Aggregation(name1, agg).
		Do(es.ctx)
	if err != nil {
		log.Println(err)
	}
	ret, ok := resp.Aggregations.Terms(name1)
	if !ok {
		log.Println("agg results is nil")
	}
	fieldsMap := make(map[string][]interface{})
	for i, _ := range ret.Buckets {
		k := ret.Buckets[i].Key.(string)
		// subAggregation
		agg = elastic.NewTermsAggregation().Field(field2).Size(10000)
		resp, err = es.client.Search().
			Index(index).
			Type("_doc").
			Size(0).
			Query(elastic.NewBoolQuery().Must(elastic.NewTermQuery(field1, k))).
			Aggregation(name2, agg).
			Do(es.ctx)
		if err != nil {
			log.Println(err)
		}
		ret2, _ := resp.Aggregations.Terms(name2)
		for j, _ := range ret2.Buckets {
			fieldsMap[k] = append(fieldsMap[k], ret2.Buckets[j].Key.(interface{}))
		}
	}
	return fieldsMap
}

两级terms usage

// agg sub
func (es *ElasticORM) AggSub(index string, name1 string, field1 string, name2 string, field2 string) interface{} {
	resp, err := es.client.Search(index).
		Type("_doc").
		Aggregation(
			"mainAgg",
			elastic.NewTermsAggregation().Field(field1).
				SubAggregation(
					"sub_1", elastic.NewTermsAggregation().Field(field2),
				),
		).
		Size(0).
		Pretty(true).
		Do(es.ctx)
	if err != nil {
		log.Println(err)
	}
	ret, _ := resp.Aggregations.Terms("mainAgg")
 
	for i, v := range ret.Buckets {
		log.Println("first level buckets key:", ret.Buckets[i].Key)
		ret2, _ := v.Aggregations.Terms("sub_1")
		for j, _ := range ret2.Buckets {
			log.Println("second level buckets key:", ret2.Buckets[j].Key)
		}
	}
	return nil
}

聚合stats-max

// aggregation query -> stats-max
func (es *ElasticORM) AggregationMax(index string, field string) interface{} {
	agg := elastic.NewMaxAggregation().Field(field)
	resp, err := es.client.Search().
		Index(index).
		Type("_doc").
		Aggregation("max_"+field, agg).
		Do(es.ctx)
	if err != nil {
		log.Println(err)
	}
	ret, _ := resp.Aggregations.Max("max_"+field)
	ans := *ret.Value
	return ans
}

通过json或者map的source

代码：

        queryStr := `{
		"size": 2,
		"query": {
			"bool": {
				"must": [
					{"term": {
						"scores": 0
				}}
				]
			}
		}
	}`
	var sourceMap map[string]interface{}
	json.Unmarshal([]byte(queryStr), &sourceMap)
	searchRes, err := client.Search("anomalies_scores").Type("_doc").Source(sourceMap).Do(context.Background())

参考文档：

• 1.golang操作elasticsearch(oliver/elastic使用文档)
• 2.Golang封装Elasticsearch常用功能
• 3.golang中使用elasticsearch之olivere elastic汇总
• 4.golang批量写入elasticsearch
• 5.go中 elastic 聚合统计(1)
• 6.go中 elastic 聚合统计(2)
• 7.elasticsearch，golang客户端聚合查询
• 8.olivere/elasticsearch官方文档