[go-每日一库] golang olivere/elasticsearch/v6 的基本使用 -> 索引|文档常用操作及查询
准备工作
本文演示内容基于olivere/elasticsearch/v6
下载该库：
go get github.com/olivere/elastic/v6
初始化es
es初始化代码
func initES() *elastic.Client {
sniffopt := elastic.SetSniff(false) // 非集群下，关闭嗅探机制
urlopt := elastic.SetURL(URL) // url自行设置，比如我的 http://<user>:<passwd>@ip:9200
//authOpt := elastic.SetBasicAuth(USER, PASSWORD) // 不知道为何，olivere的库这样设置认证参数有问题，故把auth参数放到url中
checkOpt := elastic.SetHealthcheck(false)
options := []elastic.ClientOptionFunc{urlopt, sniffopt, checkOpt}
client, err := elastic.NewClient(options...)
if err != nil {
log.Fatal(err)
}
// check es conn
_, statusCode, err := client.Ping(URL).Do(context.Background())
if err != nil {
log.Fatal(err)
}
if statusCode == 200 {
log.Println("connect es success.")
}
return client
}
创建es结构体并创建new函数提供对外接口
es客户端结构体
type ElasticORM struct{
client *elastic.Client
index string
ctx context.Context
}
func NewElasticORM() *ElasticORM {
return &ElasticORM{
client: initES(),
ctx: context.Background(),
}
}
1.索引的增删改查
查看索引是否存在
// check index exist or not
func (es *ElasticORM) IsIndexExists(index string) bool {
exist, err := es.client.IndexExists(index).Do(es.ctx)
if err != nil {
log.Println(err)
}
return exist
}
获取索引mapping
// get index mapping
func (es *ElasticORM) GetIndexMappings(index string) interface{} {
mapping, err := es.client.GetMapping().Index(index).Do(es.ctx)
if err != nil {
log.Println(err)
return nil
}
return mapping
}
设置mapping
// put mapping
func (es *ElasticORM) PutIndexMapping(index string, mapping interface{}) bool {
var putResp *elastic.PutMappingResponse
var err error
switch reflect.TypeOf(mapping).Kind() {
case reflect.String:
putResp, err = es.client.PutMapping().Index(index).
Type("_doc").IgnoreUnavailable(true).
BodyString(mapping.(string)).Do(es.ctx)
case reflect.Map:
putResp, err = es.client.PutMapping().Index(index).
Type("_doc").IgnoreUnavailable(true).
BodyJson(mapping.(map[string]interface{})).Do(es.ctx)
}
if err != nil {
log.Println(err)
return false
}
if putResp.Acknowledged {
return true
}
return false
}
创建索引并设置settings
// create index, and put settings
func (es *ElasticORM) CreateIndex(index string, body interface{}) bool {
resp, err := es.client.CreateIndex(index).BodyJson(body.(map[string]interface{})).Do(es.ctx)
if err != nil {
log.Println(err)
}
if resp.Acknowledged {
return true
}
return false
}
删除索引
// delete index
func (es *ElasticORM) DeleteIndex(index string) bool {
resp, err := es.client.DeleteIndex(index).Do(es.ctx)
if err != nil {
log.Println(err)
return false
}
log.Println(resp.Acknowledged) // true
return true
}
2.文档的增删改查
插入文档
// insert data -> created
func (es *ElasticORM) InsertDocToIndex(index string, doc interface{}) bool {
resp, err := es.client.Index().Index(index).Type("_doc").
BodyJson(doc.(map[string]interface{})).Do(es.ctx)
if err != nil {
log.Println(err)
}
if resp.Result == "created" || resp.Result == "updated" {
log.Println(resp.Result)
}
return true
}
创建或更新文档
注意用update更新个别字段时，请用map结构，struct会更新全部的。
// create or update data, same doc, no operation
func (es *ElasticORM) CreateOrUpdateDoc(index string, id string, doc interface{}) bool {
resp, err := es.client.Update().Index(index).Type("_doc").Id(id).
Doc(doc).DocAsUpsert(true).Do(es.ctx)
if err != nil {
log.Println(err)
}
if resp.Result == "created" || resp.Result == "updated" || resp.Result == "noop" {
log.Println(resp.Result)
}
return true
}
批量插入
// bulk insert docs
func (es *ElasticORM) BulkInsertDocs(index string, docs []interface{}, bulkNum int) bool {
length := len(docs)
// case-1, no need to care add, just add and do
if length < bulkNum {
bulkReq := es.client.Bulk()
for i:=0; i<length; i++ {
doc := docs[i]
req := elastic.NewBulkIndexRequest().Index(index).Type("_doc").Doc(doc)
bulkReq = bulkReq.Add(req)
}
_, err := bulkReq.Do(es.ctx)
if err != nil {
log.Println(err)
return false
}
return true
}
// case-2, length > bulkNum
idx := 0
for {
bulkReq := es.client.Bulk()
for i:=0; i<bulkNum; i++ {
doc := docs[i]
req := elastic.NewBulkIndexRequest().Index(index).Type("_doc").Doc(doc)
bulkReq = bulkReq.Add(req)
idx++
if idx >= length {
break
}
}
_, err := bulkReq.Do(es.ctx)
if err != nil {
log.Println(err)
return false
}
if idx >= length {
break
}
}
return true
}
删除文档
// delete doc
func (es *ElasticORM) DeleteDoc(index, id string) bool {
resp, err := es.client.Delete().Index(index).Id(id).
Type("_doc").Do(es.ctx)
if err != nil {
log.Println(err)
return false
}
if resp.Result == "deleted" {
log.Println(resp.Result)
}
return true
}
获取单个文档
// get specific doc
func (es *ElasticORM) GetDoc(index string, docID string) interface{} {
resp, err := es.client.Get().Index(index).Type("_doc").Id(docID).Do(es.ctx)
if err != nil {
log.Println(err)
}
return resp.Source // *json.RawMessage
}
3.数据的bool查询 && 聚合查询
通过构造NewxxxQuery
精确字段查询-个值或数组
// advanced query
// single must cond, default size=10000
func (es *ElasticORM) MustTermOrTermsQuery(index string, name string, val interface{}) []*elastic.SearchHit {
query := elastic.NewBoolQuery()
if reflect.TypeOf(val).Kind() == reflect.Slice {
query.Must(elastic.NewTermsQuery(name, val))
} else {
query.Must(elastic.NewTermsQuery(name, val))
}
resp, err := es.client.Search().Index(index).Query(query).
Type("_doc").Size(10000).Do(es.ctx)
if err != nil {
log.Println(err)
}
return resp.Hits.Hits
}
范围查询
// single must-range cond, deafult size=10000
func (es *ElasticORM) MustRangeQuery(index string, name string, lte interface{}, gte interface{}) []*elastic.SearchHit {
query := elastic.NewBoolQuery()
query.Must(elastic.NewRangeQuery(name).Gte(gte).Lte(lte)) // must_not/filter类似，故不再赘述
resp, err := es.client.Search().Index(index).
Query(query).Type("_doc").
Size(10000).Do(es.ctx)
if err != nil {
log.Println(err)
}
return resp.Hits.Hits
}
指定数量返回
// specify size
func (es *ElasticORM) SizeQuery(index string, size int) []*elastic.SearchHit {
resp, err := es.client.Search().Index(index).
Type("_doc").Size(size).Do(es.ctx)
if err != nil {
log.Println(err)
}
return resp.Hits.Hits
}
指定返回字段
// specify source fields
func (es *ElasticORM) SourceQuery(index string, source interface{}) []*elastic.SearchHit {
resp, err := es.client.Search().Index(index).
Source(source).Type("_doc").
Size(10000).Do(es.ctx)
if err != nil {
log.Println(err)
}
return resp.Hits.Hits
}
指定顺序查询
// specify sort order
func (es *ElasticORM) SortQuery(index string, field string, ascend bool) []*elastic.SearchHit {
resp, err := es.client.Search().Index(index).Size(10000).
Sort(field, ascend).Type("_doc").Do(es.ctx)
if err != nil {
log.Println(err)
}
return resp.Hits.Hits
}
复杂查询
所谓复杂，无非是多种条件加一起，通过接口提供的不断链式添加即可。
// some complicated query, e.g.
func (es *ElasticORM) ComplicatedQueryForEg(index string) []*elastic.SearchHit {
query := elastic.NewBoolQuery()
query.Must(
elastic.NewTermQuery("name-1", "val"),
elastic.NewTermQuery("name-2", 100),
elastic.NewTermsQuery("name-3", []string{"val-1", "val-2"}),
elastic.NewRangeQuery("name-4").Gte(10).Lte(1000),
)
query.Filter(
elastic.NewTermQuery("name-5", "val"),
elastic.NewTermQuery("name-6", 100),
elastic.NewTermsQuery("name-7", []string{"val-1", "val-2"}),
)
// must_not/filter like above
resp, err := es.client.Search().Index(index).
Query(query).Type("_doc").Do(es.ctx)
if err != nil {
log.Println(err)
}
return resp.Hits.Hits
}
简单桶聚合
// aggregation query -> terms bucket
func (es *ElasticORM) AggregationTerms(index string, name string, field string) interface{} {
agg := elastic.NewTermsAggregation().Field(field).Size(10000)
resp, err := es.client.Search().
Index(index).
Aggregation(name, agg).
Type("_doc").
Do(es.ctx)
if err != nil {
log.Println(err)
}
ret, ok := resp.Aggregations.Terms(name)
if !ok {
log.Println("agg results is nil")
}
bucketKeys := make([]string, len(ret.Buckets))
for i, _ := range ret.Buckets {
bucketKeys[i] = ret.Buckets[i].Key.(string)
}
return bucketKeys
}
多级聚合查询
由于多级聚合不像kibana或者python操作，可以在一级的buckets下再往下buckets，索引多查几次实现多级查找。或者在一级buckets中找Aggregations的terms，然后遍历获取具体的key。
// aggregation query -> subAggs terms
func (es *ElasticORM) AggregationTermsWithSubAgg(index string, name1 string, field1 string, name2 string, field2 string) interface{} {
agg := elastic.NewTermsAggregation().Field(field1).Size(10000)
resp, err := es.client.Search().
Index(index).
Type("_doc").
Size(0).
Aggregation(name1, agg).
Do(es.ctx)
if err != nil {
log.Println(err)
}
ret, ok := resp.Aggregations.Terms(name1)
if !ok {
log.Println("agg results is nil")
}
fieldsMap := make(map[string][]interface{})
for i, _ := range ret.Buckets {
k := ret.Buckets[i].Key.(string)
// subAggregation
agg = elastic.NewTermsAggregation().Field(field2).Size(10000)
resp, err = es.client.Search().
Index(index).
Type("_doc").
Size(0).
Query(elastic.NewBoolQuery().Must(elastic.NewTermQuery(field1, k))).
Aggregation(name2, agg).
Do(es.ctx)
if err != nil {
log.Println(err)
}
ret2, _ := resp.Aggregations.Terms(name2)
for j, _ := range ret2.Buckets {
fieldsMap[k] = append(fieldsMap[k], ret2.Buckets[j].Key.(interface{}))
}
}
return fieldsMap
}
两级terms usage
// agg sub
func (es *ElasticORM) AggSub(index string, name1 string, field1 string, name2 string, field2 string) interface{} {
resp, err := es.client.Search(index).
Type("_doc").
Aggregation(
"mainAgg",
elastic.NewTermsAggregation().Field(field1).
SubAggregation(
"sub_1", elastic.NewTermsAggregation().Field(field2),
),
).
Size(0).
Pretty(true).
Do(es.ctx)
if err != nil {
log.Println(err)
}
ret, _ := resp.Aggregations.Terms("mainAgg")
for i, v := range ret.Buckets {
log.Println("first level buckets key:", ret.Buckets[i].Key)
ret2, _ := v.Aggregations.Terms("sub_1")
for j, _ := range ret2.Buckets {
log.Println("second level buckets key:", ret2.Buckets[j].Key)
}
}
return nil
}
聚合stats-max
// aggregation query -> stats-max
func (es *ElasticORM) AggregationMax(index string, field string) interface{} {
agg := elastic.NewMaxAggregation().Field(field)
resp, err := es.client.Search().
Index(index).
Type("_doc").
Aggregation("max_"+field, agg).
Do(es.ctx)
if err != nil {
log.Println(err)
}
ret, _ := resp.Aggregations.Max("max_"+field)
ans := *ret.Value
return ans
}
通过json或者map的source
代码：
queryStr := `{
"size": 2,
"query": {
"bool": {
"must": [
{"term": {
"scores": 0
}}
]
}
}
}`
var sourceMap map[string]interface{}
json.Unmarshal([]byte(queryStr), &sourceMap)
searchRes, err := client.Search("anomalies_scores").Type("_doc").Source(sourceMap).Do(context.Background())
参考文档：
