elasticsearch 在.net core中使用Script脚本批量更新
目的:
目前有一家供应商数据有110w条数据,需要每天更新全部数据的Createtime创建时间(创建时间到秒不能全部都是一样的),如果使用es游标或者使用分页查询出来再更新,方案是行不通的,这时想到使用Script脚本来批量更新(半小时内全部更新完成),代码如下所示:
/// <summary> /// 定时作业调用,设置xx电子物料创建时间每日更新 /// </summary> /// <returns></returns> private void SetCreateTimeBySiTime() { int record = 0; short flag = (short)QuoteEnum.SupplierFlag.SiTime; //short flag = (short)QuoteEnum.SupplierFlag.Mobius; //当月的索引中取数据 var count = _supplierQuoteMonthEsClient.Count(c => c.Term(t => t.SupplierFlag, flag)); if (count.Count == 0) { LogUtil.NLog.Error("未查询到xx电子当月物料数据,更新时间失败!"); return ; } while (true) { //昨天的最大时间 var maxDT = DateTime.Parse(DateTime.Now.AddDays(-1).ToString("yyyy-MM-dd") + " 23:59:59"); var maxDTStrmp = TimeStampUtil.GenerateTimeStampBySeconds(maxDT); var count2 = _supplierQuoteMonthEsClient .Count(c => c.Bool(b => b.Must(m => m.Term(t => t.SupplierFlag, flag) && m.LongRange(t => t.Field(f => f.CreateTimeStamp).LessThanOrEquals(maxDTStrmp))))); if (count2.Count == 0) { break; } //当前时间 var currentDT = DateTime.Now; var currentDTStrmp = TimeStampUtil.GenerateTimeStampBySeconds(currentDT); var indexName = EsIndexExtend.Quote.SUPPLIER_QUOTE_MONTH; var updateByQuery = new UpdateByQueryRequest<SupplierQuoteEsModel>(indexName); //查询昨天的物料数据 updateByQuery.Query = new QueryContainer(new BoolQuery() { Must = new List<QueryContainer>() { new TermQuery() { Field = new Field("SupplierFlag"),Value = flag }, new LongRangeQuery() { Field = new Field("CreateTimeStamp"), LessThan = maxDTStrmp }, } }); updateByQuery.Script = new ScriptDescriptor() .Source($"ctx._source.CreateTimeStamp = params.CreateTimeStamp;ctx._source.CreateTime = params.CreateTime;") .Params(new Dictionary<string, object>() { { "CreateTimeStamp",currentDTStrmp }, { "CreateTime",currentDT} }); updateByQuery.WaitForCompletion = true; //(可选,整数)支持操作的滚动请求的大小。默认为 1000。 updateByQuery.ScrollSize = 3000; //(可选,整数)要处理的最大文档数。默认为所有文档。当设置为小于或等于scroll_size 的值时,将不会使用滚动来检索操作的结果。 updateByQuery.MaximumDocuments = 3000; //版本冲突时继续,不返回409 updateByQuery.Conflicts = Conflicts.Proceed; var res = _supplierQuoteMonthEsClient.Client.UpdateByQuery(updateByQuery); if(!res.IsValid) LogUtil.NLog.Error("今天xx电子物料数据更新Es失败,"+res.DebugInformation); Thread.Sleep(5000); record++; } LogUtil.NLog.Info($"今天xx电子物料更新时间共:{record}批次!"); } }
对应dsl语句写法
get /supplier_quote-2024.02/_update_by_query?wait_for_completion=false { "query": { "term": { "SupplierFlag": { "value": 49 } } }, "script": { "source": "ctx._source['CreateTimeStamp']=1708920000;ctx._source['CreateTime']='2024-02-26T12:00:00'" } }