应用NEST将数据批量导入Elasticsearch
.net下将关系数据库的数据批量导入Elasticsearch中。
Elasticsearch使用版本的是7.x
下面是主要应用到的方法,代码参考:Elasticsearch.Net、Nest批量插入BulkAll,感谢作者。
public static bool BulkAll<T>(IElasticClient elasticClient, IndexName indexName, IEnumerable<T> list) where T : class
{
const int size = 1000;
var tokenSource = new CancellationTokenSource();
var observableBulk = elasticClient.BulkAll(list, f => f
.MaxDegreeOfParallelism(8)
.BackOffTime(TimeSpan.FromSeconds(10))
.BackOffRetries(2)
.Size(size)
.RefreshOnCompleted()
.Index(indexName)
.BufferToBulk((r, buffer) => r.IndexMany(buffer))
, tokenSource.Token);
var countdownEvent = new CountdownEvent(1);
Exception exception = null;
void OnCompleted()
{
WriteLine("BulkAll Finished");
countdownEvent.Signal();
}
var bulkAllObserver = new BulkAllObserver(
onNext: response =>
{
WriteLine($"Indexed {response.Page * size} with {response.Retries} retries");
},
onError: ex =>
{
WriteLine("BulkAll Error : {0}", ex);
exception = ex;
countdownEvent.Signal();
},
OnCompleted);
observableBulk.Subscribe(bulkAllObserver);
countdownEvent.Wait(tokenSource.Token);
if (exception != null)
{
WriteLine("BulkHotelGeo Error : {0}", exception);
return false;
}
else
{
return true;
}
}
本文为原创文章,想了解更多信息欢迎访问我的博客
https://www.cnblogs.com/ihappycat/