HttpClient 模拟Curl 调用elasticsearch操作索引
最近需要实现一个功能,批量删除索引中的doc,约有100W+条,想到了使用httpclient的方式,实现 curl来操作索引。
实现方式很简单,就是一般发送post请求,比较坑的是,一定要注意,参数后面要加换行\n,否则会报:
HTTP/1.1 400 Bad Request [Warning: 299 Elasticsearch-5.6.9-877a590 "Content type detection for rest requests is deprecated. Specify the content type using the [Content-Type] header." "Thu, 19 Nov 2020 07:39:03 GMT", content-type: application/json; charset=UTF-8, content-length: 235] org.apache.http.conn.BasicManagedEntity@5824a83d
另外,需要注意 contentType的设置,如果是使用es 的批量功能
如:post http://10.64.6.32:9200/my_test_index1/product/_bulk
参数:
{"delete":{"_type":"product","_id":"AXWJMZFP321l4tnBy7Z1"}}
{"delete":{"_type":"product","_id":"AXWJMZFP321l4tnBy7Z2"}}
上面参数是要换行的。
另外,对于批量操作content-type需要设置为:text/plain。其它操作一般设置:application/json即可。
下面是代码:
--------------------------参数后面一定要加换行\n-----------------------参数后面一定要加换行\n-----------------------参数后面一定要加换行\n-----------------------参数后面一定要加换行\n--------------------------
public class FileTest {
public static void main(String[] args) throws IOException {
// 读取的CSV文件
String path = "C://deleteIndexDoc.csv";
readCsv(path);
}
public static void readCsv(String path) {
try {
DataInputStream in = new DataInputStream(new FileInputStream(new File(path)));
CSVReader csvReader = new CSVReader(new InputStreamReader(in, "GBK"), CSVParser.DEFAULT_SEPARATOR,
CSVParser.DEFAULT_QUOTE_CHARACTER, CSVParser.DEFAULT_ESCAPE_CHARACTER, 1);
String[] strs;
String url = "http://ip:9200/my_test_index/product/_bulk";
int i = 0;
StringBuilder deletBuild = new StringBuilder();
while ((strs = csvReader.readNext()) != null) {
i=i+1;
String param = "{\"delete\":{\"_id\":\""+strs[0]+"\"}}\n";
if(i%100==0)
{
deletBuild.append(param);
sendRequest(url,deletBuild.toString());
deletBuild = new StringBuilder();
System.out.println("finished num:"+i);
}else
{
deletBuild.append(param);
}
}
if(deletBuild.length()>0)
{
sendRequest(url,deletBuild.toString());
}
csvReader.close();
} catch (Exception e) {
e.printStackTrace();
}
}
public static void sendRequest(String url,String reqestParm) {
System.out.println("reqestParm:\n"+reqestParm);
JSONObject rsp = HttpRequestUtil.httpPost(url, reqestParm, false);
System.out.println("rsp:"+rsp.toJSONString());
}
}
HttpRequestUtil.java :
public static JSONObject httpPost(String url,String jsonParam, boolean noNeedResponse){
//post请求返回结果
DefaultHttpClient httpClient = new DefaultHttpClient();
JSONObject jsonResult = null;
HttpPost method = new HttpPost(url);
String sdf = jsonParam.toString();
String contentType = "text/plain";
//String contentType = "application/json";
try {
if (null != jsonParam) {
//解决中文乱码问题
StringEntity entity = new StringEntity(jsonParam.toString(), "utf-8");
entity.setContentEncoding("UTF-8");
entity.setContentType(contentType);
method.setEntity(entity);
}
HttpResponse result = httpClient.execute(method);
url = URLDecoder.decode(url, "UTF-8");
/**请求发送成功,并得到响应**/
if (result.getStatusLine().getStatusCode() == 200) {
String str = "";
try {
/**读取服务器返回过来的json字符串数据**/
str = EntityUtils.toString(result.getEntity());
if (noNeedResponse) {
return null;
}
/**把json字符串转换成json对象**/
jsonResult = JSONObject.parseObject(str);
} catch (Exception e) {
e.printStackTrace();
System.out.println("post请求提交失败1:" + url);
}
}
} catch (IOException e) {
e.printStackTrace();
System.out.println("post请求提交失败:" + url);
}
return jsonResult;
}