HttpClient 模拟Curl 调用elasticsearch操作索引

最近需要实现一个功能，批量删除索引中的doc,约有100W+条，想到了使用httpclient的方式，实现 curl来操作索引。

实现方式很简单，就是一般发送post请求，比较坑的是，一定要注意，参数后面要加换行\n，否则会报：

HTTP/1.1 400 Bad Request [Warning: 299 Elasticsearch-5.6.9-877a590 "Content type detection for rest requests is deprecated. Specify the content type using the [Content-Type] header." "Thu, 19 Nov 2020 07:39:03 GMT", content-type: application/json; charset=UTF-8, content-length: 235] org.apache.http.conn.BasicManagedEntity@5824a83d

另外，需要注意  contentType的设置，如果是使用es 的批量功能  

如：post  http://10.64.6.32:9200/my_test_index1/product/_bulk

参数：

{"delete":{"_type":"product","_id":"AXWJMZFP321l4tnBy7Z1"}}

{"delete":{"_type":"product","_id":"AXWJMZFP321l4tnBy7Z2"}}

上面参数是要换行的。

另外，对于批量操作content-type需要设置为：text/plain。其它操作一般设置:application/json即可。

 

下面是代码：

--------------------------参数后面一定要加换行\n-----------------------参数后面一定要加换行\n-----------------------参数后面一定要加换行\n-----------------------参数后面一定要加换行\n--------------------------

public class FileTest {
    public static void main(String[] args) throws IOException {
        // 读取的CSV文件
         String path = "C://deleteIndexDoc.csv";
         readCsv(path);
    }

    public static void readCsv(String path) {
        try {
            DataInputStream in = new DataInputStream(new FileInputStream(new File(path)));
            CSVReader csvReader = new CSVReader(new InputStreamReader(in, "GBK"), CSVParser.DEFAULT_SEPARATOR,
                    CSVParser.DEFAULT_QUOTE_CHARACTER, CSVParser.DEFAULT_ESCAPE_CHARACTER, 1);
            String[] strs;
            String url = "http://ip:9200/my_test_index/product/_bulk";
            int i = 0;
            StringBuilder deletBuild = new StringBuilder();
             while ((strs = csvReader.readNext()) != null) {
                 i=i+1;
                 String param = "{\"delete\":{\"_id\":\""+strs[0]+"\"}}\n";
                 if(i%100==0)
                 {
                     deletBuild.append(param);
                     sendRequest(url,deletBuild.toString());
                     deletBuild = new StringBuilder();
                     System.out.println("finished num:"+i);
                 }else
                 {
                     deletBuild.append(param);
                 }
             }
             if(deletBuild.length()>0)
             {
                 sendRequest(url,deletBuild.toString());
             }
            csvReader.close();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    public static void sendRequest(String url,String reqestParm) {
        System.out.println("reqestParm:\n"+reqestParm);
        JSONObject rsp = HttpRequestUtil.httpPost(url, reqestParm, false);
        System.out.println("rsp:"+rsp.toJSONString());
    }
}

HttpRequestUtil.java :

public static JSONObject httpPost(String url,String jsonParam, boolean noNeedResponse){

    //post请求返回结果
    DefaultHttpClient httpClient = new DefaultHttpClient();
    JSONObject jsonResult = null;
    HttpPost method = new HttpPost(url);
    String sdf = jsonParam.toString();
    String contentType = "text/plain";
    //String contentType = "application/json";
    try {
        if (null != jsonParam) {
            //解决中文乱码问题
            StringEntity entity = new StringEntity(jsonParam.toString(), "utf-8");
            entity.setContentEncoding("UTF-8");
            entity.setContentType(contentType);
            method.setEntity(entity);
        }
        HttpResponse result = httpClient.execute(method);
        url = URLDecoder.decode(url, "UTF-8");
        /**请求发送成功，并得到响应**/
        if (result.getStatusLine().getStatusCode() == 200) {
            String str = "";
            try {
                /**读取服务器返回过来的json字符串数据**/
                str = EntityUtils.toString(result.getEntity());
                if (noNeedResponse) {
                    return null;
                }
                /**把json字符串转换成json对象**/
                jsonResult = JSONObject.parseObject(str);
            } catch (Exception e) {
                e.printStackTrace();
                System.out.println("post请求提交失败1:" + url);
            }
        }
    } catch (IOException e) {
        e.printStackTrace();
        System.out.println("post请求提交失败:" + url);
    }
    return jsonResult;
}