ElasticSearch 搜索原理
运行结果:返回5条数据
参考代码ESTestDocumentAPI.java
package com.dajiangtai.djt_spider.elasticsearch; import java.io.IOException; import java.net.InetAddress; import java.net.UnknownHostException; import java.util.Date; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import static org.elasticsearch.node.NodeBuilder.*; import static org.elasticsearch.common.xcontent.XContentFactory.*; import org.elasticsearch.action.bulk.BackoffPolicy; import org.elasticsearch.action.bulk.BulkProcessor; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.codehaus.jackson.map.ObjectMapper; import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.delete.DeleteRequestBuilder; import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.get.MultiGetItemResponse; import org.elasticsearch.action.get.MultiGetResponse; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.client.Client; import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.InetSocketTransportAddress; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.node.Node; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.SearchHits; import org.junit.Before; import org.junit.Test; /** * Document API 操作 * * @author 大讲台 * */ public class ESTestDocumentAPI { private TransportClient client; @Before public void test0() throws UnknownHostException { // 开启client.transport.sniff功能，探测集群所有节点 Settings settings = Settings.settingsBuilder() .put("cluster.name", "escluster") .put("client.transport.sniff", true).build(); // on startup // 获取TransportClient client = TransportClient .builder() .settings(settings) .build() .addTransportAddress( new InetSocketTransportAddress(InetAddress .getByName("master"), 9300)) .addTransportAddress( new InetSocketTransportAddress(InetAddress .getByName("slave1"), 9300)) .addTransportAddress( new InetSocketTransportAddress(InetAddress .getByName("slave2"), 9300)); } /** * 创建索引：use ElasticSearch helpers * * @throws IOException */ @Test public void test1() throws IOException { IndexResponse response = client .prepareIndex("twitter", "tweet", "1") .setSource( jsonBuilder().startObject().field("user", "kimchy") .field("postDate", new Date()) .field("message", "trying out Elasticsearch") .endObject()).get(); System.out.println(response.getId()); client.close(); } /** * 创建索引：do it yourself * * @throws IOException */ @Test public void test2() throws IOException { String json = "{" + "\"user\":\"kimchy\"," + "\"postDate\":\"2013-01-30\"," + "\"message\":\"trying out Elasticsearch\"" + "}"; IndexResponse response = client.prepareIndex("twitter", "tweet") .setSource(json).get(); System.out.println(response.getId()); client.close(); } /** * 创建索引：use map * * @throws IOException */ @Test public void test3() throws IOException { Map<String, Object> json = new HashMap<String, Object>(); json.put("user", "kimchy"); json.put("postDate", new Date()); json.put("message", "trying out Elasticsearch"); IndexResponse response = client.prepareIndex("twitter", "tweet") .setSource(json).get(); System.out.println(response.getId()); client.close(); } /** * 创建索引：serialize your beans * * @throws IOException */ @Test public void test4() throws IOException { User user = new User(); user.setUser("kimchy"); user.setPostDate(new Date()); user.setMessage("trying out Elasticsearch"); // instance a json mapper ObjectMapper mapper = new ObjectMapper(); // create once, reuse // generate json byte[] json = mapper.writeValueAsBytes(user); IndexResponse response = client.prepareIndex("twitter", "tweet") .setSource(json).get(); System.out.println(response.getId()); client.close(); } /** * 查询索引：get * * @throws IOException */ @Test public void test5() throws IOException { GetResponse response = client.prepareGet("twitter", "tweet", "1").get(); System.out.println(response.getSourceAsString()); client.close(); } /** * 删除索引：delete * * @throws IOException */ @Test public void test6() throws IOException { client.prepareDelete("twitter", "tweet", "1").get(); client.close(); } /** * 更新索引：Update API-UpdateRequest * * @throws IOException * @throws ExecutionException * @throws InterruptedException */ @Test public void test7() throws IOException, InterruptedException, ExecutionException { UpdateRequest updateRequest = new UpdateRequest(); updateRequest.index("twitter"); updateRequest.type("tweet"); updateRequest.id("AVyi3OORot7zkId708s8"); updateRequest.doc(jsonBuilder().startObject().field("gender", "male") .endObject()); client.update(updateRequest).get(); System.out.println(updateRequest.version()); client.close(); } /** * 更新索引：Update API-prepareUpdate()-doc * * @throws IOException * @throws ExecutionException * @throws InterruptedException */ @Test public void test8() throws IOException, InterruptedException, ExecutionException { client.prepareUpdate("twitter", "tweet", "AVyikSKIot7zkId708s6") .setDoc(jsonBuilder().startObject().field("gender", "female") .endObject()).get(); client.close(); } /** * 更新索引：Update API-prepareUpdate()-script * 需要开启：script.engine.groovy.inline.update: on * * @throws IOException * @throws ExecutionException * @throws InterruptedException */ @Test public void test9() throws IOException, InterruptedException, ExecutionException { client.prepareUpdate("twitter", "tweet", "AVyi4oZfot7zkId708s-") .setScript( new Script("ctx._source.gender = \"female\"", ScriptService.ScriptType.INLINE, null, null)) .get(); client.close(); } /** * 更新索引：Update API-UpdateRequest-upsert * * @throws IOException * @throws ExecutionException * @throws InterruptedException */ @Test public void test10() throws IOException, InterruptedException, ExecutionException { IndexRequest indexRequest = new IndexRequest("twitter", "tweet", "1") .source(jsonBuilder() .startObject() .field("name", "Joe Smith") .field("gender", "male") .endObject()); UpdateRequest updateRequest = new UpdateRequest("twitter", "tweet", "1") .doc(jsonBuilder() .startObject() .field("gender", "female") .endObject()).upsert(indexRequest); client.update(updateRequest).get(); client.close(); } /** * 批量查询索引：Multi Get API * * @throws IOException * @throws ExecutionException * @throws InterruptedException */ @Test public void test11() throws IOException, InterruptedException, ExecutionException { MultiGetResponse multiGetItemResponses = client.prepareMultiGet() .add("twitter", "tweet", "1") .add("twitter", "tweet", "AVyi4oZfot7zkId708s-", "AVyi3OORot7zkId708s8", "AVyikSKIot7zkId708s6") .add("djt2", "user", "1") .get(); for (MultiGetItemResponse itemResponse : multiGetItemResponses) { GetResponse response = itemResponse.getResponse(); if (response.isExists()) { String json = response.getSourceAsString(); System.out.println(json); } } client.close(); } /** * 批量操作索引：Bulk API * * @throws IOException * @throws ExecutionException * @throws InterruptedException */ @Test public void test12() throws IOException, InterruptedException, ExecutionException { BulkRequestBuilder bulkRequest = client.prepareBulk(); // either use client#prepare, or use Requests# to directly build index/delete requests bulkRequest.add(client.prepareIndex("twitter", "tweet", "3") .setSource(jsonBuilder() .startObject() .field("user", "kimchy") .field("postDate", new Date()) .field("message", "trying out Elasticsearch") .endObject() ) ); bulkRequest.add(client.prepareIndex("twitter", "tweet", "2") .setSource(jsonBuilder() .startObject() .field("user", "kimchy") .field("postDate", new Date()) .field("message", "another post") .endObject() ) ); DeleteRequestBuilder prepareDelete = client.prepareDelete("twitter", "tweet", "AVyikSKIot7zkId708s6"); bulkRequest.add(prepareDelete); BulkResponse bulkResponse = bulkRequest.get(); //批量操作：其中一个操作失败不影响其他操作成功执行 if (bulkResponse.hasFailures()) { // process failures by iterating through each bulk response item BulkItemResponse[] items = bulkResponse.getItems(); for (BulkItemResponse bulkItemResponse : items) { System.out.println(bulkItemResponse.getFailureMessage()); } }else{ System.out.println("bulk process success!"); } client.close(); } /** * 批量操作索引：Using Bulk Processor * 优化：先关闭副本，再添加副本，提升效率 * @throws IOException * @throws ExecutionException * @throws InterruptedException */ @Test public void test13() throws IOException, InterruptedException, ExecutionException { BulkProcessor bulkProcessor = BulkProcessor.builder( client, new BulkProcessor.Listener() { public void beforeBulk(long executionId, BulkRequest request) { // TODO Auto-generated method stub System.out.println(request.numberOfActions()); } public void afterBulk(long executionId, BulkRequest request, Throwable failure) { // TODO Auto-generated method stub System.out.println(failure.getMessage()); } public void afterBulk(long executionId, BulkRequest request, BulkResponse response) { // TODO Auto-generated method stub System.out.println(response.hasFailures()); } }) .setBulkActions(1000) // 每个批次的最大数量 .setBulkSize(new ByteSizeValue(1, ByteSizeUnit.GB))// 每个批次的最大字节数 .setFlushInterval(TimeValue.timeValueSeconds(5))// 每批提交时间间隔 .setConcurrentRequests(1) //设置多少个并发处理线程 //可以允许用户自定义当一个或者多个bulk请求失败后,该执行如何操作 .setBackoffPolicy( BackoffPolicy.exponentialBackoff(TimeValue.timeValueMillis(100), 3)) .build(); String json = "{" + "\"user\":\"kimchy\"," + "\"postDate\":\"2013-01-30\"," + "\"message\":\"trying out Elasticsearch\"" + "}"; for (int i = 0; i < 1000; i++) { bulkProcessor.add(new IndexRequest("djt6", "user").source(json)); } //阻塞至所有的请求线程处理完毕后，断开连接资源 bulkProcessor.awaitClose(3, TimeUnit.MINUTES); client.close(); } /** * SearchType使用方式 * @throws Exception */ @Test public void test14() throws Exception { SearchResponse response = client.prepareSearch("djt") .setTypes("user") .setSearchType(SearchType.DFS_QUERY_THEN_FETCH) // .setSearchType(SearchType.QUERY_AND_FETCH) .execute() .actionGet(); SearchHits hits = response.getHits(); System.out.println(hits.getTotalHits()); } }
先创建索引库djt1
curl -XPUT 'master:9200/djt1/user/1' -d '{"name":"tom","age":18, "info":"tom"}'
curl -XPUT 'master:9200/djt1/user/2' -d '{"name":"jack","age":29, "info":"jack"}'
curl -XPUT 'master:9200/djt1/user/3' -d '{"name":"jetty","age":18, "info":"jetty"}'
curl -XPUT 'master:9200/djt1/user/4' -d '{"name":"daival","age":19, "info":"daival"}'
curl -XPUT 'master:9200/djt1/user/5' -d '{"name":"lilei","age":18, "info":"lilei"}'
curl -XPUT 'master:9200/djt1/user/6' -d '{"name":"lili","age":29, "info":"lili"}'
curl -XPUT 'master:9200/djt1/user/7' -d '{"name":"tom1","age":18, "info":"tom1"}'
curl -XPUT 'master:9200/djt1/user/8' -d '{"name":"tom2","age":19, "info":"tom2"}'
curl -XPUT 'master:9200/djt1/user/9' -d '{"name":"tom3","age":11, "info":"tom3"}'
curl -XPUT 'master:9200/djt1/user/10' -d '{"name":"tom4","age":18, "info":"tom4"}'
curl -XPUT 'master:9200/djt1/user/11' -d '{"name":"tom5","age":18, "info":"tom5"}'
curl -XPUT 'master:9200/djt1/user/12' -d '{"name":"tom6","age":18, "info":"tom6"}'
curl -XPUT 'master:9200/djt1/user/13' -d '{"name":"tom7","age":18, "info":"tom7"}'
curl -XPUT 'master:9200/djt1/user/14' -d '{"name":"tom8","age":18, "info":"tom8"}'
curl -XPUT 'master:9200/djt1/user/15' -d '{"name":"tom9","age":18, "info":"tom9"}'
curl -XPUT 'master:9200/djt1/user/16' -d '{"name":"john","age":18, "info":"john"}'
curl -XPUT 'master:9200/djt1/user/17' -d '{"name":"marry","age":38, "info":"marry"}'
curl -XPUT 'master:9200/djt1/user/18' -d '{"name":"jjk1","age":18, "info":"jjk1"}'
执行程序（这里是条件匹配查询）
这里是把djt1库的所以数据都查出来
对多字段查询
模糊字符匹配查询方法
根据权重来查询
完全匹配查询
从0开始只查询2条数据
对年龄进行升序查询2条数据
查询某个年龄段的10条数据
参考代码 ESTestQuery.java
package com.dajiangtai.djt_spider.elasticsearch; import java.net.InetAddress; import java.net.UnknownHostException; import java.util.HashMap; import java.util.Map; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.InetSocketTransportAddress; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.sort.SortOrder; import org.junit.Before; import org.junit.Test; /** * Query 操作 * * @author 大讲台 * */ public class ESTestQuery { private TransportClient client; @Before public void test0() throws UnknownHostException { // 开启client.transport.sniff功能，探测集群所有节点 Settings settings = Settings.settingsBuilder() .put("cluster.name", "escluster") .put("client.transport.sniff", true).build(); // on startup // 获取TransportClient client = TransportClient .builder() .settings(settings) .build() .addTransportAddress( new InetSocketTransportAddress(InetAddress .getByName("master"), 9300)) .addTransportAddress( new InetSocketTransportAddress(InetAddress .getByName("slave1"), 9300)) .addTransportAddress( new InetSocketTransportAddress(InetAddress .getByName("slave2"), 9300)); } /** * 查询：query * 分页：from to * 排序：sort * 过滤：filter * * @throws Exception */ @Test public void test1() throws Exception { SearchRequestBuilder builder = client.prepareSearch("djt1"); builder//.setQuery(QueryBuilders.matchQuery("info", "marry and john")) .setQuery(QueryBuilders.matchAllQuery()) // .setQuery(QueryBuilders.multiMatchQuery("john", "name","info")) // .setQuery(QueryBuilders.queryStringQuery("name:tom*")) //.setQuery(QueryBuilders.boolQuery().should(QueryBuilders.matchQuery("name","tom").boost(3.0f)).should(QueryBuilders.matchQuery("age",18).boost(1.0f))) //.setQuery(QueryBuilders.termQuery("info", "tom")) .setFrom(0) .setSize(10) .addSort("age", SortOrder.ASC) .setPostFilter(QueryBuilders.rangeQuery("age").from(18).to(32)) .setExplain(false) ; SearchResponse searchResponse = builder.get(); SearchHits hits = searchResponse.getHits(); Map<String, Object> map = new HashMap<String, Object>(); SearchHit[] hits2 = hits.getHits(); for (SearchHit searchHit : hits2) { System.out.println(searchHit.getSource().toString()); } } }
