elasticsearch查询去重

实现查询去重、分页

例如：实现依据qid去重，createTime排序

DSL：

GET /nb_luban_answer/_search { "query": { "match": { "status": 1 } }, "sort": [ { "createTime": { "order": "desc" } } ],"aggs": { "qid": { "terms": { "field": "qid", "size": 10 },"aggs": { "rated": { "top_hits": { "sort": [{ "createTime": {"order": "desc"} }], "size": 1 } } } } }, "size": 0, "from": 0 } 

执行的结果：

{ "_shards": { "total": 4, "failed": 0, "successful": 4 }, "hits": { "hits": [], "total": 4, "max_score": 0 }, "took": 4, "timed_out": false, "aggregations": { "qid": { "doc_count_error_upper_bound": 0, "sum_other_doc_count": 0, "buckets": [ { "rated": { "hits": { "hits": [ { "_index": "nb_luban_answer", "_type": "luban_answer", "_source": { "img": "{\r\n \"img1\" : \"11111'><SCRIPT>alert(11111)</SCRIPT>'\",\r\n \"img2\" : \"2222'><SCRIPT>alert(11111)</SCRIPT>'\",\r\n \"img3\" : \"33333'><SCRIPT>alert(11111)</SCRIPT>'\"\r\n}", "pin": "motai869", "createTime": "2017-03-09 00:00:00", "id": "SN-0000001792-9-1", "tableNameSuffix": "1", "qid": "SN-0000001536-9-3", "content": "888'><SCRIPT>alert(11111)</SCRIPT>'", "status": 1 }, "_id": "SN-0000001792-9-1", "sort": [ 1489017600000 ], "_score": null } ], "total": 2, "max_score": null } }, "doc_count": 2, "key": "SN-0000001536-9-3" }, { "rated": { "hits": { "hits": [ { "_index": "nb_luban_answer", "_type": "luban_answer", "_source": { "img": "{\r\n \"img1\" : \"jfs/t3184/361/7557351412/2685/17143f65/58b91982N4e71a5d8.jpg\",\r\n \"img2\" : \"jfs/t3184/361/7557351412/2685/17143f65/58b91982N4e71a5d8.jpg\",\r\n \"img3\" : \"jfs/t3184/361/7557351412/2685/17143f65/58b91982N4e71a5d8.jpg\"\r\n}", "pin": "motai869", "createTime": "2017-03-03 00:00:00", "id": "SN-0000001280-9-3", "tableNameSuffix": "3", "qid": "SN-0000000256-0-3", "content": "hahaha", "status": 1 }, "_id": "SN-0000001280-9-3", "sort": [ 1488499200000 ], "_score": null } ], "total": 1, "max_score": null } }, "doc_count": 1, "key": "SN-0000000256-0-3" }, { "rated": { "hits": { "hits": [ { "_index": "nb_luban_answer", "_type": "luban_answer", "_source": { "pin": "motai869", "createTime": "2017-03-02 00:00:00", "id": "SN-0000208128-8-1", "tableNameSuffix": "1", "qid": "SN-0000207872-9-3", "content": "hahaha", "status": 1 }, "_id": "SN-0000208128-8-1", "sort": [ 1488412800000 ], "_score": null } ], "total": 1, "max_score": null } }, "doc_count": 1, "key": "SN-0000207872-9-3" } ] } } } JAVA API:

String indexName="nb_luban_answer"; String typeName="luban_answer"; AggregationBuilder aggregation = AggregationBuilders .terms("agg").field("qid") .subAggregation( AggregationBuilders.topHits("top").addSort("createTime",SortOrder.DESC).setSize(1) ); SearchResponse sResponse = Tool.CLIENT.prepareSearch(indexName).setTypes(typeName) .setQuery(QueryBuilders.matchQuery("status",1)) .addSort("createTime", SortOrder.DESC) .addAggregation(aggregation) .execute().actionGet(); Terms agg = sResponse.getAggregations().get("agg"); for (Terms.Bucket entry : agg.getBuckets()) { String key = String.valueOf(entry.getKey()) ; long docCount = entry.getDocCount(); System.out.println("key:"+ key +" doc_count:"+ docCount); TopHits topHits= entry.getAggregations().get("top"); for (SearchHit hit : topHits.getHits()){ System.out.println(" -> id: "+ hit.getId()+" createTime: "+hit.getSource().get("createTime")); } }

ES中所有数据：

参考：Elasticsearch 5.x 字段折叠的使用点击打开链接