ElasticSearch封装（创建索引，删除索引，创建Mapping，批量插入，批量删除，搜索）

本文转载自查看原文 2013-01-22 16:48 17917 ElasticSearch/ 搜索引擎

接口定义

  1 package com.koolearn.framework.search.declare;
  2 
  3 import java.util.HashMap;
  4 import java.util.List;
  5 import java.util.Map;
  6 import org.elasticsearch.common.Nullable;
  7 import com.koolearn.framework.search.declare.MySearchOption.SearchLogic;
  8 
  9 /*new Object[] { "TESTNAME1,TESTNAME2"}会识别为一个搜索条件*/
 10 /*new Object[] { "TESTNAME1","TESTNAME2"}会识别为两个搜索条件*/
 11 public interface SearchService {
 12 
 13     // 批量更新数据，先删除，再插入，只需要传递新数据的差异键值
 14     boolean autoBulkUpdateData(String indexName, HashMap<String, Object[]> oldContentMap, HashMap<String, Object[]> newContentMap);
 15 
 16     // 批量删除数据，危险
 17     boolean bulkDeleteData(String indexName, HashMap<String, Object[]> contentMap);
 18 
 19     // 批量插入数据
 20     boolean bulkInsertData(String indexName, HashMap<String, Object[]> contentMap);
 21 
 22     // 批量更新数据，先删除，再插入，需要传递新数据的完整数据
 23     boolean bulkUpdateData(String indexName
 24             , HashMap<String, Object[]> oldContentMap, HashMap<String, Object[]> newContentMap);
 25 
 26     /*
 27      * 搜索
 28      * indexNames    索引名称
 29      * mustSearchContentMap    must内容HashMap
 30      * shouldSearchContentMap    should内容HashMap
 31      * from    从第几条记录开始（必须大于等于0）
 32      * offset    一共显示多少条记录（必须大于0）
 33      * sortField    排序字段名称
 34      * sortType    排序方式（asc，desc）
 35      * */
 36     List<Map<String, Object>> complexSearch(String[] indexNames
 37             ,@Nullable HashMap<String, Object[]> mustSearchContentMap
 38             ,@Nullable HashMap<String, Object[]> shouldSearchContentMap
 39             , int from, int offset, @Nullable String sortField, @Nullable String sortType);
 40 
 41     /*
 42      * 获得搜索结果总数
 43      * indexNames    索引名称
 44      * mustSearchContentMap    must内容HashMap
 45      * shouldSearchContentMap    should内容HashMap
 46      * */
 47     long getComplexCount(String[] indexNames
 48             ,@Nullable HashMap<String, Object[]> mustSearchContentMap
 49             ,@Nullable HashMap<String, Object[]> shouldSearchContentMap);
 50 
 51     /*
 52      * 获得搜索结果总数，支持json版本
 53      * */
 54     long getCount(String[] indexNames, byte[] queryString);
 55 
 56     /*
 57      * 获得搜索结果总数
 58      * indexNames    索引名称
 59      * searchContentMap    搜索内容HashMap
 60      * filterContentMap    过滤内容HashMap
 61      * */
 62     long getCount(String[] indexNames
 63             , HashMap<String, Object[]> searchContentMap
 64             , @Nullable HashMap<String, Object[]> filterContentMap);
 65 
 66     /*
 67      * 获得搜索结果总数
 68      * indexNames    索引名称
 69      * searchContentMap    搜索内容HashMap
 70      * searchLogic    搜索条件之间的逻辑关系（must表示条件必须都满足，should表示只要有一个条件满足就可以）
 71      * filterContentMap    过滤内容HashMap
 72      * filterLogic    过滤条件之间的逻辑关系（must表示条件必须都满足，should表示只要有一个条件满足就可以）
 73      * */
 74     long getCount(String[] indexNames
 75             , HashMap<String, Object[]> searchContentMap, SearchLogic searchLogic
 76             , @Nullable HashMap<String, Object[]> filterContentMap, @Nullable SearchLogic filterLogic);
 77 
 78     //获得推荐列表
 79     List<String> getSuggest(String[] indexNames, String fieldName, String value, int count);
 80 
 81     /*
 82      * 分组统计
 83      * indexName    索引名称
 84      * mustSearchContentMap    must内容HashMap
 85      * shouldSearchContentMap    should内容HashMap
 86      * groupFields    分组字段
 87      * */
 88     Map<String, String> group(String indexName
 89             ,@Nullable HashMap<String, Object[]> mustSearchContentMap
 90             ,@Nullable HashMap<String, Object[]> shouldSearchContentMap
 91             , String[] groupFields);
 92 
 93     /*
 94      *     搜索，支持json版本
 95      * */
 96     List<Map<String, Object>> simpleSearch(String[] indexNames, byte[] queryString
 97             , int from, int offset, @Nullable String sortField, @Nullable String sortType);
 98 
 99     /*
100      * 搜索
101      * indexNames    索引名称
102      * searchContentMap    搜索内容HashMap
103      * filterContentMap    过滤内容HashMap
104      * from    从第几条记录开始（必须大于等于0）
105      * offset    一共显示多少条记录（必须大于0）
106      * sortField    排序字段名称
107      * sortType    排序方式（asc，desc）
108      * */
109     List<Map<String, Object>> simpleSearch(String[] indexNames
110             , HashMap<String, Object[]> searchContentMap
111             , @Nullable HashMap<String, Object[]> filterContentMap
112             , int from, int offset, @Nullable String sortField, @Nullable String sortType);
113 
114     /*
115      * 去掉排序参数的简化版本
116      * */
117     List<Map<String, Object>> simpleSearch(String[] indexNames
118             , HashMap<String, Object[]> searchContentMap
119             , @Nullable HashMap<String, Object[]> filterContentMap
120             , int from, int offset);
121 
122     /*
123      * 搜索
124      * indexNames    索引名称
125      * searchContentMap    搜索内容HashMap
126      * searchLogic    搜索条件之间的逻辑关系（must表示条件必须都满足，should表示只要有一个条件满足就可以）
127      * filterContentMap    过滤内容HashMap
128      * filterLogic    过滤条件之间的逻辑关系（must表示条件必须都满足，should表示只要有一个条件满足就可以）
129      * from    从第几条记录开始（必须大于等于0）
130      * offset    一共显示多少条记录（必须大于0）
131      * sortField    排序字段名称
132      * sortType    排序方式（asc，desc）
133      * */
134     List<Map<String, Object>> simpleSearch(String[] indexNames
135             , HashMap<String, Object[]> searchContentMap, SearchLogic searchLogic
136             , @Nullable HashMap<String, Object[]> filterContentMap, @Nullable SearchLogic filterLogic
137             , int from, int offset, @Nullable String sortField, @Nullable String sortType);
138 }

搜索选项定义

  1 package com.koolearn.framework.search.declare;
  2 
  3 import java.io.Serializable;
  4 import java.text.SimpleDateFormat;
  5 import java.util.Date;
  6 import java.util.regex.Pattern;
  7 
  8 public class MySearchOption implements Serializable {
  9     /**
 10      *
 11      */
 12     private static final long serialVersionUID = 1L;
 13     private SearchLogic searchLogic = SearchLogic.must;
 14     private SearchType searchType = SearchType.querystring;
 15     private DataFilter dataFilter = DataFilter.exists;
 16     /*querystring精度，取值[1-100]的整数*/
 17     private String queryStringPrecision = "100";
 18     /*排名权重*/
 19     private float boost = 1.0f;
 20     private boolean highlight = false;
 21 
 22     public enum SearchType {
 23         /*按照quert_string搜索，搜索非词组时候使用*/
 24         querystring
 25         /*按照区间搜索*/
 26         , range
 27         /*按照词组搜索，搜索一个词时候使用*/
 28         , term
 29     }
 30     public enum SearchLogic {
 31         /*逻辑must关系*/
 32         must
 33         /*逻辑should关系*/
 34         , should
 35     }
 36     public enum DataFilter {
 37         /*只显示有值的*/
 38         exists
 39         /*显示没有值的*/
 40         , notExists
 41         /*显示全部*/
 42         , all
 43     }
 44 
 45     public MySearchOption(SearchType searchType
 46             , SearchLogic searchLogic
 47             , String queryStringPrecision
 48             , DataFilter dataFilter
 49             , float boost
 50             , int highlight) {
 51         this.setSearchLogic(searchLogic);
 52         this.setSearchType(searchType);
 53         this.setQueryStringPrecision(queryStringPrecision);
 54         this.setDataFilter(dataFilter);
 55         this.setBoost(boost);
 56         this.setHighlight(highlight > 0 ? true : false);
 57     }
 58 
 59     public MySearchOption() {
 60     }
 61 
 62     public DataFilter getDataFilter() {
 63         return this.dataFilter;
 64     }
 65 
 66     public void setDataFilter(DataFilter dataFilter) {
 67         this.dataFilter = dataFilter;
 68     }
 69 
 70     public boolean isHighlight() {
 71         return this.highlight;
 72     }
 73 
 74     public void setHighlight(boolean highlight) {
 75         this.highlight = highlight;
 76     }
 77 
 78     public float getBoost() {
 79         return this.boost;
 80     }
 81 
 82     public void setBoost(float boost) {
 83         this.boost = boost;
 84     }
 85 
 86     public SearchLogic getSearchLogic() {
 87         return this.searchLogic;
 88     }
 89 
 90     public void setSearchLogic(SearchLogic searchLogic) {
 91         this.searchLogic = searchLogic;
 92     }
 93 
 94     public SearchType getSearchType() {
 95         return this.searchType;
 96     }
 97 
 98     public void setSearchType(SearchType searchType) {
 99         this.searchType = searchType;
100     }
101 
102     public String getQueryStringPrecision() {
103         return this.queryStringPrecision;
104     }
105 
106     public void setQueryStringPrecision(String queryStringPrecision) {
107         this.queryStringPrecision = queryStringPrecision;
108     }
109 
110     public static long getSerialversionuid() {
111         return MySearchOption.serialVersionUID;
112     }
113 
114     public static String formatDate(Object object) {
115         if (object instanceof java.util.Date) {
116             return MySearchOption.formatDateFromDate((java.util.Date)object);
117         }
118         return MySearchOption.formatDateFromString(object.toString());
119     }
120 
121     public static boolean isDate(Object object) {
122         return object instanceof java.util.Date || Pattern.matches("[1-2][0-9][0-9][0-9]-[0-9][0-9].*", object.toString());
123     }
124 
125     public static String formatDateFromDate(Date date) {
126         SimpleDateFormat dateFormat_hms = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
127         SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
128         try {
129             String result = dateFormat_hms.format(date);
130             return result;
131         }
132         catch (Exception e) {
133         }
134         try {
135             String result = dateFormat.format(date) + "00:00:00";
136             return result;
137         }
138         catch (Exception e) {
139         }
140         return dateFormat_hms.format(new Date());
141     }
142 
143     public static String formatDateFromString(String date) {
144         SimpleDateFormat dateFormat_hms = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
145         SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
146         try {
147             Date value = dateFormat_hms.parse(date);
148             return MySearchOption.formatDateFromDate(value);
149         }
150         catch (Exception e) {
151         }
152         try {
153             Date value = dateFormat.parse(date);
154             return MySearchOption.formatDateFromDate(value);
155         }
156         catch (Exception e) {
157         }
158         return dateFormat_hms.format(new Date());
159     }
160 }

搜索实现定义

  1 package com.koolearn.framework.search.implement;
  2 
  3 import java.io.IOException;
  4 import java.util.ArrayList;
  5 import java.util.Arrays;
  6 import java.util.HashMap;
  7 import java.util.Iterator;
  8 import java.util.List;
  9 import java.util.Map;
 10 import java.util.Map.Entry;
 11 import java.util.regex.Pattern;
 12 import org.apache.commons.collections.CollectionUtils;
 13 import org.apache.log4j.Logger;
 14 import org.elasticsearch.action.bulk.BulkRequestBuilder;
 15 import org.elasticsearch.action.bulk.BulkResponse;
 16 import org.elasticsearch.action.search.SearchRequestBuilder;
 17 import org.elasticsearch.action.search.SearchResponse;
 18 import org.elasticsearch.action.search.SearchType;
 19 import org.elasticsearch.action.suggest.SuggestResponse;
 20 import org.elasticsearch.client.Client;
 21 import org.elasticsearch.client.action.suggest.SuggestRequestBuilder;
 22 import org.elasticsearch.client.transport.TransportClient;
 23 import org.elasticsearch.common.Nullable;
 24 import org.elasticsearch.common.settings.ImmutableSettings;
 25 import org.elasticsearch.common.settings.Settings;
 26 import org.elasticsearch.common.transport.InetSocketTransportAddress;
 27 import org.elasticsearch.common.xcontent.XContentBuilder;
 28 import org.elasticsearch.common.xcontent.XContentFactory;
 29 import org.elasticsearch.index.query.AndFilterBuilder;
 30 import org.elasticsearch.index.query.BoolQueryBuilder;
 31 import org.elasticsearch.index.query.ExistsFilterBuilder;
 32 import org.elasticsearch.index.query.FilterBuilders;
 33 import org.elasticsearch.index.query.NotFilterBuilder;
 34 import org.elasticsearch.index.query.QueryBuilder;
 35 import org.elasticsearch.index.query.QueryBuilders;
 36 import org.elasticsearch.index.query.QueryFilterBuilder;
 37 import org.elasticsearch.index.query.QueryStringQueryBuilder;
 38 import org.elasticsearch.index.query.RangeQueryBuilder;
 39 import org.elasticsearch.search.SearchHit;
 40 import org.elasticsearch.search.facet.FacetBuilders;
 41 import org.elasticsearch.search.facet.terms.TermsFacet;
 42 import org.elasticsearch.search.facet.terms.TermsFacetBuilder;
 43 import org.elasticsearch.search.highlight.HighlightField;
 44 import org.springframework.beans.factory.DisposableBean;
 45 import org.springframework.beans.factory.InitializingBean;
 46 import com.koolearn.framework.common.utils.PropertiesConfigUtils;
 47 import com.koolearn.framework.search.declare.MySearchOption;
 48 import com.koolearn.framework.search.declare.MySearchOption.DataFilter;
 49 import com.koolearn.framework.search.declare.MySearchOption.SearchLogic;
 50 import com.koolearn.framework.search.declare.SearchService;
 51 
 52 public class SearchServiceImp implements SearchService, InitializingBean, DisposableBean {
 53     private List<String> clusterList = null;
 54     private Logger logger = Logger.getLogger(SearchServiceImp.class);
 55     private Client searchClient = null;
 56     private HashMap<String, String> searchClientConfigureMap = null;
 57     private String highlightCSS = "";
 58 
 59     public void setHighlightCSS(String highlightCSS) {
 60         this.highlightCSS = highlightCSS;
 61     }
 62 
 63     public void setSearchClientConfigureMap(HashMap<String, String> searchClientConfigureMap) {
 64         this.searchClientConfigureMap = searchClientConfigureMap;
 65     }
 66 
 67     private boolean _bulkInsertData(String indexName, XContentBuilder xContentBuilder) {
 68         try {
 69             BulkRequestBuilder bulkRequest = this.searchClient.prepareBulk();
 70             bulkRequest.add(this.searchClient.prepareIndex(indexName, indexName).setSource(xContentBuilder));
 71             BulkResponse bulkResponse = bulkRequest.execute().actionGet();
 72             if (!bulkResponse.hasFailures()) {
 73                 return true;
 74             }
 75             else {
 76                 this.logger.error(bulkResponse.buildFailureMessage());
 77             }
 78         }
 79         catch (Exception e) {
 80             this.logger.error(e.getMessage());
 81         }
 82         return false;
 83     }
 84 
 85     public void afterPropertiesSet() throws Exception {
 86         this.logger.info("连接搜索服务器");
 87         this.open();
 88     }
 89 
 90     public boolean bulkDeleteData(String indexName, HashMap<String, Object[]> contentMap) {
 91         try {
 92             QueryBuilder queryBuilder = null;
 93             queryBuilder = this.createQueryBuilder(contentMap, SearchLogic.must);
 94             this.logger.warn("[" + indexName + "]" + queryBuilder.toString());
 95             this.searchClient.prepareDeleteByQuery(indexName).setQuery(queryBuilder).execute().actionGet();
 96             return true;
 97         }
 98         catch (Exception e) {
 99             this.logger.error(e.getMessage());
100         }
101         return false;
102     }
103 
104     public boolean bulkInsertData(String indexName, HashMap<String, Object[]> insertContentMap) {
105         XContentBuilder xContentBuilder = null;
106         try {
107             xContentBuilder = XContentFactory.jsonBuilder().startObject();
108         }
109         catch (IOException e) {
110             this.logger.error(e.getMessage());
111             return false;
112         }
113         Iterator<Entry<String, Object[]>> iterator = insertContentMap.entrySet().iterator();
114         while (iterator.hasNext()) {
115             Entry<String, Object[]> entry = iterator.next();
116             String field = entry.getKey();
117             Object[] values = entry.getValue();
118             String formatValue = this.formatInsertData(values);
119             try {
120                 xContentBuilder = xContentBuilder.field(field, formatValue);
121             }
122             catch (IOException e) {
123                 this.logger.error(e.getMessage());
124                 return false;
125             }
126         }
127         try {
128             xContentBuilder = xContentBuilder.endObject();
129         }
130         catch (IOException e) {
131             this.logger.error(e.getMessage());
132             return false;
133         }
134         try {
135             this.logger.debug("[" + indexName + "]" + xContentBuilder.string());
136         }
137         catch (IOException e) {
138             this.logger.error(e.getMessage());
139         }
140         return this._bulkInsertData(indexName, xContentBuilder);
141     }
142 
143     public boolean bulkUpdateData(String indexName, HashMap<String, Object[]> oldContentMap, HashMap<String, Object[]> newContentMap) {
144         if (this.bulkDeleteData(indexName, oldContentMap)) {
145             return this.bulkInsertData(indexName, newContentMap);
146         }
147         this.logger.warn("删除数据失败");
148         return false;
149     }
150 
151     public boolean autoBulkUpdateData(String indexName, HashMap<String, Object[]> oldContentMap, HashMap<String, Object[]> newContentMap) {
152         try {
153             List<Map<String, Object>> searchResult = this.simpleSearch(new String[] { indexName }, oldContentMap, null, 0, 1, null, null);
154             if (searchResult == null || searchResult.size() == 0) {
155                 this.logger.warn("未找到需要更新的数据");
156                 return false;
157             }
158             if (!this.bulkDeleteData(indexName, oldContentMap)) {
159                 this.logger.warn("删除数据失败");
160                 return false;
161             }
162             HashMap<String, Object[]> insertContentMap = new HashMap<String, Object[]>();
163             for (Map<String, Object> contentMap : searchResult) {
164                 Iterator<Entry<String, Object>> oldContentIterator = contentMap.entrySet().iterator();
165                 while (oldContentIterator.hasNext()) {
166                     Entry<String, Object> entry = oldContentIterator.next();
167                     insertContentMap.put(entry.getKey(), new Object[] { entry.getValue() });
168                 }
169             }
170             Iterator<Entry<String, Object[]>> newContentIterator = newContentMap.entrySet().iterator();
171             while (newContentIterator.hasNext()) {
172                 Entry<String, Object[]> entry = newContentIterator.next();
173                 insertContentMap.put(entry.getKey(), entry.getValue());
174             }
175             return this.bulkInsertData(indexName, insertContentMap);
176         }
177         catch (Exception e) {
178             this.logger.error(e.getMessage());
179         }
180         return false;
181     }
182 
183     /*简单的值校验*/
184     private boolean checkValue(Object[] values) {
185         if (values == null) {
186             return false;
187         }
188         else if (values.length == 0) {
189             return false;
190         }
191         else if (values[0] == null) {
192             return false;
193         }
194         else if (values[0].toString().trim().isEmpty()) {
195             return false;
196         }
197         return true;
198     }
199 
200     private void close() {
201         if (this.searchClient == null) {
202             return;
203         }
204         this.searchClient.close();
205         this.searchClient = null;
206     }
207 
208     private RangeQueryBuilder createRangeQueryBuilder(String field, Object[] values) {
209         if (values.length == 1 || values[1] == null || values[1].toString().trim().isEmpty()) {
210             this.logger.warn("[区间搜索]必须传递两个值，但是只传递了一个值，所以返回null");
211             return null;
212         }
213         boolean timeType = false;
214         if (MySearchOption.isDate(values[0])) {
215             if (MySearchOption.isDate(values[1])) {
216                 timeType = true;
217             }
218         }
219         String begin = "", end = "";
220         if (timeType) {
221             /*
222              * 如果时间类型的区间搜索出现问题，有可能是数据类型导致的：
223              *     （1）在监控页面（elasticsearch-head）中进行range搜索，看看什么结果，如果也搜索不出来，则：
224              *     （2）请确定mapping中是date类型，格式化格式是yyyy-MM-dd HH:mm:ss
225              *    （3）请确定索引里的值是类似2012-01-01 00:00:00的格式
226              *    （4）如果是从数据库导出的数据，请确定数据库字段是char或者varchar类型，而不是date类型（此类型可能会有问题）
227              * */
228             begin = MySearchOption.formatDate(values[0]);
229             end = MySearchOption.formatDate(values[1]);
230         }
231         else {
232             begin = values[0].toString();
233             end = values[1].toString();
234         }
235         return QueryBuilders.rangeQuery(field).from(begin).to(end);
236     }
237 
238     /*
239      * 创建过滤条件
240      * */
241     private QueryBuilder createFilterBuilder(SearchLogic searchLogic, QueryBuilder queryBuilder, HashMap<String, Object[]> searchContentMap, HashMap<String, Object[]> filterContentMap) throws Exception
242     {
243         try {
244             Iterator<Entry<String, Object[]>> iterator = searchContentMap.entrySet().iterator();
245             AndFilterBuilder andFilterBuilder = null;
246             while (iterator.hasNext()) {
247                 Entry<String, Object[]> entry = iterator.next();
248                 Object[] values = entry.getValue();
249                 /*排除非法的搜索值*/
250                 if (!this.checkValue(values)) {
251                     continue;
252                 }
253                 MySearchOption mySearchOption = this.getSearchOption(values);
254                 if (mySearchOption.getDataFilter() == DataFilter.exists) {
255                     /*被搜索的条件必须有值*/
256                     ExistsFilterBuilder existsFilterBuilder = FilterBuilders.existsFilter(entry.getKey());
257                     if (andFilterBuilder == null) {
258                         andFilterBuilder = FilterBuilders.andFilter(existsFilterBuilder);
259                     }
260                     else {
261                         andFilterBuilder = andFilterBuilder.add(existsFilterBuilder);
262                     }
263                 }
264             }
265             if (filterContentMap == null || filterContentMap.isEmpty()) {
266                 /*如果没有其它过滤条件，返回*/
267                 return QueryBuilders.filteredQuery(queryBuilder, andFilterBuilder);
268             }
269             /*构造过滤条件*/
270             QueryFilterBuilder queryFilterBuilder = FilterBuilders.queryFilter(this.createQueryBuilder(filterContentMap, searchLogic));
271             /*构造not过滤条件，表示搜索结果不包含这些内容，而不是不过滤*/
272             NotFilterBuilder notFilterBuilder = FilterBuilders.notFilter(queryFilterBuilder);
273             return QueryBuilders.filteredQuery(queryBuilder, FilterBuilders.andFilter(andFilterBuilder, notFilterBuilder));
274         }
275         catch (Exception e) {
276             this.logger.error(e.getMessage());
277         }
278         return null;
279     }
280 
281     private QueryBuilder createSingleFieldQueryBuilder(String field, Object[] values, MySearchOption mySearchOption) {
282         try {
283             if (mySearchOption.getSearchType() == com.koolearn.framework.search.declare.MySearchOption.SearchType.range) {
284                 /*区间搜索*/
285                 return this.createRangeQueryBuilder(field, values);
286             }
287             //        String[] fieldArray = field.split(",");/*暂时不处理多字段[field1,field2,......]搜索情况*/
288             BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
289             for (Object valueItem : values) {
290                 if (valueItem instanceof MySearchOption) {
291                     continue;
292                 }
293                 QueryBuilder queryBuilder = null;
294                 String formatValue = valueItem.toString().trim().replace("*", "");//格式化搜索数据
295                 if (mySearchOption.getSearchType() == com.koolearn.framework.search.declare.MySearchOption.SearchType.term) {
296                     queryBuilder = QueryBuilders.termQuery(field, formatValue).boost(mySearchOption.getBoost());
297                 }
298                 else if (mySearchOption.getSearchType() == com.koolearn.framework.search.declare.MySearchOption.SearchType.querystring) {
299                     if (formatValue.length() == 1) {
300                         /*如果搜索长度为1的非数字的字符串，格式化为通配符搜索，暂时这样，以后有时间改成multifield搜索，就不需要通配符了*/
301                         if (!Pattern.matches("[0-9]", formatValue)) {
302                             formatValue = "*"+formatValue+"*";
303                         }
304                     }
305                     QueryStringQueryBuilder queryStringQueryBuilder = QueryBuilders.queryString(formatValue).minimumShouldMatch(mySearchOption.getQueryStringPrecision());
306                     queryBuilder = queryStringQueryBuilder.field(field).boost(mySearchOption.getBoost());
307                 }
308                 if (mySearchOption.getSearchLogic() == SearchLogic.should) {
309                     boolQueryBuilder = boolQueryBuilder.should(queryBuilder);
310                 }
311                 else {
312                     boolQueryBuilder = boolQueryBuilder.must(queryBuilder);
313                 }
314             }
315             return boolQueryBuilder;
316         }
317         catch (Exception e) {
318             this.logger.error(e.getMessage());
319         }
320         return null;
321     }
322 
323     /*
324      * 创建搜索条件
325      * */
326     private QueryBuilder createQueryBuilder(HashMap<String, Object[]> searchContentMap, SearchLogic searchLogic) {
327         try {
328             if (searchContentMap == null || searchContentMap.size() ==0) {
329                 return null;
330             }
331             BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
332             Iterator<Entry<String, Object[]>> iterator = searchContentMap.entrySet().iterator();
333             /*循环每一个需要搜索的字段和值*/
334             while (iterator.hasNext()) {
335                 Entry<String, Object[]> entry = iterator.next();
336                 String field = entry.getKey();
337                 Object[] values = entry.getValue();
338                 /*排除非法的搜索值*/
339                 if (!this.checkValue(values)) {
340                     continue;
341                 }
342                 /*获得搜索类型*/
343                 MySearchOption mySearchOption = this.getSearchOption(values);
344                 QueryBuilder queryBuilder = this.createSingleFieldQueryBuilder(field, values, mySearchOption);
345                 if (queryBuilder != null) {
346                     if (searchLogic == SearchLogic.should) {
347                         /*should关系，也就是说，在A索引里有或者在B索引里有都可以*/
348                         boolQueryBuilder = boolQueryBuilder.should(queryBuilder);
349                     }
350                     else {
351                         /*must关系，也就是说，在A索引里有，在B索引里也必须有*/
352                         boolQueryBuilder = boolQueryBuilder.must(queryBuilder);
353                     }
354                 }
355             }
356             return boolQueryBuilder;
357         }
358         catch (Exception e) {
359             this.logger.error(e.getMessage());
360         }
361         return null;
362     }
363 
364     public void destroy() throws Exception {
365         this.logger.info("关闭搜索客户端");
366         this.close();
367     }
368 
369     private String formatInsertData(Object[] values) {
370         if (!this.checkValue(values)) {
371             return "";
372         }
373         if (MySearchOption.isDate(values[0])) {
374             this.logger.warn("[" + values[0].toString() + "] formatDate");
375             return MySearchOption.formatDate(values[0]);
376         }
377         String formatValue = values[0].toString();
378         for (int index = 1; index < values.length; ++index) {
379             formatValue += "," + values[index].toString();
380         }
381         return formatValue.trim();
382     }
383 
384     public long getCount(String[] indexNames, HashMap<String, Object[]> searchContentMap, HashMap<String, Object[]> filterContentMap) {
385         SearchLogic searchLogic = indexNames.length > 1 ? SearchLogic.should : SearchLogic.must;
386         return this.getCount(indexNames, searchContentMap, searchLogic, filterContentMap, searchLogic);
387     }
388 
389     private SearchResponse searchCountRequest(String[] indexNames, Object queryBuilder) {
390         try {
391             SearchRequestBuilder searchRequestBuilder = this.searchClient.prepareSearch(indexNames).setSearchType(SearchType.COUNT);
392             if (queryBuilder instanceof QueryBuilder) {
393                 searchRequestBuilder = searchRequestBuilder.setQuery((QueryBuilder)queryBuilder);
394                 this.logger.debug(searchRequestBuilder.toString());
395             }
396             if (queryBuilder instanceof byte[]) {
397                 String query = new String((byte[])queryBuilder);
398                 searchRequestBuilder = searchRequestBuilder.setQuery(QueryBuilders.wrapperQuery(query));
399                 this.logger.debug(query);
400             }
401             return searchRequestBuilder.execute().actionGet();
402         }
403         catch (Exception e) {
404             this.logger.error(e.getMessage());
405         }
406         return null;
407     }
408 
409     public long getCount(String[] indexNames, byte[] queryString) {
410         try {
411             SearchResponse searchResponse = this.searchCountRequest(indexNames, queryString);
412             return searchResponse.hits().totalHits();
413         }
414         catch (Exception e) {
415             this.logger.error(e.getMessage());
416         }
417         return 0;
418     }
419 
420     /*获得搜索结果*/
421     private List<Map<String, Object>> getSearchResult(SearchResponse searchResponse) {
422         try {
423             List<Map<String, Object>> resultList = new ArrayList<Map<String, Object>>();
424             for (SearchHit searchHit : searchResponse.getHits()) {
425                 Iterator<Entry<String, Object>> iterator = searchHit.getSource().entrySet().iterator();
426                 HashMap<String, Object> resultMap = new HashMap<String, Object>();
427                 while (iterator.hasNext()) {
428                     Entry<String, Object> entry = iterator.next();
429                     resultMap.put(entry.getKey(), entry.getValue());
430                 }
431                 Map<String, HighlightField> highlightMap = searchHit.highlightFields();
432                 Iterator<Entry<String, HighlightField>> highlightIterator = highlightMap.entrySet().iterator();
433                 while (highlightIterator.hasNext()) {
434                     Entry<String, HighlightField> entry = highlightIterator.next();
435                     Object[] contents = entry.getValue().fragments();
436                     if (contents.length == 1) {
437                         resultMap.put(entry.getKey(), contents[0].toString());
438                         System.out.println(contents[0].toString());
439                     }
440                     else {
441                         this.logger.warn("搜索结果中的高亮结果出现多数据contents.length = " + contents.length);
442                     }
443                 }
444                 resultList.add(resultMap);
445             }
446             return resultList;
447         }
448         catch (Exception e) {
449             this.logger.error(e.getMessage());
450         }
451         return null;
452     }
453 
454     /*获得搜索选项*/
455     private MySearchOption getSearchOption(Object[] values) {
456         try {
457             for (Object item : values) {
458                 if (item instanceof MySearchOption) {
459                     return (MySearchOption) item;
460                 }
461             }
462         }
463         catch (Exception e) {
464             this.logger.error(e.getMessage());
465         }
466         return new MySearchOption();
467     }
468 
469     /*获得搜索建议
470      * 服务器端安装elasticsearch-plugin-suggest
471      * 客户端加入elasticsearch-plugin-suggest的jar包
472      * https://github.com/spinscale/elasticsearch-suggest-plugin
473      * */
474     public List<String> getSuggest(String[] indexNames, String fieldName, String value, int count) {
475         try {
476             SuggestRequestBuilder suggestRequestBuilder = new SuggestRequestBuilder(this.searchClient);
477             suggestRequestBuilder = suggestRequestBuilder.setIndices(indexNames).field(fieldName).term(value).size(count);//.similarity(0.5f);
478             SuggestResponse suggestResponse = suggestRequestBuilder.execute().actionGet();
479             return suggestResponse.suggestions();
480         }
481         catch (Exception e) {
482             this.logger.error(e.getMessage());
483         }
484         return null;
485     }
486 
487     /*
488      * 创建搜索客户端
489      * tcp连接搜索服务器
490      * 创建索引
491      * 创建mapping
492      * */
493     private void open() {
494         try {
495             /*如果10秒没有连接上搜索服务器，则超时*/
496             Settings settings = ImmutableSettings.settingsBuilder()
497                     .put(this.searchClientConfigureMap)
498 //                    .put("client.transport.ping_timeout", "10s")
499 //                    .put("client.transport.sniff", "true")
500 //                    .put("client.transport.ignore_cluster_name", "true")
501                     .build();
502             /*创建搜索客户端*/
503             this.searchClient = new TransportClient(settings);
504             if (CollectionUtils.isEmpty(this.clusterList)) {
505                 String cluster = PropertiesConfigUtils.getProperty("search.clusterList");
506                 if (cluster != null) {
507                     this.clusterList = Arrays.asList(cluster.split(","));
508                 }
509             }
510             for (String item : this.clusterList) {
511                 String address = item.split(":")[0];
512                 int port = Integer.parseInt(item.split(":")[1]);
513                 /*通过tcp连接搜索服务器，如果连接不上，有一种可能是服务器端与客户端的jar包版本不匹配*/
514                 this.searchClient = ((TransportClient) this.searchClient).addTransportAddress(new InetSocketTransportAddress(address, port));
515             }
516         }
517         catch (Exception e) {
518             this.logger.error(e.getMessage());
519         }
520     }
521 
522     public void setClusterList(List<String> clusterList) {
523         this.clusterList = clusterList;
524     }
525 
526     public List<Map<String, Object>> simpleSearch(String[] indexNames, HashMap<String, Object[]> searchContentMap, HashMap<String, Object[]> filterContentMap, int from, int offset) {
527         return this.simpleSearch(indexNames, searchContentMap, filterContentMap, from, offset, null, null);
528     }
529 
530     public List<Map<String, Object>> simpleSearch(String[] indexNames, HashMap<String, Object[]> searchContentMap, HashMap<String, Object[]> filterContentMap, int from, int offset, String sortField, String sortType)
531     {
532         SearchLogic searchLogic = indexNames.length > 1 ? SearchLogic.should : SearchLogic.must;
533         return this.simpleSearch(indexNames, searchContentMap, searchLogic, filterContentMap, searchLogic, from, offset, sortField, sortType);
534     }
535 
536     public long getComplexCount(String[] indexNames
537             , HashMap<String, Object[]> mustSearchContentMap
538             , HashMap<String, Object[]> shouldSearchContentMap) {
539         /*创建must搜索条件*/
540         QueryBuilder mustQueryBuilder = this.createQueryBuilder(mustSearchContentMap, SearchLogic.must);
541         /*创建should搜索条件*/
542         QueryBuilder shouldQueryBuilder = this.createQueryBuilder(shouldSearchContentMap, SearchLogic.should);
543         if (mustQueryBuilder == null && shouldQueryBuilder == null) {
544             return 0;
545         }
546         BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
547         if (mustQueryBuilder != null) {
548             boolQueryBuilder = boolQueryBuilder.must(mustQueryBuilder);
549         }
550         if (shouldQueryBuilder != null) {
551             boolQueryBuilder = boolQueryBuilder.must(shouldQueryBuilder);
552         }
553         try {
554             SearchResponse searchResponse = this.searchCountRequest(indexNames, boolQueryBuilder);
555             return searchResponse.hits().totalHits();
556         }
557         catch (Exception e) {
558             this.logger.error(e.getMessage());
559         }
560         return 0;
561     }
562 
563     public List<Map<String, Object>> complexSearch(String[] indexNames
564             , HashMap<String, Object[]> mustSearchContentMap
565             , HashMap<String, Object[]> shouldSearchContentMap
566             , int from, int offset, @Nullable String sortField, @Nullable String sortType) {
567         if (offset <= 0) {
568             return null;
569         }
570         /*创建must搜索条件*/
571         QueryBuilder mustQueryBuilder = this.createQueryBuilder(mustSearchContentMap, SearchLogic.must);
572         /*创建should搜索条件*/
573         QueryBuilder shouldQueryBuilder = this.createQueryBuilder(shouldSearchContentMap, SearchLogic.should);
574         if (mustQueryBuilder == null && shouldQueryBuilder == null) {
575             return null;
576         }
577         BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
578         if (mustQueryBuilder != null) {
579             boolQueryBuilder = boolQueryBuilder.must(mustQueryBuilder);
580         }
581         if (shouldQueryBuilder != null) {
582             boolQueryBuilder = boolQueryBuilder.must(shouldQueryBuilder);
583         }
584         try {
585             SearchRequestBuilder searchRequestBuilder = null;
586             searchRequestBuilder = this.searchClient.prepareSearch(indexNames).setSearchType(SearchType.DEFAULT)
587                     .setQuery(boolQueryBuilder).setFrom(from).setSize(offset).setExplain(true);
588             if (sortField == null || sortField.isEmpty() || sortType == null || sortType.isEmpty()) {
589                 /*如果不需要排序*/
590             }
591             else {
592                 /*如果需要排序*/
593                 org.elasticsearch.search.sort.SortOrder sortOrder = sortType.equals("desc") ? org.elasticsearch.search.sort.SortOrder.DESC : org.elasticsearch.search.sort.SortOrder.ASC;
594                 searchRequestBuilder = searchRequestBuilder.addSort(sortField, sortOrder);
595             }
596             searchRequestBuilder = this.createHighlight(searchRequestBuilder, mustSearchContentMap);
597             searchRequestBuilder = this.createHighlight(searchRequestBuilder, shouldSearchContentMap);
598             this.logger.debug(searchRequestBuilder.toString());
599             SearchResponse searchResponse = searchRequestBuilder.execute().actionGet();
600             return this.getSearchResult(searchResponse);
601         }
602         catch (Exception e) {
603             this.logger.error(e.getMessage());
604         }
605         return null;
606     }
607 
608     public List<Map<String, Object>> simpleSearch(String[] indexNames, byte[] queryString, int from, int offset, String sortField, String sortType) {
609         if (offset <= 0) {
610             return null;
611         }
612         try {
613             SearchRequestBuilder searchRequestBuilder = this.searchClient.prepareSearch(indexNames).setSearchType(SearchType.DEFAULT)
614                     .setFrom(from).setSize(offset).setExplain(true);
615             if (sortField == null || sortField.isEmpty() || sortType == null || sortType.isEmpty()) {
616                 /*如果不需要排序*/
617             }
618             else {
619                 /*如果需要排序*/
620                 org.elasticsearch.search.sort.SortOrder sortOrder = sortType.equals("desc") ? org.elasticsearch.search.sort.SortOrder.DESC : org.elasticsearch.search.sort.SortOrder.ASC;
621                 searchRequestBuilder = searchRequestBuilder.addSort(sortField, sortOrder);
622             }
623 
624             String query = new String(queryString);
625             searchRequestBuilder = searchRequestBuilder.setQuery(QueryBuilders.wrapperQuery(query));
626             this.logger.debug(query);
627 
628             SearchResponse searchResponse = searchRequestBuilder.execute().actionGet();
629             return this.getSearchResult(searchResponse);
630         }
631         catch (Exception e) {
632             this.logger.error(e.getMessage());
633         }
634         return null;
635     }
636 
637     private Map<String, String> _group(String indexName, QueryBuilder queryBuilder, String[] groupFields) {
638         try {
639             TermsFacetBuilder termsFacetBuilder = FacetBuilders.termsFacet("group").fields(groupFields).size(9999);
640             SearchRequestBuilder searchRequestBuilder = this.searchClient.prepareSearch(indexName).setSearchType(SearchType.DEFAULT)
641                     .addFacet(termsFacetBuilder).setQuery(queryBuilder).setFrom(0).setSize(1).setExplain(true);
642             this.logger.debug(searchRequestBuilder.toString());
643             SearchResponse searchResponse = searchRequestBuilder.execute().actionGet();
644             TermsFacet termsFacet = searchResponse.getFacets().facet("group");
645             HashMap<String, String> result = new HashMap<String, String>();
646             for (org.elasticsearch.search.facet.terms.TermsFacet.Entry entry : termsFacet.entries()) {
647                 result.put(entry.getTerm(), entry.count() + "");
648             }
649             return result;
650         }
651         catch (Exception e) {
652             this.logger.error(e.getMessage());
653         }
654         return null;
655     }
656 
657     public Map<String, String> group(String indexName
658             , HashMap<String, Object[]> mustSearchContentMap
659             , HashMap<String, Object[]> shouldSearchContentMap
660             , String[] groupFields) {
661         /*创建must搜索条件*/
662         QueryBuilder mustQueryBuilder = this.createQueryBuilder(mustSearchContentMap, SearchLogic.must);
663         /*创建should搜索条件*/
664         QueryBuilder shouldQueryBuilder = this.createQueryBuilder(shouldSearchContentMap, SearchLogic.should);
665         if (mustQueryBuilder == null && shouldQueryBuilder == null) {
666             return null;
667         }
668         BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
669         if (mustQueryBuilder != null) {
670             boolQueryBuilder = boolQueryBuilder.must(mustQueryBuilder);
671         }
672         if (shouldQueryBuilder != null) {
673             boolQueryBuilder = boolQueryBuilder.must(shouldQueryBuilder);
674         }
675         try {
676             return this._group(indexName, boolQueryBuilder, groupFields);
677         }
678         catch (Exception e) {
679             this.logger.error(e.getMessage());
680         }
681         return null;
682     }
683 
684     public List<Map<String, Object>> simpleSearch(String[] indexNames
685             , HashMap<String, Object[]> searchContentMap, SearchLogic searchLogic
686             , HashMap<String, Object[]> filterContentMap, SearchLogic filterLogic
687             , int from, int offset, String sortField, String sortType)
688     {
689         if (offset <= 0) {
690             return null;
691         }
692         try {
693             QueryBuilder queryBuilder = null;
694             queryBuilder = this.createQueryBuilder(searchContentMap, searchLogic);
695             queryBuilder = this.createFilterBuilder(filterLogic, queryBuilder, searchContentMap, filterContentMap);
696             SearchRequestBuilder searchRequestBuilder = this.searchClient.prepareSearch(indexNames).setSearchType(SearchType.DEFAULT)
697                     .setQuery(queryBuilder).setFrom(from).setSize(offset).setExplain(true);
698             if (sortField == null || sortField.isEmpty() || sortType == null || sortType.isEmpty()) {
699                 /*如果不需要排序*/
700             }
701             else {
702                 /*如果需要排序*/
703                 org.elasticsearch.search.sort.SortOrder sortOrder = sortType.equals("desc") ? org.elasticsearch.search.sort.SortOrder.DESC : org.elasticsearch.search.sort.SortOrder.ASC;
704                 searchRequestBuilder = searchRequestBuilder.addSort(sortField, sortOrder);
705             }
706             searchRequestBuilder = this.createHighlight(searchRequestBuilder, searchContentMap);
707             this.logger.debug(searchRequestBuilder.toString());
708             SearchResponse searchResponse = searchRequestBuilder.execute().actionGet();
709             return this.getSearchResult(searchResponse);
710         }
711         catch (Exception e) {
712             this.logger.error(e.getMessage());
713         }
714         return null;
715     }
716 
717     private SearchRequestBuilder createHighlight(SearchRequestBuilder searchRequestBuilder, HashMap<String, Object[]> searchContentMap) {
718         Iterator<Entry<String, Object[]>> iterator = searchContentMap.entrySet().iterator();
719         /*循环每一个需要搜索的字段和值*/
720         while (iterator.hasNext()) {
721             Entry<String, Object[]> entry = iterator.next();
722             String field = entry.getKey();
723             Object[] values = entry.getValue();
724             /*排除非法的搜索值*/
725             if (!this.checkValue(values)) {
726                 continue;
727             }
728             /*获得搜索类型*/
729             MySearchOption mySearchOption = this.getSearchOption(values);
730             if (mySearchOption.isHighlight()) {
731                 /*
732                  * http://www.elasticsearch.org/guide/reference/api/search/highlighting.html
733                  *
734                  * fragment_size设置成1000，默认值会造成返回的数据被截断
735                  * */
736                 searchRequestBuilder = searchRequestBuilder.addHighlightedField(field, 1000)
737                         .setHighlighterPreTags("<"+this.highlightCSS.split(",")[0]+">")
738                         .setHighlighterPostTags("</"+this.highlightCSS.split(",")[1]+">");
739             }
740         }
741         return searchRequestBuilder;
742     }
743 
744     public long getCount(String[] indexNames
745             , HashMap<String, Object[]> searchContentMap, SearchLogic searchLogic
746             , @Nullable HashMap<String, Object[]> filterContentMap, @Nullable SearchLogic filterLogic)
747     {
748         QueryBuilder queryBuilder = null;
749         try {
750             queryBuilder = this.createQueryBuilder(searchContentMap, searchLogic);
751             queryBuilder = this.createFilterBuilder(searchLogic, queryBuilder, searchContentMap, filterContentMap);
752             SearchResponse searchResponse = this.searchCountRequest(indexNames, queryBuilder);
753             return searchResponse.hits().totalHits();
754         }
755         catch (Exception e) {
756             this.logger.error(e.getMessage());
757         }
758         return 0;
759     }
760 }

免责声明！

本站转载的文章为个人学习借鉴使用，本站对版权不负任何法律责任。如果侵犯了您的隐私权益，请联系本站邮箱yoyou2525@163.com删除。

猜您在找 Elasticsearch索引的操作，利用kibana 创建/删除一个es的索引及mapping映射 ELK批量删除索引 ElasticSearch Kibana 创建索引,删除索引,查看索引配置 Elasticsearch入门之索引操作（创建、查看、删除） mysql 创建索引和删除索引 mysql 创建索引和删除索引 SQL创建索引和删除索引 mysql 创建索引和删除索引 kibana 删除创建索引模式超长耗时搜索索引 Elasticsearch索引的操作，利用kibana(如何创建/删除一个es的索引？)