Elasticsearch 索引Index API(7.5.0)

1. Java代碼層配置模板，執行成功將打印自定義的放置模板成功幾個字，可前往官網地址：https://www.elastic.co/guide/en/elasticsearch/client/java-rest/current/java-rest-high-put-template.html

package com.ruhuanxingyun.index;

import org.apache.http.HttpHost;
import org.elasticsearch.action.admin.indices.alias.Alias;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.client.indices.PutIndexTemplateRequest;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;

import java.io.IOException;
import java.util.Arrays;

/**
 * @description: 配置索引模板
 * @author: rup
 * @date: Create in 2019/8/29 10:09
 * @company: ruhuanxingyun
 */
public class NginxIndexTemplate {

    public static void main(String[] args) {
        // 模板名
        PutIndexTemplateRequest request = new PutIndexTemplateRequest("t-nginx");
        // 模板匹配模式，即匹配采用當前模板的索引名格式
        request.patterns(Arrays.asList("nginx*"));

        // 索引配置，number_of_shards索引分片、number_of_replicas副本，默認為1，如果是本地使用單節點，則改為0
        request.settings(Settings.builder().put("index.number_of_shards", 1)
                .put("index.number_of_replicas", 0));

        RestHighLevelClient client = null;
        try {
            // 映射配置
            XContentBuilder xContentBuilder = XContentFactory.jsonBuilder();
            xContentBuilder.startObject()
                                .startObject("properties")
                                 　　// @timestamp時間索引，專門提供給kibana用的字段，日期類型，格式
                                    .startObject("@timestamp").field("type", "date").field("format", "yyyy-MM-dd HH:mm:ss")
                                    .endObject()
                                    .startObject("mac").field("type", "keyword")
                                    .endObject()
                                    .startObject("sn").field("type", "keyword")
                                    .endObject()
                                    .startObject("productType").field("type", "keyword")
                                    .endObject()
                                    .startObject("status").field("type", "keyword")
                                    .endObject()
                                    .startObject("updateTime").field("type", "date").field("format", "yyyy-MM-dd HH:mm:ss")
                                    .endObject()
                                .endObject()
                            .endObject();
            request.mapping(xContentBuilder);
            // 別名
            request.alias(new Alias("nginx"));
            // 模板的排序
            //request.order(1);
            // 版本號
            //request.version(2);

　　　　　　　// 客戶端初始化
            client = new RestHighLevelClient(
                    RestClient.builder(new HttpHost("localhost", 9200, "http")));
            AcknowledgedResponse acknowledgedResponse = client.indices().putTemplate(request, RequestOptions.DEFAULT);

            if (acknowledgedResponse.isAcknowledged()) {
                System.out.println(String.format("放置模板名為%s成功!", request.name()));
            } else {
                System.out.println(String.format("放置模板名為%s失敗!", request.name()));
            }
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            try {
                // 關閉資源
                if (client != null) {
                    client.close();
                }
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }

}

 2. 可通過elasticsearch-head/postman等插件發送put請求 http://localhost:9200/_template/t-nginx，放置成功如下圖，可前往官網地址：https://www.elastic.co/guide/en/elasticsearch/reference/7.3/indices-templates.html

{
    "index_patterns": ["nginx*"],
    "settings": {
        "number_of_shards": 1,
        "number_of_replicas": 0
    },
    "mappings": {
        "properties" : {
            "@timestamp": {
                "format": "yyyy-MM-dd HH:mm:ss",
                "type": "date"
            },
            "mac": {
                "type": "keyword"
            },
            "sn": {
                "type": "keyword"
            },
            "productType": {
                "type": "keyword"
            },
            "updatetime": {
                "format": "yyyy-MM-dd HH:mm:ss",
                "type": "date"
            },
            "status": {
                "type": "keyword"
            }
        }
    },
    "aliases": {
        "nginx": {}
    }
}

3. 索引模板相關的兩種方式

　　A. 獲取模板：GET請求 http://localhost:9200/_template/t-nginx ，其中t-nginx是模板名   或者   Java代碼層獲取https://www.elastic.co/guide/en/elasticsearch/client/java-rest/current/java-rest-high-get-templates.html

　　B. 刪除模板：DELETE請求 http://localhost:9200/_template/t-nginx   或者  Java代碼層獲取https://www.elastic.co/guide/en/elasticsearch/client/java-rest/current/java-rest-high-delete-template.html

 　　C. 放置模板：見2、3介紹

4. 獲取索引

　　A. elasticsearch-head/postman/kibana

　　　　GET /<index> 

　　　　路徑參數：index是索引名稱，多個以逗號分隔，或者通配符表達式，可使用_all來獲取集群所有的索引；

　　　   查詢參數：expand_wildcards是控制通配符表達式可以擴展到的索引類型，默認值為open，all-展示開放和關閉的索引，open-僅展示開放的索引，closed-僅展示關閉的索引，none-不接受通配符表達式；

　　B. kibana操作

　　C. java編寫

　　/**
     * 刪除超過預存天數的日志
     *
     * @param preserveDay 預留天數
     */
    private void deleteLogByTime(Integer preserveDay) {
        String[] indexPrefixs = this.indexPrefix.split(StrUtil.COMMA);
        GetIndexRequest getIndexRequest;
        DeleteIndexRequest deleteIndexRequest;
        List<String> list = new ArrayList<>();
        Set<String> set = new HashSet<>();

        try {
            for (String indexPrefix : indexPrefixs) {
                getIndexRequest = new GetIndexRequest(String.format("%s*", indexPrefix));
                GetIndexResponse response = restHighLevelClient.indices().get(getIndexRequest, RequestOptions.DEFAULT);
                // 索引數據第一條就是本索引最先的存儲的數據
                String[] indexs = response.getIndices();
                int diff = indexs.length - preserveDay;
                if (diff > 0) {
                    list.clear();
                    for (int i = 0; i < diff; i++) {
                        list.add(indexs[i]);
                        set.add(indexs[i].replace(indexPrefix, ""));
                    }
                    deleteIndexRequest = new DeleteIndexRequest(list.toArray(new String[0]));
                    restHighLevelClient.indices().delete(deleteIndexRequest, RequestOptions.DEFAULT);
                    log.info(String.format("刪除索引名稱為%s日志", list.toString()));
                }
            }

            if (CollUtil.isNotEmpty(set)) {
                this.deleteDiskSpaceByLogTime(set);
            }
        } catch (IOException e) {
            e.printStackTrace();
            log.info("刪除超過預存天數的日志失敗");
        }
    }

5. 創建索引

　　A. elasticsearch-head/postman/kibana

　　　　PUT /<index>

　　　　路徑參數：index索引名稱僅小寫；

　　　　請求正文：aliases是索引別名，mappings是索引中字段，映射對象，settings是索引的配置項

　　B. kibana操作

　　C.  java編寫

6. 刪除索引

　　A. elasticsearch-head/postman/kibana

　　　　DELETE /<index>

　　　　路徑參數：不能使用別名刪除索引，要禁用_all或通配符表達式刪除索引，請將elasticsearch.yml文件中action.destructive_requires_name群集設置設為true

　　B. kibana操作

　　C. java編寫：見獲取索引里的代碼

7. 關閉索引

　　A. elasticsearch-head/postman

　　　　POST /<index>/_close 關閉索引禁止進行讀寫操作，並且不允許開放索引允許的所有操作，無法對文檔建立索引或者在關閉索引中進行搜索文檔。這使關閉索引不必維護用於文檔建立索引或搜索文檔的內部數據結構，從而減少了集群上的開銷，但關閉索引會占用大量的磁盤空間。

　　B. java代碼層　　

CloseIndexRequest request = new CloseIndexRequest("index");
AcknowledgedResponse closeIndexResponse = client.indices().close(request, RequestOptions.DEFAULT);
boolean acknowledged = closeIndexResponse.isAcknowledged();

8. 開放索引

　　A. elasticsearch-head/postman

　　　　POST /<index>/_open 重新打開封閉索引，當打開或關閉索引時，主服務器負責重新啟動索引分片以反映索引的新狀態，然后索引將經歷正常的恢復過程

　　B. java代碼層

OpenIndexRequest request = new OpenIndexRequest("index");
OpenIndexResponse openIndexResponse = client.indices().open(request, RequestOptions.DEFAULT);
boolean acknowledged = openIndexResponse.isAcknowledged(); 
boolean shardsAcked = openIndexResponse.isShardsAcknowledged();

 

package com.logcollector.config;

import com.entity.IndexName;
import lombok.extern.slf4j.Slf4j;
import org.elasticsearch.action.admin.indices.alias.Alias;
import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateRequest;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.client.indices.GetIndexTemplatesRequest;
import org.elasticsearch.client.indices.GetIndexTemplatesResponse;
import org.elasticsearch.client.indices.IndexTemplatesExistRequest;
import org.elasticsearch.client.indices.PutIndexTemplateRequest;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

import javax.annotation.PostConstruct;
import java.io.IOException;
import java.util.Collections;

@Component
@Slf4j
public class ElasticsearchIndexTemplate {

    @Autowired
    private RestHighLevelClient restHighLevelClient;

    private static final Integer VERSION = 2;

    @PostConstruct
    public void init() {
        String template = IndexName.RWS_BEHAVIOR_LOG_TEMPLATE.getName();
        try {
            // 判斷索引模板是否存在
            boolean status = this.templateExist(template);
            if (status) {
                // 獲取索引模板版本
                Integer version = this.getTemplate(template);
                if (VERSION.equals(version)) {
                    return;
                }

                // 刪除以前索引模板
                status = this.deleteTemplate(template);
                if (!status) {
                    return;
                }
            }

            // 創建新模板
            this.createTemplate(template);
        } catch (IOException e) {
            log.error("創建索引模板{}失敗：{}", e);
        }
    }

    /**
     * 判斷索引模板是否存在
     *
     * @param template 索引模板
     * @return 存在與否
     * @throws IOException 異常
     */
    private boolean templateExist(String template) throws IOException {
        IndexTemplatesExistRequest request = new IndexTemplatesExistRequest(template);

        return restHighLevelClient.indices().existsTemplate(request, RequestOptions.DEFAULT);
    }

    /**
     * 獲取索引模板版本
     *
     * @param template 索引模板
     * @return 版本
     * @throws IOException 異常
     */
    private Integer getTemplate(String template) throws IOException {
        GetIndexTemplatesRequest request = new GetIndexTemplatesRequest(template);
        GetIndexTemplatesResponse response = restHighLevelClient.indices().getIndexTemplate(request, RequestOptions.DEFAULT);

        return response.getIndexTemplates().get(0).version();
    }

    /**
     * 刪除以前索引模板
     *
     * @param template 索引模板
     * @return 刪除與否
     * @throws IOException 異常
     */
    private boolean deleteTemplate(String template) throws IOException {
        DeleteIndexTemplateRequest request = new DeleteIndexTemplateRequest(template);
        AcknowledgedResponse response = restHighLevelClient.indices().deleteTemplate(request, RequestOptions.DEFAULT);

        return response.isAcknowledged();
    }

    /**
     * 創建索引模板
     *
     * @param template 索引模板
     * @throws IOException 異常
     */
    private void createTemplate(String template) throws IOException {
        PutIndexTemplateRequest request = new PutIndexTemplateRequest(template);
        request.patterns(Collections.singletonList(IndexName.RWS_BEHAVIOR_LOG.getName()))
            .settings(Settings.builder().put("index.number_of_shards", 5))
            .mapping(this.buildXContent())
            .alias(new Alias(template))
            .version(VERSION);
        AcknowledgedResponse response = restHighLevelClient.indices().putTemplate(request, RequestOptions.DEFAULT);
        log.info("創建索引模板{}{}", template, response.isAcknowledged() ? "成功" : "失敗");
    }

    /**
     * 構建mapping
     *
     * @return mapping
     * @throws IOException 異常
     */
    private XContentBuilder buildXContent() throws IOException {
        XContentBuilder builder = XContentFactory.jsonBuilder();
        builder.startObject()
            .startObject("properties")
                .startObject("tid")
                    .field("type", "integer")
                .endObject()
                .startObject("@timestamp")
                    .field("type", "date")
                    .field("format", "yyyy-MM-dd HH:mm:ss.SSS")
                .endObject()
                .startObject("log")
                    .startObject("properties")
                        .startObject("type")
                            .field("type", "integer")
                        .endObject()
                        .startObject("subtype")
                            .field("type", "integer")
                        .endObject()
                    .endObject()
                .endObject()
                .startObject("user")
                    .startObject("properties")
                        .startObject("account")
                            .field("type", "keyword")
                        .endObject()
                        .startObject("userId")
                            .field("type", "long")
                        .endObject()
                        .startObject("realname")
                            .field("type", "keyword")
                        .endObject()
                        .startObject("ip")
                            .field("type", "keyword")
                        .endObject()
                        .startObject("terminalType")
                            .field("type", "keyword")
                         .endObject()
                        .startObject("terminalId")
                            .field("type", "keyword")
                        .endObject()
                    .endObject()
                .endObject()
                .startObject("behavior")
                    .startObject("properties")
                        .startObject("uploadPage")
                            .field("type", "keyword")
                        .endObject()
                        .startObject("uploadDomain")
                            .field("type", "keyword")
                        .endObject()
                        .startObject("filePath")
                            .field("type", "keyword")
                        .endObject()
                        .startObject("downloadPage")
                            .field("type", "keyword")
                         .endObject()
                        .startObject("downloadLink")
                            .field("type", "keyword")
                        .endObject()
                        .startObject("fileLink")
                            .field("type", "keyword")
                        .endObject()
                        .startObject("fileSize")
                            .field("type", "long")
                        .endObject()
                    .endObject()
                .endObject()
                .startObject("security")
                    .startObject("properties")
                        .startObject("softType")
                            .field("type", "integer")
                        .endObject()
                        .startObject("soft")
                            .field("type", "keyword")
                        .endObject()
                        .startObject("softImage")
                            .field("type", "keyword")
                        .endObject()
                        .startObject("createTime")
                            .field("type", "date")
                            .field("format", "yyyy-MM-dd HH:mm:ss")
                        .endObject()
                        .startObject("up2Date")
                            .field("type", "boolean")
                        .endObject()
                        .startObject("softVersion")
                            .field("type", "keyword")
                        .endObject()
                    .endObject()
                .endObject()
                .startObject("remote")
                    .startObject("properties")
                        .startObject("name")
                            .field("type", "keyword")
                        .endObject()
                        .startObject("protocol")
                            .field("type", "keyword")
                        .endObject()
                        .startObject("ip")
                            .field("type", "keyword")
                        .endObject()
                        .startObject("port")
                            .field("type", "integer")
                        .endObject()
                        .startObject("pic")
                            .field("type", "keyword")
                        .endObject()
                    .endObject()
                .endObject()
            .endObject()
        .endObject();

        return builder;
    }

}