Elasticsearch 索引Index API(7.5.0)

1. Java代码层配置模板，执行成功将打印自定义的放置模板成功几个字，可前往官网地址：https://www.elastic.co/guide/en/elasticsearch/client/java-rest/current/java-rest-high-put-template.html

package com.ruhuanxingyun.index;

import org.apache.http.HttpHost;
import org.elasticsearch.action.admin.indices.alias.Alias;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.client.indices.PutIndexTemplateRequest;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;

import java.io.IOException;
import java.util.Arrays;

/**
 * @description: 配置索引模板
 * @author: rup
 * @date: Create in 2019/8/29 10:09
 * @company: ruhuanxingyun
 */
public class NginxIndexTemplate {

    public static void main(String[] args) {
        // 模板名
        PutIndexTemplateRequest request = new PutIndexTemplateRequest("t-nginx");
        // 模板匹配模式，即匹配采用当前模板的索引名格式
        request.patterns(Arrays.asList("nginx*"));

        // 索引配置，number_of_shards索引分片、number_of_replicas副本，默认为1，如果是本地使用单节点，则改为0
        request.settings(Settings.builder().put("index.number_of_shards", 1)
                .put("index.number_of_replicas", 0));

        RestHighLevelClient client = null;
        try {
            // 映射配置
            XContentBuilder xContentBuilder = XContentFactory.jsonBuilder();
            xContentBuilder.startObject()
                                .startObject("properties")
                                 　　// @timestamp时间索引，专门提供给kibana用的字段，日期类型，格式
                                    .startObject("@timestamp").field("type", "date").field("format", "yyyy-MM-dd HH:mm:ss")
                                    .endObject()
                                    .startObject("mac").field("type", "keyword")
                                    .endObject()
                                    .startObject("sn").field("type", "keyword")
                                    .endObject()
                                    .startObject("productType").field("type", "keyword")
                                    .endObject()
                                    .startObject("status").field("type", "keyword")
                                    .endObject()
                                    .startObject("updateTime").field("type", "date").field("format", "yyyy-MM-dd HH:mm:ss")
                                    .endObject()
                                .endObject()
                            .endObject();
            request.mapping(xContentBuilder);
            // 别名
            request.alias(new Alias("nginx"));
            // 模板的排序
            //request.order(1);
            // 版本号
            //request.version(2);

　　　　　　　// 客户端初始化
            client = new RestHighLevelClient(
                    RestClient.builder(new HttpHost("localhost", 9200, "http")));
            AcknowledgedResponse acknowledgedResponse = client.indices().putTemplate(request, RequestOptions.DEFAULT);

            if (acknowledgedResponse.isAcknowledged()) {
                System.out.println(String.format("放置模板名为%s成功!", request.name()));
            } else {
                System.out.println(String.format("放置模板名为%s失败!", request.name()));
            }
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            try {
                // 关闭资源
                if (client != null) {
                    client.close();
                }
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }

}

 2. 可通过elasticsearch-head/postman等插件发送put请求 http://localhost:9200/_template/t-nginx，放置成功如下图，可前往官网地址：https://www.elastic.co/guide/en/elasticsearch/reference/7.3/indices-templates.html

{
    "index_patterns": ["nginx*"],
    "settings": {
        "number_of_shards": 1,
        "number_of_replicas": 0
    },
    "mappings": {
        "properties" : {
            "@timestamp": {
                "format": "yyyy-MM-dd HH:mm:ss",
                "type": "date"
            },
            "mac": {
                "type": "keyword"
            },
            "sn": {
                "type": "keyword"
            },
            "productType": {
                "type": "keyword"
            },
            "updatetime": {
                "format": "yyyy-MM-dd HH:mm:ss",
                "type": "date"
            },
            "status": {
                "type": "keyword"
            }
        }
    },
    "aliases": {
        "nginx": {}
    }
}

3. 索引模板相关的两种方式

　　A. 获取模板：GET请求 http://localhost:9200/_template/t-nginx ，其中t-nginx是模板名   或者   Java代码层获取https://www.elastic.co/guide/en/elasticsearch/client/java-rest/current/java-rest-high-get-templates.html

　　B. 删除模板：DELETE请求 http://localhost:9200/_template/t-nginx   或者  Java代码层获取https://www.elastic.co/guide/en/elasticsearch/client/java-rest/current/java-rest-high-delete-template.html

 　　C. 放置模板：见2、3介绍

4. 获取索引

　　A. elasticsearch-head/postman/kibana

　　　　GET /<index> 

　　　　路径参数：index是索引名称，多个以逗号分隔，或者通配符表达式，可使用_all来获取集群所有的索引；

　　　   查询参数：expand_wildcards是控制通配符表达式可以扩展到的索引类型，默认值为open，all-展示开放和关闭的索引，open-仅展示开放的索引，closed-仅展示关闭的索引，none-不接受通配符表达式；

　　B. kibana操作

　　C. java编写

　　/**
     * 删除超过预存天数的日志
     *
     * @param preserveDay 预留天数
     */
    private void deleteLogByTime(Integer preserveDay) {
        String[] indexPrefixs = this.indexPrefix.split(StrUtil.COMMA);
        GetIndexRequest getIndexRequest;
        DeleteIndexRequest deleteIndexRequest;
        List<String> list = new ArrayList<>();
        Set<String> set = new HashSet<>();

        try {
            for (String indexPrefix : indexPrefixs) {
                getIndexRequest = new GetIndexRequest(String.format("%s*", indexPrefix));
                GetIndexResponse response = restHighLevelClient.indices().get(getIndexRequest, RequestOptions.DEFAULT);
                // 索引数据第一条就是本索引最先的存储的数据
                String[] indexs = response.getIndices();
                int diff = indexs.length - preserveDay;
                if (diff > 0) {
                    list.clear();
                    for (int i = 0; i < diff; i++) {
                        list.add(indexs[i]);
                        set.add(indexs[i].replace(indexPrefix, ""));
                    }
                    deleteIndexRequest = new DeleteIndexRequest(list.toArray(new String[0]));
                    restHighLevelClient.indices().delete(deleteIndexRequest, RequestOptions.DEFAULT);
                    log.info(String.format("删除索引名称为%s日志", list.toString()));
                }
            }

            if (CollUtil.isNotEmpty(set)) {
                this.deleteDiskSpaceByLogTime(set);
            }
        } catch (IOException e) {
            e.printStackTrace();
            log.info("删除超过预存天数的日志失败");
        }
    }

5. 创建索引

　　A. elasticsearch-head/postman/kibana

　　　　PUT /<index>

　　　　路径参数：index索引名称仅小写；

　　　　请求正文：aliases是索引别名，mappings是索引中字段，映射对象，settings是索引的配置项

　　B. kibana操作

　　C.  java编写

6. 删除索引

　　A. elasticsearch-head/postman/kibana

　　　　DELETE /<index>

　　　　路径参数：不能使用别名删除索引，要禁用_all或通配符表达式删除索引，请将elasticsearch.yml文件中action.destructive_requires_name群集设置设为true

　　B. kibana操作

　　C. java编写：见获取索引里的代码

7. 关闭索引

　　A. elasticsearch-head/postman

　　　　POST /<index>/_close 关闭索引禁止进行读写操作，并且不允许开放索引允许的所有操作，无法对文档建立索引或者在关闭索引中进行搜索文档。这使关闭索引不必维护用于文档建立索引或搜索文档的内部数据结构，从而减少了集群上的开销，但关闭索引会占用大量的磁盘空间。

　　B. java代码层　　

CloseIndexRequest request = new CloseIndexRequest("index");
AcknowledgedResponse closeIndexResponse = client.indices().close(request, RequestOptions.DEFAULT);
boolean acknowledged = closeIndexResponse.isAcknowledged();

8. 开放索引

　　A. elasticsearch-head/postman

　　　　POST /<index>/_open 重新打开封闭索引，当打开或关闭索引时，主服务器负责重新启动索引分片以反映索引的新状态，然后索引将经历正常的恢复过程

　　B. java代码层

OpenIndexRequest request = new OpenIndexRequest("index");
OpenIndexResponse openIndexResponse = client.indices().open(request, RequestOptions.DEFAULT);
boolean acknowledged = openIndexResponse.isAcknowledged(); 
boolean shardsAcked = openIndexResponse.isShardsAcknowledged();

 

package com.logcollector.config;

import com.entity.IndexName;
import lombok.extern.slf4j.Slf4j;
import org.elasticsearch.action.admin.indices.alias.Alias;
import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateRequest;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.client.indices.GetIndexTemplatesRequest;
import org.elasticsearch.client.indices.GetIndexTemplatesResponse;
import org.elasticsearch.client.indices.IndexTemplatesExistRequest;
import org.elasticsearch.client.indices.PutIndexTemplateRequest;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

import javax.annotation.PostConstruct;
import java.io.IOException;
import java.util.Collections;

@Component
@Slf4j
public class ElasticsearchIndexTemplate {

    @Autowired
    private RestHighLevelClient restHighLevelClient;

    private static final Integer VERSION = 2;

    @PostConstruct
    public void init() {
        String template = IndexName.RWS_BEHAVIOR_LOG_TEMPLATE.getName();
        try {
            // 判断索引模板是否存在
            boolean status = this.templateExist(template);
            if (status) {
                // 获取索引模板版本
                Integer version = this.getTemplate(template);
                if (VERSION.equals(version)) {
                    return;
                }

                // 删除以前索引模板
                status = this.deleteTemplate(template);
                if (!status) {
                    return;
                }
            }

            // 创建新模板
            this.createTemplate(template);
        } catch (IOException e) {
            log.error("创建索引模板{}失败：{}", e);
        }
    }

    /**
     * 判断索引模板是否存在
     *
     * @param template 索引模板
     * @return 存在与否
     * @throws IOException 异常
     */
    private boolean templateExist(String template) throws IOException {
        IndexTemplatesExistRequest request = new IndexTemplatesExistRequest(template);

        return restHighLevelClient.indices().existsTemplate(request, RequestOptions.DEFAULT);
    }

    /**
     * 获取索引模板版本
     *
     * @param template 索引模板
     * @return 版本
     * @throws IOException 异常
     */
    private Integer getTemplate(String template) throws IOException {
        GetIndexTemplatesRequest request = new GetIndexTemplatesRequest(template);
        GetIndexTemplatesResponse response = restHighLevelClient.indices().getIndexTemplate(request, RequestOptions.DEFAULT);

        return response.getIndexTemplates().get(0).version();
    }

    /**
     * 删除以前索引模板
     *
     * @param template 索引模板
     * @return 删除与否
     * @throws IOException 异常
     */
    private boolean deleteTemplate(String template) throws IOException {
        DeleteIndexTemplateRequest request = new DeleteIndexTemplateRequest(template);
        AcknowledgedResponse response = restHighLevelClient.indices().deleteTemplate(request, RequestOptions.DEFAULT);

        return response.isAcknowledged();
    }

    /**
     * 创建索引模板
     *
     * @param template 索引模板
     * @throws IOException 异常
     */
    private void createTemplate(String template) throws IOException {
        PutIndexTemplateRequest request = new PutIndexTemplateRequest(template);
        request.patterns(Collections.singletonList(IndexName.RWS_BEHAVIOR_LOG.getName()))
            .settings(Settings.builder().put("index.number_of_shards", 5))
            .mapping(this.buildXContent())
            .alias(new Alias(template))
            .version(VERSION);
        AcknowledgedResponse response = restHighLevelClient.indices().putTemplate(request, RequestOptions.DEFAULT);
        log.info("创建索引模板{}{}", template, response.isAcknowledged() ? "成功" : "失败");
    }

    /**
     * 构建mapping
     *
     * @return mapping
     * @throws IOException 异常
     */
    private XContentBuilder buildXContent() throws IOException {
        XContentBuilder builder = XContentFactory.jsonBuilder();
        builder.startObject()
            .startObject("properties")
                .startObject("tid")
                    .field("type", "integer")
                .endObject()
                .startObject("@timestamp")
                    .field("type", "date")
                    .field("format", "yyyy-MM-dd HH:mm:ss.SSS")
                .endObject()
                .startObject("log")
                    .startObject("properties")
                        .startObject("type")
                            .field("type", "integer")
                        .endObject()
                        .startObject("subtype")
                            .field("type", "integer")
                        .endObject()
                    .endObject()
                .endObject()
                .startObject("user")
                    .startObject("properties")
                        .startObject("account")
                            .field("type", "keyword")
                        .endObject()
                        .startObject("userId")
                            .field("type", "long")
                        .endObject()
                        .startObject("realname")
                            .field("type", "keyword")
                        .endObject()
                        .startObject("ip")
                            .field("type", "keyword")
                        .endObject()
                        .startObject("terminalType")
                            .field("type", "keyword")
                         .endObject()
                        .startObject("terminalId")
                            .field("type", "keyword")
                        .endObject()
                    .endObject()
                .endObject()
                .startObject("behavior")
                    .startObject("properties")
                        .startObject("uploadPage")
                            .field("type", "keyword")
                        .endObject()
                        .startObject("uploadDomain")
                            .field("type", "keyword")
                        .endObject()
                        .startObject("filePath")
                            .field("type", "keyword")
                        .endObject()
                        .startObject("downloadPage")
                            .field("type", "keyword")
                         .endObject()
                        .startObject("downloadLink")
                            .field("type", "keyword")
                        .endObject()
                        .startObject("fileLink")
                            .field("type", "keyword")
                        .endObject()
                        .startObject("fileSize")
                            .field("type", "long")
                        .endObject()
                    .endObject()
                .endObject()
                .startObject("security")
                    .startObject("properties")
                        .startObject("softType")
                            .field("type", "integer")
                        .endObject()
                        .startObject("soft")
                            .field("type", "keyword")
                        .endObject()
                        .startObject("softImage")
                            .field("type", "keyword")
                        .endObject()
                        .startObject("createTime")
                            .field("type", "date")
                            .field("format", "yyyy-MM-dd HH:mm:ss")
                        .endObject()
                        .startObject("up2Date")
                            .field("type", "boolean")
                        .endObject()
                        .startObject("softVersion")
                            .field("type", "keyword")
                        .endObject()
                    .endObject()
                .endObject()
                .startObject("remote")
                    .startObject("properties")
                        .startObject("name")
                            .field("type", "keyword")
                        .endObject()
                        .startObject("protocol")
                            .field("type", "keyword")
                        .endObject()
                        .startObject("ip")
                            .field("type", "keyword")
                        .endObject()
                        .startObject("port")
                            .field("type", "integer")
                        .endObject()
                        .startObject("pic")
                            .field("type", "keyword")
                        .endObject()
                    .endObject()
                .endObject()
            .endObject()
        .endObject();

        return builder;
    }

}