實際工作中使用ES有一段時間了，比起一直在理論上接觸ES還是要好上一些的。今天就來總結一些實際工作中用到的一些ES功能吧。本文編排順序，按使用的先后可能性排序編排。ES的功能很強大，但我們能用到的，也許並不會太多，所以本文可作為一個簡單速查手冊使用喲。
1. 查看集群狀態
# 健康檢查
GET _cluster/health?pretty # 作用：可以幫助我們排查es八九是否有故障 # 結果樣例如下： { "cluster_name" : "testcluster", "status" : "green", "timed_out" : false, "number_of_nodes" : 2, "number_of_data_nodes" : 2, "active_primary_shards" : 5, "active_shards" : 10, "relocating_shards" : 0, "initializing_shards" : 0, "unassigned_shards" : 0 }
2. 查看分片信息
GET _cat/shards?pretty
# 作用：可以幫助我們查看數據分片情況
# 結果樣例如下：
test_maping_20200507 1 p STARTED 17 41.1kb 10.19.70.15 master-node-01
test_maping_20200507 1 r UNASSIGNED
c_maping_20200507 3 p STARTED 14 25.5kb 10.19.70.15 master-node-01
3. 查看節點信息
GET _nodes/stats
# 幫助我們排查es集群問題，知道節點分布情況
# 結果樣例如下：
{ "_nodes": { "total": 1, "successful": 1, "failed": 0 }, "cluster_name": "n-cluster", "nodes": { "kSi1H-3pQm6nmkmIbVpywg": { "timestamp": 1607316420983, "name": "master-node-01", "transport_address": "10.19.70.15:9300", "host": "10.19.70.15", "ip": "10.19.70.15:9300", "roles": [ "master", "data", "ingest" ], "attributes": { "ml.enabled": "true" }, "indices": { "docs": { "count": 4286963, "deleted": 19416 }, "store": { "size_in_bytes": 2647850406, "throttle_time_in_millis": 0 }, "indexing": { "index_total": 50056272, "index_time_in_millis": 12151146, "index_current": 0, "index_failed": 0, "delete_total": 213120, "delete_time_in_millis": 11643, "delete_current": 0, "noop_update_total": 0, "is_throttled": false, "throttle_time_in_millis": 0 }, "get": { "total": 5608635, "time_in_millis": 239343, "exists_total": 4523772, "exists_time_in_millis": 211376, "missing_total": 1084863, "missing_time_in_millis": 27967, "current": 0 }, "search": { "open_contexts": 0, "query_total": 7644956, "query_time_in_millis": 1051829, "query_current": 0, "fetch_total": 4710420, "fetch_time_in_millis": 432477, "fetch_current": 0, "scroll_total": 23265, "scroll_time_in_millis": 14799501190, "scroll_current": 0, "suggest_total": 0, "suggest_time_in_millis": 0, "suggest_current": 0 }, "merges": { "current": 0, "current_docs": 0, "current_size_in_bytes": 0, "total": 49638, "total_time_in_millis": 32459437, "total_docs": 304866346, "total_size_in_bytes": 180897863590, "total_stopped_time_in_millis": 0, "total_throttled_time_in_millis": 7534, "total_auto_throttle_in_bytes": 3078827106 }, "refresh": { "total": 1635812, "total_time_in_millis": 18344665, "listeners": 0 }, "flush": { "total": 472, "total_time_in_millis": 44558 }, "warmer": { "current": 0, "total": 1338114, "total_time_in_millis": 396987 }, "query_cache": { "memory_size_in_bytes": 464936, "total_count": 365237, "hit_count": 304671, "miss_count": 60566, "cache_size": 72, "cache_count": 2098, "evictions": 2026 }, "fielddata": { "memory_size_in_bytes": 0, "evictions": 0 }, "completion": { "size_in_bytes": 0 }, "segments": { "count": 459, "memory_in_bytes": 12780066, "terms_memory_in_bytes": 6311772, "stored_fields_memory_in_bytes": 571552, "term_vectors_memory_in_bytes": 0, "norms_memory_in_bytes": 219328, "points_memory_in_bytes": 1008902, "doc_values_memory_in_bytes": 4668512, "index_writer_memory_in_bytes": 0, "version_map_memory_in_bytes": 0, "fixed_bit_set_memory_in_bytes": 41928, "max_unsafe_auto_id_timestamp": -1, "file_sizes": {} }, "translog": { "operations": 93313, "size_in_bytes": 287688877 }, "request_cache": { "memory_size_in_bytes": 187385, "evictions": 0, "hit_count": 87503, "miss_count": 1563 }, "recovery": { "current_as_source": 0, "current_as_target": 0, "throttle_time_in_millis": 0 } }, "os": { "timestamp": 1607316420999, "cpu": { "percent": 8, "load_average": { "1m": 0.34, "5m": 0.34, "15m": 0.39 } }, "mem": { "total_in_bytes": 33465651200, "free_in_bytes": 432812032, "used_in_bytes": 33032839168, "free_percent": 1, "used_percent": 99 }, "swap": { "total_in_bytes": 0, "free_in_bytes": 0, "used_in_bytes": 0 }, "cgroup": { "cpuacct": { "control_group": "/", "usage_nanos": 4704757205576508 }, "cpu": { "control_group": "/", "cfs_period_micros": 100000, "cfs_quota_micros": -1, "stat": { "number_of_elapsed_periods": 0, "number_of_times_throttled": 0, "time_throttled_nanos": 0 } } } }, "process": { "timestamp": 1607316420999, "open_file_descriptors": 577, "max_file_descriptors": 655360, "cpu": { "percent": 0, "total_in_millis": 225252750 }, "mem": { "total_virtual_in_bytes": 26505314304 } }, "jvm": { "timestamp": 1607316421001, "uptime_in_millis": 10785587445, "mem": { "heap_used_in_bytes": 1780673064, "heap_used_percent": 10, "heap_committed_in_bytes": 16750411776, "heap_max_in_bytes": 16750411776, "non_heap_used_in_bytes": 167806944, "non_heap_committed_in_bytes": 176631808, "pools": { "young": { "used_in_bytes": 648635928, "max_in_bytes": 3436052480, "peak_used_in_bytes": 3436052480, "peak_max_in_bytes": 3436052480 }, "survivor": { "used_in_bytes": 44905528, "max_in_bytes": 429457408, "peak_used_in_bytes": 195628936, "peak_max_in_bytes": 429457408 }, "old": { "used_in_bytes": 1087131608, "max_in_bytes": 12884901888, "peak_used_in_bytes": 1087131608, "peak_max_in_bytes": 12884901888 } } }, "threads": { "count": 185, "peak_count": 190 }, "gc": { "collectors": { "young": { "collection_count": 12012, "collection_time_in_millis": 1407041 }, "old": { "collection_count": 1, "collection_time_in_millis": 386 } } }, "buffer_pools": { "direct": { "count": 188, "used_in_bytes": 271132937, "total_capacity_in_bytes": 271132936 }, "mapped": { "count": 876, "used_in_bytes": 2642222120, "total_capacity_in_bytes": 2642222120 } }, "classes": { "current_loaded_count": 15268, "total_loaded_count": 15268, "total_unloaded_count": 0 } }, "thread_pool": { "bulk": { "threads": 8, "queue": 0, "active": 0, "rejected": 0, "largest": 8, "completed": 3234430 }, "fetch_shard_started": { "threads": 0, "queue": 0, "active": 0, "rejected": 0, "largest": 0, "completed": 0 }, "fetch_shard_store": { "threads": 0, "queue": 0, "active": 0, "rejected": 0, "largest": 0, "completed": 0 }, "flush": { "threads": 1, "queue": 0, "active": 0, "rejected": 0, "largest": 4, "completed": 895 }, "force_merge": { "threads": 0, "queue": 0, "active": 0, "rejected": 0, "largest": 0, "completed": 0 }, "generic": { "threads": 8, "queue": 0, "active": 0, "rejected": 0, "largest": 8, "completed": 14018425 }, "get": { "threads": 8, "queue": 0, "active": 0, "rejected": 0, "largest": 8, "completed": 5381151 }, "index": { "threads": 8, "queue": 0, "active": 0, "rejected": 0, "largest": 8, "completed": 213116 }, "listener": { "threads": 0, "queue": 0, "active": 0, "rejected": 0, "largest": 0, "completed": 0 }, "management": { "threads": 5, "queue": 0, "active": 1, "rejected": 0, "largest": 5, "completed": 14542562 }, "ml_autodetect": { "threads": 0, "queue": 0, "active": 0, "rejected": 0, "largest": 0, "completed": 0 }, "ml_datafeed": { "threads": 0, "queue": 0, "active": 0, "rejected": 0, "largest": 0, "completed": 0 }, "ml_utility": { "threads": 40, "queue": 0, "active": 0, "rejected": 0, "largest": 40, "completed": 125 }, "refresh": { "threads": 4, "queue": 0, "active": 0, "rejected": 0, "largest": 4, "completed": 569855652 }, "search": { "threads": 13, "queue": 0, "active": 0, "rejected": 0, "largest": 13, "completed": 12725792 }, "snapshot": { "threads": 0, "queue": 0, "active": 0, "rejected": 0, "largest": 0, "completed": 0 }, "vanguard.data": { "threads": 0, "queue": 0, "active": 0, "rejected": 0, "largest": 0, "completed": 0 }, "vanguard.planner": { "threads": 0, "queue": 0, "active": 0, "rejected": 0, "largest": 0, "completed": 0 }, "vanguard.task": { "threads": 0, "queue": 0, "active": 0, "rejected": 0, "largest": 0, "completed": 0 }, "warmer": { "threads": 4, "queue": 0, "active": 0, "rejected": 0, "largest": 4, "completed": 2034280 }, "watcher": { "threads": 40, "queue": 0, "active": 0, "rejected": 0, "largest": 40, "completed": 213116 } }, "fs": { "timestamp": 1607316421001, "total": { "total_in_bytes": 536608768000, "free_in_bytes": 533629661184, "available_in_bytes": 533629661184, "spins": "true" }, "data": [ { "path": "/srv/data01/nodes/0", "mount": "/srv/data01 (/dev/vdb)", "type": "xfs", "total_in_bytes": 536608768000, "free_in_bytes": 533629661184, "available_in_bytes": 533629661184, "spins": "true" } ], "io_stats": { "devices": [ { "device_name": "vdb", "operations": 13972095, "read_operations": 41, "write_operations": 13972054, "read_kilobytes": 75, "write_kilobytes": 284301092 } ], "total": { "operations": 13972095, "read_operations": 41, "write_operations": 13972054, "read_kilobytes": 75, "write_kilobytes": 284301092 } } }, "transport": { "server_open": 39, "rx_count": 16621987, "rx_size_in_bytes": 973956417, "tx_count": 16621987, "tx_size_in_bytes": 5155351992 }, "http": { "current_open": 4, "total_opened": 349641 }, "breakers": { "request": { "limit_size_in_bytes": 10050247065, "limit_size": "9.3gb", "estimated_size_in_bytes": 0, "estimated_size": "0b", "overhead": 1, "tripped": 0 }, "fielddata": { "limit_size_in_bytes": 10050247065, "limit_size": "9.3gb", "estimated_size_in_bytes": 0, "estimated_size": "0b", "overhead": 1.03, "tripped": 0 }, "in_flight_requests": { "limit_size_in_bytes": 16750411776, "limit_size": "15.6gb", "estimated_size_in_bytes": 0, "estimated_size": "0b", "overhead": 1, "tripped": 0 }, "parent": { "limit_size_in_bytes": 11725288243, "limit_size": "10.9gb", "estimated_size_in_bytes": 0, "estimated_size": "0b", "overhead": 1, "tripped": 0 } }, "script": { "compilations": 17, "cache_evictions": 0 }, "discovery": { "cluster_state_queue": { "total": 0, "pending": 0, "committed": 0 } }, "ingest": { "total": { "count": 0, "time_in_millis": 0, "current": 0, "failed": 0 }, "pipelines": { "xpack_monitoring_2": { "count": 0, "time_in_millis": 0, "current": 0, "failed": 0 } } } } } }
4. 查看所有索引狀態及信息
GET _cat/indices?v&pretty
# 作用：可以讓我們快速查看都有些什么索引及其數據量，讓我們在不理解業務的情況查詢數據
# 類似於sql中的 show databases; show tables;
# 結果樣例如下：
green open utest_20201201 xxavc 5 0 2 0 7.6kb 7.6kb
yellow open .watcher-history-3-2020.08.05 O8wwxxsaQ0TaomQ 1 1 7200 0 5mb 5mb
5. 查看所有別名情況
GET _cat/aliases
# 作用：可以幫助你快速知道目前使用的索引可能，因為外部用戶可能只知道別名，但內部索引可能會很多
# 和 show databases; show tables; 語義相近
# 結果樣例如下：
test test_20200330 - - -
c_maping c_20200507 - - -
user user_20200717 - - -
6. 查看創建或修改mapping
# 查看某索引mapping GET /index_name/_mapping/type_name # 創建或修改mapping PUT /index_name { "mapping":{ "article":{ "properties":{ "content":{ "type": "string", "analyzer":"english" }, "post_date":{ "type":"date }, "title":{ "type":"keyword" } } } } }
# 作用：類似sql中創建修改表結構，可以讓你快速了解此索引都由些什么構成，並排查可能出線問題的原因，比如text是不能用於搜索的，需要添加 fielddata:true 才可以。
# 結果樣例如下：
{ "cus_20200110": { "mappings": { "job": { "properties": { "v1d": { "type": "text", "fields": { "keyword": { "type": "keyword", "ignore_above": 256 } } }, "amt": { "type": "long" } } } } } }
7. 創建索引
# 一般你可以在首次插入數據時創建索引，也可以先手動創建索引，創建mapping，這樣控制更好
# 創建空索引
POST index_name # 創建mapping，如上 # 也可以帶分片數設置創建索引 POST index_name { "settings": { "number_of_shards": 3, "number_of_replicas": 2 } }
8 插入數據更新數據
# 和添加數據時一樣，不過需要指定id
# 更新分個別字段更新和全記錄更新
# 個別字段更新使用 _update，如下 POST index_name/anytype/id1/_update { "title": "test title 001", "createDate": "2018-01-12" } # 全記錄更新 POST index_name/anytype/id1/_doc { "title":"overwrtite title full" }
9.常用查詢
# 毫無疑問，es的作用就在於查詢，快速及有效
# 查詢場景有很多，我們按照sql的方式類比，簡單分為普通查詢，聚合查詢，分組查詢，子查詢
# 1. 普通查詢（可帶分頁），類比sql：select * from index_name where title="abc" order by id limit 0, 10 # es 查詢如下 GET userindex/anytype/_search { "size": 10, "from": 0 , "query": { "term": { "title": "abc" } }, "sort": [ { "id": { "order": "asc" } } ] }
# 也可以使用如下match查詢
{
"query":{
"match":{
"title":"java"
}
}
}
# 其他查詢條件可以任意在term中添加 # 2. 聚合查詢，類比sql：select max(id) from index_name where title="abc" # es 查詢 GET userindex/anytype/_search { "size": 10, "from": 0 , "query": { "term": { "title": "abc" } }, "aggs": { "max_id": { "max": { "field": "id" } } } } # 可以自行組合其他聚合方式，一起得出聚合結果，max，min，
10.主鍵查詢
# get 查詢單個doc GET 'indexName/type/1' # mget多個查詢 get indexName/type/_mget { "docs":[{ "_id":["id1", "id2"] "_source":["field1", "field2"] }] } # 類似sql中的select f1, f2 from tb where id in (xxx);
11.索引open/close
# 索引打開關閉，是為了一些特殊場景的考量，比如我們做了索引別名，那么老的索引可能沒用了，但又不敢完全保證新索引正確，所以先將舊索引關閉然后觀察一段時間無誤后，再將其刪除，一旦出現問題，則將舊索引open恢復啟用
#打開索引，使其可用 POST test/_open #關閉索引，減少資源消耗，關閉后索引不可被搜索添加操作 POST test/_close
12.刪除索引
# 刪除索引就是刪除索引及其對應的所有資源，類似於sql中的 drop table 或者 drop database，所以需要小心操作，后果很嚴重
# 刪除某個索引
DELETE indexName
# 刪除所有索引：
DELETE * 或者 DELETE _all
13.索引merge
# 索引merge的目的在於將小segment合並為大文件，從而減少文件打開數量，這個動作一般是es自動完成的，但有時我們可能需要自己執行
// 強制merge POST indexname/_forcemerge?max_num_segments=1
14.es中幾個搜索相關名詞
# 如果僅僅是kv的搜索方式，則只需用 k:v 就可以進行搜索，當然你得先將字段類型設置為keyword或者 fielddata=true 才行，即開啟索引的索引
# match/match_all: 普通搜索過濾
# bool/filter/must: 搜索結果過濾
# term(s) : 字段包含某些詞搜索
# agg: 聚合結果
# _source: 設置字段信息
# sort: 排序結果
# exists: 空值判定查詢
# prefix: 前綴匹配查詢
# wildcard: 通配符查詢，模糊匹配
# regexp: 正則查詢
# score: 搜索評分
15.es中的幾個核心問題
es為什么這么快？
system cache的重要性？
segment
translog
memorybuffer
flush
分片
集群通信raft
16.es總覽架構圖
架構圖：
ES寫數據流程圖示例：
搜索流程圖示例：
。。。