一.顯示執行日志
clickhouse-client --send_logs_level=trace
或者進入client session 后輸入
set send_logs_level = 'trace'
select 1
set send_log_level='none'
可以跟蹤執行日志
二.對字段進行編碼
1. 創建表時聲明對列編碼
create table test__codecs ( a String,
a_lc LowCardinality(String) default a,
b UInt32,
b_delta UInt32 default b Codec(Delta),
b_delta_lz4 UInt32 default b Codec(Delta,LZ4),
b_dd UInt32 default b Codec(DoubleDelta),
b_dd_lz4 UInt32 default b Codec(DoubleDelta,LZ4)
)
engine = MergeTree
partiton by tuple() order by tuple();
字段 a 為原字符字段, a_lc 對 a 的字典編碼類型.b 為原 int 字段,b_xxx 對 b 字段值進行不同級別的 codec
Codec 可接受兩個參數,第一個指定編碼方式,第二個指定壓縮算法
partition by tuple() 指不分區
2. 加載數據
inert into test_codecs(a,b)
select
concat('prefix',toString(rand() * 1000)),now()+(number * 10)
from system.numbers
limit 100000000
settings max_block_size=1000000
3. 查看數據
select name , sum(data_compressed_bytes) comp,
sum(data_uncompressed_bytes) uncomp,
round(comp/uncomp * 100.0,2) as percent
from system.columns where table = 'test_codecs'
group by name order by name
可以比較不同壓縮率
4. 編碼同時可加速查詢
select a as a,count(*) as c from test_codecs
group by a order by c asc limit 10
select a_lc as a,count(*) as c from test_codecs
group by a order by c asc limit 10
第二個編碼列的查詢要快得多
5. 各種編碼方式比較
| Name | Best for |
|---|---|
| LowCardinality | 少於10 K 個值的字符串 |
| Delta | 時間序列 |
| Double Delta | 遞增的計數 |
| Gorilla | 儀表數據(均值波動) |
| T64 | 非隨機整數 |
三.善用物化視圖
常見場景
例如展示每一個節點cpu 利用率的當前值
使用argMaxState 聚合列
create materialized view cpu_last_point_idle_mv
engine = AggregatingMergeTree()
partition by tuple()
order by tags_id
populate
as select
argMaxState(create_date,created_at) as created_data,
maxState(create_at) as max_created_max,
argMaxState(time,created_at) as time,
tags_id,
argMaxState(usage_idle,created_at) as usage_idle
from cpu
group by tags_id
argMax(a,b) 函數返回 b 最大值時 a的值
State 為聚合函數的后綴,聚合函數加此后綴不直接返回結果,返回聚合函數的中間結果,該中間結果可在AggregatingMergeTree 引擎中使用
使用Merge函數后綴得到聚合結果
create view cpu_last_point_idle_v as
select
argMaxMerge(created_date) as created_date,
maxMerge(max_created_at) as created_at,
argMaxMerge(time) as time,
tags_id,
argMaxMerge(usage_idle) as usage_idle
from cpu_last_point_idle_mv
group by tags_id
查詢結果視圖
select
tags_id,
100 - usage_idle usage
from cpu_last_point_idle_v
order by usage desc,tags_id asc
limit 10
可以看到,查詢速度非常快
四.使用數組類型存儲 k-v 對
常規的表結構
create table cpu (
created_date Date default today(),
created_at DateTime default now(),
time Strng,
tags_id UInt32,
usage_user Float64,
usage_system Float64,
...
additional_tags String default '')
enginge = MergeTree()
partition by created_date
order by (tags_id,created_at)
使用數組字段將獲得更多靈活性
create table cpu_dynamic(
created_data Date,
created_at DateTime,
time String,
tags_id UInt32,
metrics_name Array(String),
metrics_value Array(Float64)
)
engine = MergeTree()
partition by created_date
order by (tags_id,created_at)
metrics_name 為 key 數組 metrics_value 為 value 數組
以json串格式插入數據
clickhouse-client -d default \
--query="insert into cpu_dynamic format JSONEachRow"
<<DATA
{"created_data":"2016-01-13",
"created_at":"2016-01-13 00:00:00",
"time":"2016-01-13 00:00:00 +0000",
"tags_id":6220,
"metrics_name":["user","system","idle","nice"],
"metrics_value":[35,47,77,21]}
DATA
可以把 metrics_name 和 metrics_value 當成對應的 KV 對: user:35 idle:77
然后可用 array join 把 數組鋪平
select tags_id,name,value
from cpu_dynamic
array join metrics_name as name,metrics_value as value
| tags_id | name | value |
|---|---|---|
| 6220 | user | 35 |
| 6220 | system | 47 |
| 6220 | idle | 77 |
| 6220 | nice | 21 |
五.用物化做預計算
創建一個物化列
alter table cpu_dynamic add column usage_user materialized
metrics_value[indexOf(metrics_name,'user']
after tags_id
select time,tags_id,usage_user
from cpu_dynamic
新增一物化列,這一列計算出 metrics_value 中 key 為 user 對應的值
| time | tags_id | usage_user |
|---|---|---|
| 00:00:00 | 6220 | 35 |
| 00:00:10 | 6220 | 0 |
六.使用字典替代維表關聯
維度關聯是數倉模型中非常普遍
select tags.rack rack,avg(100-cpu.usage_idle) usage
from cpu
inner join tags as t on cpu.tags_id = t.id
group by rack
order by usage desc
limit 10
這種方式靈活度不夠,每獲取一個維度就需要一次 join ,並且維度往往有變動
配置外部字典
在 /etc/clickhouse-server 創建文件 tag_dict.xml
<yandex><dictionary>
<name>tags</name>
<source><mysql>
<host>localhost</host><port>3306</port>
<user>root</user><password>****</password>
<db>tsbs</db><table>tags</table>
</mysql></source>
<layout><hashed/></layout>
<structure>
<id> <name>id</name> </id>
<attribute>
<name>hostname</name><type>String</type>
<null_value></null_value>
</attribute>
...
</structure>
</yandex></dictionary>
外部字典表的配置詳細見 Configuring an External Dictionary | ClickHouse Documentation
直接使用字典來獲取維度
select
dictGetString('tags','rack',toUInt64(cpu.tags_id)) rack,
avg(100-cpu.usage_idle) usage
from cpu
group by rack
order by usage desc
limit 10
dictGetString 函數從字典表里面找出相應維度值.tags 即為上面配置的外部字典表, rack 為 字典表的屬性(維度), toUInt64(cpu.tags_id) 為字典表的 key ,字典表的 key 類型必須為 UInt64
七.直接使用 mysql 引擎將更加方便
創建 mysql 引擎庫
create database mysql_repl
engin = MySQL (
'127.0.0.1'.
'repl',
'root',
'secret'
)
接着關聯 mysql 里面表數據
select
t.datetime,t.date,t.request_id,
t.name customer,s.name sku
from (
select t.* from traffic t
join customer c on t.customer_id=c.id ) as t
join mysql_repl.sku s on t.sku_id = s.id
where customer_id = 5
order by t.request_id limit 10
這里 where 語句會觸發 clickhouse 謂詞下推優化
八.使用 TTL(time to live)刪除過期數據
指定過期字段和時間
create table traffic (
datetime DateTime,
date Date,
request_id UInt64,
cust_id UInt32,
sku UInt32
) engine = MergeTree
partition by toYYYYMM(date)
order by(cust_id,date)
TTL datetime + INTERVAL 90 DAY
最后一行意思使用 datetime 字段判斷,設置90天前的數據自動過期
更加靈活的過期時間設置
create table traffic_ttl_variable(
date Date,
retention_days UInt16,
...
) ...
TTL date + INTERVAL (retention_days * 2) DAY
過期時間也可以設置為跟字段值相關的變量,上面設置了保留時間跟 retention_days 這個字段值有關,也就是說每一行都有自己的過期時間.
指定過期數據的處理策略
create table traffic_ttl_disk(
date Date,
retention_days UInt16,
...
) ...
TTL data + INTERVAL 7 DAY to DISK 'ssd'
data + INTERVAL 30 DAY to DISK 'hdd'
data + INTERVAL 180 DAY DELETE
DISK 為 clickhouse 里配置的存儲卷,上面指定7天前數據存入 ssd,30天前數據存入 hdd,180天前數據直接刪除.
九.使用副本表來代替備份
關於副本表和分片表,單獨開篇學習.
所有代碼引用自 altinity webinar