八 Clickhouse 應用案例

1 用戶和權限

在user.xml中添加用戶配置

<?xml version="1.0"?>
<yandex>
<profiles>
•    <default>
•      <max_memory_usage>10000000000</max_memory_usage>
•      <use_uncompressed_cache>0</use_uncompressed_cache>
•      <load_balancing>random</load_balancing>
•    </default>
•    <readonly>
•      <readonly>1</readonly>
•    </readonly>
  </profiles>
  <users>
•    <default>
•      <password></password>
•      <networks incl="networks" replace="replace">
•        <ip>::/0</ip>
•      </networks>
•      <profile>default</profile>
•      <quota>default</quota>
•    </default>
<!-- 定義一個用戶 用戶名為hangge 密碼使用明文 root -->
<hangge>
<password_sha256_hex>f493c8a7a3c37088731336766459cc37e4b094e95b918038726660cc42013fcd</password_sha256_hex>
•      <networks incl="networks" replace="replace">
•        <ip>::/0</ip>
•      </networks>
•      <profile>default</profile>
•      <quota>default</quota>
•    </hangge>
  </users>
  <quotas>
•    <default>
•      <interval>
•        <duration>3600</duration>
•        <queries>0</queries>
•        <errors>0</errors>
•        <result_rows>0</result_rows>
•        <read_rows>0</read_rows>
•        <execution_time>0</execution_time>
•      </interval>
•    </default>
  </quotas>
</yandex>

　　

明文密碼

<password></password> 里面沒有配置說明是沒有密碼

　　

SHA256加密：

在使用SHA256加密算法的時候，需要通過password_sha256_hex標簽定義密碼

[root@ck1 ~]#  echo -n hangge | openssl dgst -sha256    
​
(stdin)= f493c8a7a3c37088731336766459cc37e4b094e95b918038726660cc42013fcd

　　

<hangge>
 <password_sha256_hex>f493c8a7a3c37088731336766459cc37e4b094e95b918038726660cc42013fcd</password_sha256_hex>
            <networks incl="networks" replace="replace">
                <ip>::/0</ip>
            </networks>
            <profile>default</profile>
            <quota>default</quota>
</hangge>

　　

double_sha1加密:

在使用double_sha1加密算法的時候，則需要通過 password_double_sha1_hex標簽定義密碼，

<password_double_sha1_hex>23ae809ddacaf96af0fd78ed04b6a265e05aa257</password_double_sha1_hex> 
​
\# echo -n 123 | openssl dgst -sha1 -binary | openssl dgst -sha1 
​
(stdin)= 23ae809ddacaf96af0fd78ed04b6a265e05aa257 

　　

用戶權限控制'

<hangge>
<password_sha256_hex>60cd41aedc4e47e8883682b416109e7b7e345e15decc63c2c98ecdab5e8e053a</password_sha256_hex>
    <networks incl="networks" />
    <profile>readonly</profile>   
    <quota>default</quota>
    <allow_databases>
        <database>default</database>
    </allow_databases>
</hangge>
​
​
​
## 3 JDBC和客戶端工具
​
### 3.1 JDBC
​
•```xml
 <!-- https://mvnrepository.com/artifact/ru.yandex.clickhouse/clickhouse-jdbc -->
    <dependency>
        <groupId>ru.yandex.clickhouse</groupId>
        <artifactId>clickhouse-jdbc</artifactId>
        <version>0.2.4</version>
    </dependency>
public class Demo1 {
​
    public static void main(String[] args) throws Exception {
        Class.forName("ru.yandex.clickhouse.ClickHouseDriver");
        String url = "jdbc:clickhouse://linux01:8123/default";
        String username = "default";
        String password = "";
        Connection con = DriverManager.getConnection(url, username, password);
        Statement stmt = con.createStatement();
        ResultSet resultSet = stmt.executeQuery("select * from tb_demo1");
        while (resultSet.next()) {
            int id = resultSet.getInt("id");
            String name = resultSet.getString("name");
            System.out.println(id + ":" + name);
        }
        con.close();
        stmt.close();
        resultSet.close();
    }
}

　　

高可用模式允許設置多個host地址，每次會從可用的地址中隨機選擇一個進行連接,在高可用模式下，需要通過BalancedClickhouseDataSource對象獲取連接

 public static void main(String[] args) throws Exception {
        // 初始化驅動
        Class.forName("ru.yandex.clickhouse.ClickHouseDriver");
        // url
        String url = "jdbc:clickhouse://linux01:8123,linux02:8123,linux03:8123/default";
        //設置JDBC參數
        ClickHouseProperties clickHouseProperties = new ClickHouseProperties();
        clickHouseProperties.setUser("default");
​
        //聲明數據源
        BalancedClickhouseDataSource balanced = new BalancedClickhouseDataSource(url, clickHouseProperties);
        //對每個host進行ping操作, 排除不可用的dead連接
        balanced.actualize();
        //獲得JDBC連接
        Connection con = balanced.getConnection();
        Statement stmt = con.createStatement();
        ResultSet resultSet = stmt.executeQuery("select * from demo3_all");
        while (resultSet.next()) {
            int id = resultSet.getInt("id");
            String name = resultSet.getString("name");
​
            System.out.println(id + ":" + name);
        }
​
        con.close();
        stmt.close();
        resultSet.close();
    }

3.2 客戶端工具DBeaver

 

 

 

 http://ui.tabix.io/#!/login 提供的一個頁面可視化工具

 

 

 

 

 

4 用戶行為分析

4.1 windowFunnel函數

(參數一) 時間的單位 窗口的大小 時間的單位 (時間 , 事件鏈條)

uid1 event1 1551398404
uid1 event2 1551398406
uid1 event3 1551398408
uid2 event2 1551398412
uid2 event3 1551398415
uid3 event3 1551398410
uid3 event4 1551398413
————————————————
-- 建表
drop table if exists test_funnel ;
CREATE TABLE test_funnel(
    uid String, 
    eventid String, 
    eventTime UInt64) 
ENGINE = MergeTree 
ORDER BY (uid, eventTime) ;
-- 導入數據
insert into test_funnel values
('uid1','event1',1551398404),
('uid1','event2',1551398406),
('uid1','event3',1551398408),
('uid2','event2',1551398412),
('uid2','event3',1551398415),
('uid3','event3',1551398410),
('uid3','event4',1551398413);
-- 查看數據
┌─uid──┬─eventid─┬──eventTime─┐
│ uid1 │ event1  │ 1551398404 │
│ uid1 │ event2  │ 1551398406 │
│ uid1 │ event3  │ 1551398408 │
│ uid2 │ event2  │ 1551398412 │
│ uid2 │ event3  │ 1551398415 │
│ uid3 │ event3  │ 1551398410 │
│ uid3 │ event4  │ 1551398413 │
└──────┴─────────┴────────────┘
​
select
uid ,
windowFunnel(4)(
toDateTime(eventTime),
eventid='event1' ,
eventid='event2' ,
eventid='event3' 
) as funnel
from
test_funnel 
group by uid ;
┌─uid──┬─funnel─┐
│ uid3 │      0 │
│ uid1 │      3 │
│ uid2 │      0 │
└──────┴────────┘
select
uid ,
windowFunnel(4)(
toDateTime(eventTime),
eventid='event2' ,
eventid='event3' 
) as funnel
from
test_funnel 
group by uid ;
┌─uid──┬─funnel─┐
│ uid3 │      0 │
│ uid1 │      2 │
│ uid2 │      2 │
└──────┴────────┘

　　

案例

建表 導入數據

clickhouse-client -q 'insert into test1.ods_log format JSONAsString'  < event.log 
​
drop table if exists test_log ;
create table test_log engine=MergeTree() order by (id,ts)
as
with visitParamExtractUInt(line,'timeStamp') as ts ,
visitParamExtractString(line ,'account')as account,
visitParamExtractString(line ,'deviceId')as deviceId,
visitParamExtractString(line ,'sessionId')as sessionId,
visitParamExtractString(line ,'ip')as ip,
visitParamExtractString(line ,'eventId')as eventId,
visitParamExtractRaw(line ,'properties')as properties
select 
if(account='' , deviceId , account) id ,
account ,
deviceId,
sessionId,
ip,
eventId,
properties,
ts
from
tb_ods_log  ;

　　

select
id ,
windowFunnel(100000)(
    toDateTime(ts),
    eventId='productView' ,
    eventId='adClick' ,
    eventId='productView' ,
    eventId='collect' 
) as funnel
from
test_log 
group by  id ;

　　

 

4.2 sequenceCount

sequenceCount滿足要求的次數

SELECT   id,   
sequenceCount('(?1)')
( 
 FROM_UNIXTIME(ts) ,    
 eventId='adShow' ,
 eventId='productView' ,
 eventId='collect' ,
 eventId='addCart' )AS cnt 
 FROM test_log 
 GROUP BY id having id='0T7136zA3BZI';

　　

4.3 sequenceMatch

這個函數都需要指定模式串、時間列和期望的事件序列（最多可指定32個事件）。模式串的語法有以下三種： (?N)：表示時間序列中的第N個事件，從1開始。例如上述SQL中，(?2)即表示event_type = 'shtKkclick' AND column_type = 'homePage'。 (?t op secs)：插入兩個事件之間，表示它們發生時需要滿足的時間條件（單位為秒）。例如上述SQL中，(?1)(?t<=15)(?2)即表示事件1和2發生的時間間隔在15秒以內。 .*：表示任意的非指定事件。

SELECT
  id, 
  sequenceMatch('(?1)(?t<=10)(?2)(?3).*(?4)')(
   FROM_UNIXTIME(ts) ,  
   eventId='adClick' ,
   eventId='productView' ,
   eventId='collect' ,
   eventId='addCart' 
  ) AS is_match
FROM log
GROUP BY id
having id='dGHDHV7WOrpJ';
​
​
SELECT
  id, 
  sequenceCount('(?1)(?t<=10)(?2)(?3).*(?4)')(
   FROM_UNIXTIME(ts) ,  
   eventId='adClick' ,
   eventId='productView' ,
   eventId='collect' ,
   eventId='addCart' 
  ) AS cnt
FROM test_log
GROUP BY id ;
​
​
having id='dGHDHV7WOrpJ 0T7136zA3BZI';
​
​
select id ,eventId from test_log
 where id = 'dGHDHV7WOrpJ'