Elasticsearch Jar包准備
所有節點導入elasticsearch-hadoop-5.5.1.jar
/opt/cloudera/parcels/CDH-5.12.0-1.cdh5.12.0.p0.29/lib/hive/lib/elasticsearch-hadoop-5.5.1.jar
HDFS導入數據准備
hdfs dfs -ls /user/logb/464/part-r-00000
進入HIVE shell 執行
引用Elasticsearch jar包進行hive界面
hive -hiveconf hive.aux.jars.path=file:///usr/local/elasticsearch/elasticsearch-hadoop-5.5.1.jar
創建與Elasticsearch對接log_apache_seo_d1外部表
create external table log_apache_seo_d1 (ipaddress string,uniqueid string,url string, sessionid string ,sessiontimes string, areaaddress string ,localaddress string , browsertype string,operationsys string,refeurl string , receivetime string ,userid string ) STORED BY 'org.elasticsearch.hadoop.hive.EsStorageHandler' TBLPROPERTIES('es.resource' = 'radiott/artiststt','es.index.auto.create' = 'true','es.nodes' = 'node4','es.port' = '9200');
創建源數據表log_apache_seo_source_d1
CREATE TABLE log_apache_seo_source_d1 (ipaddress string,uniqueid string,url string, sessionid string ,sessiontimes string, areaaddress string ,localaddress string , browsertype string,operationsys string,refeurl string , receivetime string ,userid string ) row format delimited fields terminated by '\t' stored as textfile;
加載MR結果到HIVE
load data inpath '/user/logb/464/part-r-00000' into table log_apache_seo_source_d1 ;
將HIVE數據加載到Elasticsearch所需表中
insert overwrite table log_apache_seo_d1 select s.ipaddress,s.uniqueid,s.url,s.sessionid,s.sessiontimes,s.areaaddress,s.localaddress,s.browsertype,s.operationsys,s.refeurl,s.receivetime,s.userid from log_apache_seo_source_d1 s;
編寫shell腳本
#!/bin/sh
# upload logs to hdfs
hive -e "
set hive.enforce.bucketing=true;
set hive.exec.compress.output=true;
set mapred.output.compress=true;
set mapred.output.compression.codec=org.apache.hadoop.io.compress.GzipCodec;
set io.compression.codecs=org.apache.hadoop.io.compress.GzipCodec;
load data inpath '/user/logb/464/part-r-00000' into table log_apache_seo_source_d1 ;
"
執行腳本任務
0 */2 * * * /opt/bin/hive_opt/crontab_import.sh