DB2安裝步驟

##################################DB2的安裝###########################
## 安裝前准備
## 關閉內存地址隨機化機制
    vi /etc/sysctl.conf
#增加 kernel.randomize_va_space=0
    sysctl -p
  
##上傳壓縮包至/user/local目錄下，並解壓
    cd /home/db2
    #tar -xvf v9.7fp9_linuxx64_server.tar.gz
    tar -xvfv10.5_linuxx64_expc.tar.gz
    cd /usr/local/expc
## 開始安裝
    ./db2_install
2.配置實例用戶
 ## root 用戶執行
    mkdir -p /db2home
    groupadd -g 607 dagadm
    groupadd -g 608 dagfenc
    useradd -u 607 -g dagadm -d /db2home/dainst -m dainst
    useradd -u 608 -g dagfenc -d /home/dafenc -m dafenc
    useradd -u 609 -g dagadm -d /db2home/dw -m dw

## 修改配置文件,添加端口號
    vi /etc/services
  #dainst
    db2c_dainst     62000/tcp
    DB2_dainst      62001/tcp
    DB2_dainst_1    62002/tcp
    DB2_dainst_2    62003/tcp
    DB2_dainst_END  62004/tcp

 ## 新建實例
    cd /opt/ibm/db2/V10.5/instance
    ./db2icrt  -u dafenc dainst

 ## dainst 用戶執行
    su - dainst
    db2set DB2CODEPAGE=1386
    db2set DB2COMM=TCPIP
    db2set DB2_PARALLEL_IO=*

    db2 update dbm cfg using SVCENAME db2c_dainst
    db2 get dbm cfg|grep -w SVCENAME

    cat /db2home/dainst/sqllib/db2nodes.cfg

    db2start
##創建DWMM數據庫（用實例用戶執行）
  mkdir -p /db2home/dainst/etldb
  db2 "CREATE DATABASE dwmm AUTOMATIC STORAGE NO ON /db2home/dainst/etldb USING CODESET GBK TERRITORY CN RESTRICTIVE"
  

 

#系統安裝預備初始化
#-------------------------系統安裝預備初始化-----------------------------
1.防火牆操作：
    #關閉防火牆
    service iptables stop 
    #開啟防火牆：service iptables start
    #禁止自動啟動防火牆
    chkconfig iptables off
    #開啟自動啟動防火牆 ：chkconfig iptables on 
    查看防火牆：/etc/init.d/iptables status
2.命名節點名稱
    vi /etc/sysconfig/network
3.配置host文件
    vi /etc/hosts
    #如：192.168.46.131  hadoop
4.修改系統時間時區
    rm -rf /etc/localtime
    cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime

5.ssh免密登錄
    #對每個節點分別產生公鑰和私鑰：
    cd  ~/.ssh
    ssh-keygen -t dsa -P '' -f ~/.ssh/id_dsa
    #將公鑰文件復制保存為authorized_keys
    cat id_dsa.pub >> authorized_keys

##python安裝升級
#-------------------------python安裝升級----------------------
1.將python安裝包Python-2.7.13.tgz上傳至/usr/local/目錄下
    cd /usr/local
2.解壓安裝包
    tar -zxvf Python-2.7.13.tgz 
    cd /usr/local/Python-2.7.13
3.指定python安裝目錄
    ./configure --prefix=/usr/local/python-2.7/
    make && make install
    cd /usr/bin/
    ls python*
4.刪除原來的python
    rm -rf /usr/bin/python
5.創建軟鏈接
ln -s /usr/local/python-2.7.13/bin/python2  /usr/bin/python
6.驗證python版本
python -V
7.成功后可以刪除源文件,保留安裝包文件
cd /usr/local
rm -rf Python-2.7.13

#---------------------未安裝成功---------------------
#上傳setuptools包到/usr/local目錄下
cd /usr/local/
#解壓 setuptools-2.0.tar.gz包
tar -zxvf setuptools-2.0.tar.gz
cd setuptools-2.0
python setup.py install
----------------------------------------------------

#jdk安裝
#-------------------------jdk安裝-----------------------------
1.將jdk安裝包jdk-8u51-linux-x64.gz上傳至/usr/local/目錄下
    cd /usr/local
2.解壓jdk安裝包
    tar  -zxvf  jdk-8u51-linux-x64.gz
    #重命名安裝包
    mv jdk1.8.0_51 jdk
3.配置環境變量
    vi /etc/profile
    export JAVA_HOME=/usr/local/jdk
    export JAVA_BIN=$JAVA_HOME/bin
    export JAVA_LIB=$JAVA_HOME/lib
    export CLASSPATH=.:$JAVA_LIB/tools.jar:$JAVA_LIB/dt.jar
    #最后將所有的路徑加到
    export PATH=.:$JAVA_BIN:$PATH
    #使環境變量生效
    source /etc/profile
4.驗證jdk安裝是否成功
    java -version

#mysql安裝
#-----------------------安裝mysql------------------------
1.上傳mysql安裝包到/usr/local目錄下
cd /usr/local
2.卸載依賴包
    #查找安裝了的mysql
    rpm -qa | grep mysql
    #如果有,則執行命令卸載
    rpm -e  mysql-libs-5.1.71-1.el6.x86_64  --nodeps
2.安裝mysql
    rpm -ivh MySQL-client-5.6.22-1.el6.x86_64.rpm  --nodeps
    rpm -ivh MySQL-server-5.6.22-1.el6.x86_64.rpm  --nodeps
3.啟動mysql服務
service  mysql start
4.查看root賬號密碼並登陸
    cat /root/.mysql_secret   #JElTlG9gVsKAhRKS  WQlcq4jUug_KK_ZY
    #登錄mysql
    mysql -uroot -p密碼
    #設置密碼
    mysql> SET PASSWORD = PASSWORD('root');
    #測試新密碼登錄
    mysql -uroot -proot 
5設置允許遠程登錄
    GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' IDENTIFIED BY 'root' WITH GRANT OPTION;
flush privileges;
    exit;
6.設置開機自動啟動
chkconfig mysql on

#安裝hadoop
#------------------------hadoop安裝--------------------------
1.上傳hadoop安裝包到/usr/local目錄下
    #進入/usr/local/目錄下
    cd /usr/local
2.解壓hadoop安裝包
    tar -xzvf  hadoop-2.6.0.tar.gz
    #重命令hadoop
    mv hadoop-2.6.0 hadoop 
3.設置hadoop環境變量
    vi /etc/profile
    export HADOOP_HOME=/usr/local/hadoop
    #修改：
    export PATH=.:$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$JAVA_BIN:$PATH 
4.配置hadoop的參數
#4.1 修改hadoop-env.sh文件
    #添加java_home的環境變量
    cd /usr/local/hadoop/etc/hadoop
    vi hadoop-env.sh
    JAVA_HOME=/usr/local/jdk
#4.2 配置core-site.xml
    cd /usr/local/hadoop/etc/hadoop
    vi core-site.xml
<configuration>
    <property>
        <name>fs.default.name</name>
        <value>hdfs://hadoop:9000</value>
        <description>change your own hostname</description>
    </property>
    <property>
        <name>hadoop.tmp.dir</name>
        <value>/usr/local/hadoop/tmp</value>
    </property>  
</configuration>
  
#4.3 配置hdfs-site.xml  
  <configuration>
    <property>
        <name>dfs.replication</name>
        <value>1</value>
    </property>
    <property>
        <name>dfs.permissions</name>
        <value>false</value>
    </property>
  </configuration>
  
#4.4.配置mapred-site.xml
  <configuration>
    <property>
        <name>mapred.job.tracker</name>
        <value>hadoop:9001</value>
        <description>change your own hostname</description>
    </property>
</configuration>

9.64位系統錯誤問題處理
##安裝Hadoop啟動之后總有警告：
##Unable to load native-hadoop library for your platform... using builtin-Javaclasses where applicable
##這是因為在64位的linux系統中運行hadoop不兼容。
##這時候將准備好的64位的lib包解壓到已經安裝好的hadoop的lib目錄下
#注意：是lib目錄，而不是lib下的native目錄
##執行如下命令：
#tar -x hadoop-native-64-2.4.0.tar -C  hadoop/lib/
    cd /usr/local
    mv hadoop-native-64-2.6.0.tar hadoop/lib
    cd /usr/local/hadoop/lib
    tar -xvf  hadoop-native-64-2.6.0.tar
#然后在環境變量中添加如下內容：
    vi /etc/profile
    export HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_HOME/lib/native
    export HADOOP_OPTS="-Djava.library.path=$HADOOP_HOME/lib"
#最后使環境變量生效
    source  /etc/profile

5.對hadoop進行格式化
    hadoop namenode -format
6.啟動hadoop
    start-all.sh
7.驗證hadoop是否安裝成功：
 輸入命令：jps 
 #發現有五個java進程：
    DataNode
    NameNode
    SecondaryNameNode
    JobTracker
    TaskTracker
 #通過瀏覽器查看：
    HDFS：
        hadoop:50070 
    MapReduce：
        hadoop:50030 
 
8.修改windows下的文件,即可在本地電腦查看：
    C:\Windows\System32\drivers\etc\hosts

10.錯誤處理辦法
    如果在windows中頁面不能成功，有肯能
    NameNode進程啟動沒有成功？
    1.沒有格式化
    2.配置文件
    3.hostname沒有與ip綁定
    4.SSH的免密碼登錄沒有配置成功
    #多次格式化也是錯誤的
    方法：刪除/usr/local/hadoop/tmp文件夾，重新格式化


#---------------------hive安裝----------------------------
1.上傳hive安裝包到/usr/local目錄下
cd /usr/local
2.解壓hive安裝包
tar -zxvf hive-0.9.0.tar.gz
mv hive-0.9.0 hive
3.配置hive環境變量
vi /etc/profile
export HIVE_HOME=/usr/local/hive
export PATH=.:$HIVE_HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$JAVA_BIN:$PATH 
source /etc/profile
4.上傳驅動到/usr/local目錄下並添加驅動到hive的lib目錄下
cd /usr/local
cp mysql-connector-java-5.1.39-bin.jar  /usr/local/hive/lib/
5.hive安裝參數配置
#修改hive-env.sh文件，添加hadoop的環境變量
    cd /usr/local/hive/conf
    cp hive-env.sh.template  hive-env.sh
    vi hive-env.sh
    export HADOOP_HOME=/usr/local/hadoop
#修改hive-log4j.properties文件
    cd /usr/local/hive/conf
    cp hive-log4j.properties.template hive-log4j.properties
    vi hive-log4j.properties
#log4j.appender.EventCounter=org.apache.hadoop.metrics.jvm.EventCounter
    log4j.appender.EventCounter=org.apache.hadoop.log.metrics.EventCounter

#修改hive-site.xml文件
    cd /usr/local/hive/conf
    cp hive-default.xml.template hive-site.xml
    vi hive-site.xml
#添加如下內容：
<property>
  <name>javax.jdo.option.ConnectionURL</name>
  <value>jdbc:mysql://hadoop:3306/hive?createDatabaseIfNotExist=true</value>
  <description>JDBC connect string for a JDBC metastore</description>
</property>
<property>
  <name>javax.jdo.option.ConnectionDriverName</name>
  <value>com.mysql.jdbc.Driver</value>
  <description>Driver class name for a JDBC metastore</description>
</property>
<property>
  <name>javax.jdo.option.ConnectionUserName</name>
  <value>root</value>
  <description>username to use against metastore database</description>
</property>
<property>
  <name>javax.jdo.option.ConnectionPassword</name>
  <value>root</value>
  <description>password to use against metastore database</description>
</property>
6.驗證hive安裝是否成功
 hive
 show databases;
 mysql -uroot -proot 
 show databases;
 use hive;
 show tables;
7.上傳hive-builtins-0.9.0.jar包到hdfs的/usr/local/hive/lib/目錄下
 cd /usr/local/hive/lib
 hdfs dfs -mkdir -p  /usr/local/hive/lib
 hdfs dfs -put hive-builtins-0.9.0.jar /usr/local/hive/lib
 
#--------------------------DB2安裝-----------------------------------#

1.關閉內存地址隨機化機制
    vi /etc/sysctl.conf
    增加 kernel.randomize_va_space=0
    sysctl -p
2.上傳db2安裝包並解壓安裝
    mkdir /home/db2
    cd /home/db2
    tar -xvf v9.7fp9_linuxx64_server.tar.gz
    cd ./server
    ./db2_install #開始安裝db2,選擇ESE企業版安裝
3.配置實例用戶,root用戶執行
    mkdir -p /db2home
    groupadd -g 607 dagadm
    groupadd -g 608 dagfenc
    useradd -u 607 -g dagadm -d /db2home/dainst -m dainst
    useradd -u 608 -g dagfenc -d /home/dafenc -m dafenc
    useradd -u 609 -g dagadm -d /db2home/dw -m dw
    
    passwd dainst dainst
    passwd dafenc dafenc
    passwd dw dw 
4.修改配置文件,添加端口號
    vi /etc/services
    db2c_dainst     62000/tcp
    DB2_dainst      62001/tcp
    DB2_dainst_1    62002/tcp
    DB2_dainst_2    62003/tcp
    DB2_dainst_END  62004/tcp
5.新建實例
    cd /opt/ibm/db2/V9.7/instance
    ./db2icrt -s ese -u dafenc dainst
6.數據庫配置,dainst用戶執行
    su - dainst
    db2set DB2CODEPAGE=1386
    db2set DB2COMM=TCPIP
    db2set DB2_PARALLEL_IO=*
    db2 update dbm cfg using SVCENAME db2c_dainst
    db2 get dbm cfg|grep -w SVCENAME
    cat /db2home/dainst/sqllib/db2nodes.cfg
    db2start
7.創建DWMM數據庫
    mkdir -p /db2home/dainst/etldb
    db2 "CREATE DATABASE dwmm AUTOMATIC STORAGE NO ON /db2home/dainst/etldb USING CODESET GBK TERRITORY CN RESTRICTIVE"

#mysql導入數據
--全量替換數據
load data local infile "/home/python/data/stock.csv" replace into table stock CHARACTER SET gb2312 FIELDS TERMINATED BY "," LINES TERMINATED BY "\n";
--新增數據
load data local infile '/etl/etldata/input/20170216/industry_20170216.csv' into table ext_stock_industry fields terminated by "," lines terminated  by '\n';

#將ext外表指向相應的目錄
"ALTER TABLE ext_${tab_name} SET LOCATION 'hdfs://nameservice1/etldata/input/${tab_name}'"
## 清空STG增量表 ###
  echo "`dt` :start truncate table STG_${tab_name}"
  ${TDH} "truncate table STG_${tab_name}" 
  [ $? -ne 0 ] && echo "`dt` :truncate table STG_${tab_name} failed " && exit -1
  echo "`dt` :truncate Table STG_${tab_name} succeed "

  ## 將每日增量數據裝入STG表 ###
  echo "`dt` :start insert STG_${tab_name} from EXT_${tab_name}"
  ${TDH} "insert into STG_${tab_name} select A.*,${today} from EXT_${tab_name} A" 
  [ $? -ne 0 ] && echo "`dt` :insert STG_${tab_name} failed " && exit -1
  echo "`dt` :insert into Table STG_${tab_name} succeed " 
  return 0
  
  
 
#清空調度環境
1.進入清空etl調度表腳本目錄
cd /home/dainst/tmp/jjm/init
2.執行rollback.sh腳本清空環境
./rollback.sh

#配置作業流
1.配置db2驅動
在本機運行：
C:\Windows\SysWOW64\odbcad32.exe
數據源名稱：dydwmm
用戶標識：dainst
密碼：dainst
IP:192.168.101.90
端口號：61000
2.打開excel提交作業流
#在本機打開excel文件
C:\Users\Idea\Desktop\數據項目\05-大數據平台\02-數據處理\ETL\大數據平台部署\06-相關文檔
#刪除右側文件
分別提交3個sheet頁

3.檢查作業配置是否成功
 #在waterdrop中執行如下操作檢查：
SELECT * FROM etl.job_loc;
SELECT * FROM etl.job_metadata;
SELECT * FROM etl.job_seq;
#查看跑批日期
SELECT * FROM etl.job_biz_date;

4.清空歷史目錄
#清空省聯社下發文件目錄
rm -rf /etl/etldata/input/S-999000/CCRD/ADD/*
rm -rf /etl/etldata/input/S-999000/CORE/ADD/*
rm -rf /etl/etldata/input/S-999000/FMS/ADD/*
rm -rf /etl/etldata/input/S-999000/FMS/ADD/*
#清空dw數據庫導出數據目錄
rm -rf /etl/etldata/input/DW_TO_HIVE/*

5.開啟調度
cd /etl/etldata/script/job-schd-engine-0.1Silver/bin
#先停止調度
mv nohup.out nohup.out_tmp
#配置調度啟動時間
cd /etl/etldata/script/job-schd-engine-0.1Silver/config
vi quartz_jobs.xml
#開啟調度
cd /etl/etldata/script/job-schd-engine-0.1Silver/bin
nohup start.sh &
6.前台查看調度頁面
 
 推送省聯社下發文件
 cd  /home/dainst/tmp/jjm/Note/CORE
./tools.sh  20170221 FMS
./tools.sh  20170221 IMBS
./tools.sh  20170221 CCRD
./tools.sh  20170221 CORE

  DW庫中握手文件
  cd /etl/etldata/input/DW_TO_HIVE/20170220
  touch HANDFILE.OK
 7.第一個批成功跑完
 修改視圖
 drop view ETL.JOB_BIZ_DATE;
    CREATE VIEW JOB_BIZ_DATE(
        CUR_DT,
        LAST_DT,
        PPN_TSTMP
        ) AS SELECT
        '2017-02-22' AS CUR_DT,
        '2016-02-21' AS LAST_DT,
        CURRENT TIMESTAMP AS PPN_TSTMP
        FROM
        CORE.BGFMCINF
        WHERE
        ETL_FLAG = 'I'
        
select * from ETL.JOB_BIZ_DATE

修改時間配置，實例化第二批次
      cd /etl/etldata/script/job-schd-engine-0.1Silver/config
      vi quartz_jobs.xml
      
創建壓縮文件,並推送到對應目錄上
cd  /home/dainst/tmp/jjm/Note/CORE
./tools.sh  20170221 FMS
./tools.sh  20170221 IMBS
./tools.sh  20170221 CCRD
./tools.sh  20170221 CORE

查看前台調度頁面
省聯社文件解壓成功，后續作業成功執行

1.在前台頁面修改FILE_CHECK_CRM作業名稱為FILE_CHECK_DW

調度管理-作業管理-作業場景選擇FILE-查詢
選中FILE_CHECK_CRM作業，點擊編輯按鈕
將FILE_CHECK_CRM作業的作業名稱和互斥組改為FILE_CHECK_DW

select * from ETL.JOB_METADATA WHERE job_tp = 'FILE';
FILE_CHECK_CRM已經消失，變為FILE_CHECK_DW

select * from ETL.JOB_log WHERE job_nm LIKE 'FILE%';
4839948    WAITING    FILE_CHECK_CRM    2017-02-21        


2.給DW抽數據握手文件，觀察調度能否繼續執行
touch /etl/etldata/input/DW_TO_HIVE/20170221/HANDFILE.OK
如果實例化批之后，修改了作業名稱，作業不能繼續往下執行

在實例化作業之前，修改作業名稱查看作業能否執行

該作業能繼續執行，但依賴於該作業的后續作業也會執行
#一個作業依賴於一個不存在的作業，看看該作業能否執行？？？
CMD_CHK_COMMUNICATE_FILE作業依賴於FILE_CHECK_CRM,在未實例化批之前，
在前台將FILE_CHECK_CRM改名為FILE_CHECK_DW。 
FILE_CHECK_DW能繼續執行
CMD_CHK_COMMUNICATE_FILE也會繼續執行。

#前台刪除作業



修改視圖
drop view ETL.JOB_BIZ_DATE;
    CREATE VIEW JOB_BIZ_DATE(
        CUR_DT,
        LAST_DT,
        PPN_TSTMP
        ) AS SELECT
        '2017-02-20' AS CUR_DT,
        '2016-02-19' AS LAST_DT,
        CURRENT TIMESTAMP AS PPN_TSTMP
        FROM
        CORE.BGFMCINF
        WHERE
        ETL_FLAG = 'I'
 select * from ETL.JOB_BIZ_DATE;

 
修改時間配置，實例化第二批次
      cd /etl/etldata/script/job-schd-engine-0.1Silver/config
      vi quartz_jobs.xml
 

 
創建壓縮文件,並推送到對應目錄上
cd  /home/dainst/tmp/jjm/Note/CORE
./tools.sh  20170220 FMS
./tools.sh  20170220 IMBS
./tools.sh  20170220 CCRD
./tools.sh  20170220 CORE

4.
SELECT * FROM etl.job_seq WHERE PRE_JOB='CMD_CHK_COMMUNICATE_FILE';
update etl.job_seq set PRE_JOB='FILE_CHECK_DW' where PRE_JOB='CMD_CHK_COMMUNICATE_FILE';
SELECT * FROM etl.job_seq WHERE PRE_JOB='FILE_CHECK_DW';


-------------------------------------------
233數據庫中：
SELECT count(*) FROM S_CRM_CUST_PE_BASE_INFO; --1829596條

90數據庫中
SELECT count(*) FROM ext_s_crm_cust_pe_base_info;  --1829596條
SELECT count(*) FROM stg_s_crm_cust_pe_base_info;  --1829596條
SELECT count(*) FROM sdi_s_crm_cust_pe_base_info;  --2005970條
SELECT count(*) FROM his_s_crm_cust_pe_base_info;  --0條

重做作業：
再次查詢數據庫信息：

#備份bdpdb中數據庫的s_crm_cust_pe_base_info表
drop table if exists sdi_s_crm_cust_pe_base_info_20170224;
CREATE table sdi_s_crm_cust_pe_base_info_20170224 AS SELECT * FROM sdi_s_crm_cust_pe_base_info;
select count(*) from sdi_s_crm_cust_pe_base_info_20170224;

drop table if exists stg_s_crm_cust_pe_base_info_20170224;
CREATE table stg_s_crm_cust_pe_base_info_20170224 AS SELECT * FROM stg_s_crm_cust_pe_base_info;
select count(*) from stg_s_crm_cust_pe_base_info_20170224;

drop table if exists ext_s_crm_cust_pe_base_info_20170224;
CREATE table ext_s_crm_cust_pe_base_info_20170224 AS SELECT * FROM ext_s_crm_cust_pe_base_info;


drop table if exists his_s_crm_cust_pe_base_info_20170224;
CREATE table his_s_crm_cust_pe_base_info_20170224 AS SELECT * FROM his_s_crm_cust_pe_base_info;
select count(*) from his_s_crm_cust_pe_base_info_20170224;


#創建新表
DROP TABLE SDI_S_CRM_CUST_PE_BASE_INFO;
CREATE TABLE IF NOT EXISTS SDI_S_CRM_CUST_PE_BASE_INFO (
        CUST_NO VARCHAR(32) comment "CRM客戶號",
        CERT_TYPE VARCHAR(8) comment "證件類型(清洗后的)",
        CERT_NO VARCHAR(20) comment "證件號(清洗后的)",
        PECUST_NAME VARCHAR(80) comment "客戶名稱",
        CUSTNAME_SHORT VARCHAR(40) comment "客戶簡稱",
        CUST_NAME_EN VARCHAR(100) comment "客戶名稱(英文)",
        VIP_CATE VARCHAR(8) comment "貴賓類別",
        COUNTRY VARCHAR(8) comment "國家",
        CREDIT_LEVEL VARCHAR(8) comment "信用等級",
        IS_FREETAX VARCHAR(8) comment "免稅標識",
        BTFLBOOL VARCHAR(8) comment "移行客戶標志",
        DIMABOOL VARCHAR(8) comment "是否接收推廣信息",
        SEX VARCHAR(8) comment "性別",
        NTY VARCHAR(8) comment "民族",
        MRG VARCHAR(8) comment "婚姻狀況",
        BTD VARCHAR(10) comment "出生日期",
        STUDY_EXP VARCHAR(8) comment "最高學歷",
        DEGREE VARCHAR(8) comment "最高學位",
        PAY_ACCT VARCHAR(32) comment "工資賬號",
        ACCT_WT_BK VARCHAR(80) comment "開戶銀行",
        YEAR_INCOME DECIMAL(16,2) comment "年收入",
        FMY_PPL INTEGER comment "家庭人數",
        INHBT_STAT VARCHAR(8) comment "居住狀況",
        CUST_LEV VARCHAR(8) comment "客戶層次",
        BANK_EMP_IND VARCHAR(8) comment "本行員工標志",
        EMPLOYEE_TYP VARCHAR(8) comment "員工類別",
        BANK_STK_HOLDER_IND VARCHAR(8) comment "是否本行股東",
        BANK_PARTY_IND VARCHAR(8) comment "是否本行關系人",
        NOTES VARCHAR(200) comment "本行關系人備注",
        XDZX_TYPE VARCHAR(8) comment "客戶信貸類型",
        CUST_QLY VARCHAR(8) comment "客戶性質",
        PERSONAL_INSRC VARCHAR(8) comment "人身保險",
        INSURANCE_DT VARCHAR(10) comment "保險期限",
        BAD_RECORD DECIMAL(5,0) comment "不良記錄筆數",
        IS_LTC_CREDIT_CUST VARCHAR(8) comment "是否潛在信貸客戶",
        NATIVE VARCHAR(200) comment "籍貫",
        EFF_DATE VARCHAR(8) comment "批量日期",
        END_DATE VARCHAR(8) comment "數據失效日期",
        JOB_SEQ_ID VARCHAR(8) comment "批次號"
) comment "對私客戶基本信息"
CLUSTERED BY (CUST_NO) INTO 11 BUCKETS
STORED AS ORC
TBLPROPERTIES ("transactional"="true");

DROP TABLE STG_S_CRM_CUST_PE_BASE_INFO;
CREATE TABLE IF NOT EXISTS STG_S_CRM_CUST_PE_BASE_INFO (
        CUST_NO string comment  "CRM客戶號",
        CERT_TYPE string comment  "證件類型(清洗后的)",
        CERT_NO string comment  "證件號(清洗后的)",
        PECUST_NAME string comment  "客戶名稱",
        CUSTNAME_SHORT string comment  "客戶簡稱",
        CUST_NAME_EN string comment  "客戶名稱(英文)",
        VIP_CATE string comment  "貴賓類別",
        COUNTRY string comment  "國家",
        CREDIT_LEVEL string comment  "信用等級",
        IS_FREETAX string comment  "免稅標識",
        BTFLBOOL string comment  "移行客戶標志",
        DIMABOOL string comment  "是否接收推廣信息",
        SEX string comment  "性別",
        NTY string comment  "民族",
        MRG string comment  "婚姻狀況",
        BTD string comment  "出生日期",
        STUDY_EXP string comment  "最高學歷",
        DEGREE string comment  "最高學位",
        PAY_ACCT string comment  "工資賬號",
        ACCT_WT_BK string comment  "開戶銀行",
        YEAR_INCOME string comment  "年收入",
        FMY_PPL string comment  "家庭人數",
        INHBT_STAT string comment  "居住狀況",
        CUST_LEV string comment  "客戶層次",
        BANK_EMP_IND string comment  "本行員工標志",
        EMPLOYEE_TYP string comment  "員工類別",
        BANK_STK_HOLDER_IND string comment  "是否本行股東",
        BANK_PARTY_IND string comment  "是否本行關系人",
        NOTES string comment  "本行關系人備注",
        XDZX_TYPE string comment  "客戶信貸類型",
        CUST_QLY string comment  "客戶性質",
        PERSONAL_INSRC string comment  "人身保險",
        INSURANCE_DT string comment  "保險期限",
        BAD_RECORD string comment  "不良記錄筆數",
        IS_LTC_CREDIT_CUST string comment  "是否潛在信貸客戶",
        ETL_BIZ_DT string comment  "ETL插入日期",
        ETL_LOAD_DT string comment  "ETL更新日期",
        NATIVE string comment  "籍貫",
        ETL_EFF_DATE VARCHAR(8) comment "批量日期"
) comment "對私客戶基本信息"
CLUSTERED BY (CUST_NO) INTO 11 BUCKETS
STORED AS ORC
TBLPROPERTIES ("transactional"="true");

DROP TABLE HIS_S_CRM_CUST_PE_BASE_INFO;
CREATE TABLE IF NOT EXISTS HIS_S_CRM_CUST_PE_BASE_INFO (
        CUST_NO VARCHAR(32) comment "CRM客戶號",
        CERT_TYPE VARCHAR(8) comment "證件類型(清洗后的)",
        CERT_NO VARCHAR(20) comment "證件號(清洗后的)",
        PECUST_NAME VARCHAR(80) comment "客戶名稱",
        CUSTNAME_SHORT VARCHAR(40) comment "客戶簡稱",
        CUST_NAME_EN VARCHAR(100) comment "客戶名稱(英文)",
        VIP_CATE VARCHAR(8) comment "貴賓類別",
        COUNTRY VARCHAR(8) comment "國家",
        CREDIT_LEVEL VARCHAR(8) comment "信用等級",
        IS_FREETAX VARCHAR(8) comment "免稅標識",
        BTFLBOOL VARCHAR(8) comment "移行客戶標志",
        DIMABOOL VARCHAR(8) comment "是否接收推廣信息",
        SEX VARCHAR(8) comment "性別",
        NTY VARCHAR(8) comment "民族",
        MRG VARCHAR(8) comment "婚姻狀況",
        BTD VARCHAR(10) comment "出生日期",
        STUDY_EXP VARCHAR(8) comment "最高學歷",
        DEGREE VARCHAR(8) comment "最高學位",
        PAY_ACCT VARCHAR(32) comment "工資賬號",
        ACCT_WT_BK VARCHAR(80) comment "開戶銀行",
        YEAR_INCOME DECIMAL(16,2) comment "年收入",
        FMY_PPL INTEGER comment "家庭人數",
        INHBT_STAT VARCHAR(8) comment "居住狀況",
        CUST_LEV VARCHAR(8) comment "客戶層次",
        BANK_EMP_IND VARCHAR(8) comment "本行員工標志",
        EMPLOYEE_TYP VARCHAR(8) comment "員工類別",
        BANK_STK_HOLDER_IND VARCHAR(8) comment "是否本行股東",
        BANK_PARTY_IND VARCHAR(8) comment "是否本行關系人",
        NOTES VARCHAR(200) comment "本行關系人備注",
        XDZX_TYPE VARCHAR(8) comment "客戶信貸類型",
        CUST_QLY VARCHAR(8) comment "客戶性質",
        PERSONAL_INSRC VARCHAR(8) comment "人身保險",
        INSURANCE_DT VARCHAR(10) comment "保險期限",
        BAD_RECORD DECIMAL(5,0) comment "不良記錄筆數",
        IS_LTC_CREDIT_CUST VARCHAR(8) comment "是否潛在信貸客戶",
        NATIVE VARCHAR(200) comment "籍貫",
        EFF_DATE VARCHAR(8) comment "批量日期",
        END_DATE VARCHAR(8) comment "數據失效日期",
        JOB_SEQ_ID VARCHAR(8) comment "批次號"
        ,NEW_JOB_SEQ_ID VARCHAR(8) comment "歷史批次號"
) comment "對私客戶基本信息"
CLUSTERED BY (CUST_NO) INTO 11 BUCKETS
STORED AS ORC
TBLPROPERTIES ("transactional"="true");

DROP TABLE EXT_S_CRM_CUST_PE_BASE_INFO;
CREATE EXTERNAL TABLE IF NOT EXISTS EXT_S_CRM_CUST_PE_BASE_INFO (
        CUST_NO string comment "CRM客戶號",
        CERT_TYPE string comment "證件類型(清洗后的)",
        CERT_NO string comment "證件號(清洗后的)",
        PECUST_NAME string comment "客戶名稱",
        CUSTNAME_SHORT string comment "客戶簡稱",
        CUST_NAME_EN string comment "客戶名稱(英文)",
        VIP_CATE string comment "貴賓類別",
        COUNTRY string comment "國家",
        CREDIT_LEVEL string comment "信用等級",
        IS_FREETAX string comment "免稅標識",
        BTFLBOOL string comment "移行客戶標志",
        DIMABOOL string comment "是否接收推廣信息",
        SEX string comment "性別",
        NTY string comment "民族",
        MRG string comment "婚姻狀況",
        BTD string comment "出生日期",
        STUDY_EXP string comment "最高學歷",
        DEGREE string comment "最高學位",
        PAY_ACCT string comment "工資賬號",
        ACCT_WT_BK string comment "開戶銀行",
        YEAR_INCOME string comment "年收入",
        FMY_PPL string comment "家庭人數",
        INHBT_STAT string comment "居住狀況",
        CUST_LEV string comment "客戶層次",
        BANK_EMP_IND string comment "本行員工標志",
        EMPLOYEE_TYP string comment "員工類別",
        BANK_STK_HOLDER_IND string comment "是否本行股東",
        BANK_PARTY_IND string comment "是否本行關系人",
        NOTES string comment "本行關系人備注",
        XDZX_TYPE string comment "客戶信貸類型",
        CUST_QLY string comment "客戶性質",
        PERSONAL_INSRC string comment "人身保險",
        INSURANCE_DT string comment "保險期限",
        BAD_RECORD string comment "不良記錄筆數",
        IS_LTC_CREDIT_CUST string comment "是否潛在信貸客戶",
        ETL_BIZ_DT string comment "ETL插入日期",
        ETL_LOAD_DT string comment "ETL更新日期",
        NATIVE string comment "籍貫"
) comment "對私客戶基本信息"
ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.csv.serde.CSVSerde'
STORED AS INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat'
  OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'
LOCATION '/etldata';

跑20170220批數據，查詢數據量
#查看新表數據量
SELECT count(*) FROM ext_s_crm_cust_pe_base_info;  --1829596
SELECT count(*) FROM stg_s_crm_cust_pe_base_info;  --1829696
SELECT count(*) FROM sdi_s_crm_cust_pe_base_info;  --1829596
SELECT count(*) FROM his_s_crm_cust_pe_base_info;  --0

重跑20170220批作業,所有作業成功跑批，再次查看數據量
SELECT count(*) FROM ext_s_crm_cust_pe_base_info;  --1829596
SELECT count(*) FROM stg_s_crm_cust_pe_base_info;  --1829696
SELECT count(*) FROM sdi_s_crm_cust_pe_base_info;  --1829596
SELECT count(*) FROM his_s_crm_cust_pe_base_info;  --0

 

 

use stock;
load data local infile '/home/crawblog/data/FILE' into table  blog fields terminated by "," lines terminated  by '\n';

 

create or replace procedure bdpdb.sp_idv_cst_smy_bk_tl_busi()
IS
trc_tab STRING
BEGIN
    trc_tab:="truncate table bdpdb.IDV_CST_SMY_BK_TL_BUSI ;";
    EXECUTE IMMEDIATE trc_tab;
insert into bdpdb.IDV_CST_SMY_BK_TL_BUSI
select date_dt         --日期
        ,branch_id     --機構號
        ,counter_no    --櫃員號
        ,tel_txid      --交易源代碼
        ,NVL(LAG(tran_time,1) OVER(partition by date_dt,counter_no,branch_id,process_time,process_end_time
        order by date_dt,branch_id,counter_no,process_time,process_end_time,tran_time),process_time) start_time --交易起始時間
        ,tran_time     --交易結束時間
    from(
        select A.date_dt      --日期
            ,A.counter_no     --櫃員號 
            ,A.branch_id      --機構號
            ,B.tel_txid       --交易源代碼
            ,A.process_time   --叫號起始時間
            ,B.tran_time        --交易時間
            ,A.process_end_time --叫號結束時間
            from
                (
                select
                substr(A.process_time,1,4)||substr(A.process_time,6,2)||substr(A.process_time,9,2) date_dt
                ,A.BRANCH_ID
                ,A.WIN_NO
                ,B.counter_no
                ,TDH_TODATE(TIMESTAMP(A.PROCESS_TIME) + interval '180' SECOND,'yyyy-MM-dd HH:mm:ss','yyyyMMddHHmmss') PROCESS_TIME --叫號時間
                ,TDH_TODATE(TIMESTAMP(A.process_end_time) + interval '180' SECOND,'yyyy-MM-dd HH:mm:ss','yyyyMMddHHmmss') process_end_time--業務辦理結束時間
                from bdpdb.SDI_T_CUST_QUEUE A
                left join (  --去除一個櫃員多個窗口情況
                    SELECT A.date_dt,A.department BRANCH_ID ,B.win_no,A.counter_no 
                        FROM (
                            select date_dt    
                                ,department
                                ,counter_no
                            from bdpdb.SDI_BH_COUNTER
                            GROUP BY date_dt,department,counter_no
                            having count(win_no)=1 
                            ) A
                LEFT JOIN bdpdb.SDI_BH_COUNTER B
                ON A.date_dt=B.date_dt AND A.department=B.department
                AND A.counter_no=B.counter_no
                order BY 1,2,3,4
                ) B 
                on substr(A.process_time,1,4)||substr(A.process_time,6,2)||substr(A.process_time,9,2)=B.date_dt 
                and A.BRANCH_ID=B.BRANCH_ID and A.WIN_NO=B.WIN_NO
                where length(B.counter_no)=7
                order by 1,2,3,4,5,6 
) A  
right join (
            select distinct SYS_DATE,TEL_ID,TEL_TXID,TRAN_TIME
                from (    select
                        SYS_DATE --交易日期
                        ,TEL_ID --交易櫃員號
                        ,TEL_TXID --櫃員輸入原交易碼
                        ,SYS_DATE||SYS_TIME TRAN_TIME--交易時間
                        from bdpdb.SDI_F_CORE_BYFTJRN
                        where tx_ouno like '89120%' and substr(TEL_ID,1,3)='891'
                        AND SYS_DATE>='20161001' and SYS_DATE<='20161122'
                        order by 1,2,4,3)
)
B
on A.date_dt=B.sys_date and A.counter_no=B.tel_id 
and B.tran_time >= A.process_time 
and B.tran_time <= A.process_end_time
where A.date_dt is not NULL 
order by 1,2,3,5,7,6,4
)
END

 

CREATE DATABASE IF NOT EXISTS bdpdb;
USE bdpdb;
DROP TABLE IF NOT EXISTS bdpdb.idv_cst_smy_bk_tl_busi;
create table bdpdb.idv_cst_smy_bk_tl_busi(
    date_dt     string      comment  "日期"         ,
    branch_id   string      comment  "機構號"       ,
    counter_no  string      comment  "櫃員號"       ,
    tel_txid    string      comment  "交易源代碼"   ,
    start_time  varchar(50) comment  "交易起始時間" ,
    tran_time   varchar(50) comment  "交易結束時間"
)
CLUSTERED BY (date_dt) INTO 11 BUCKETS
STORED AS ORC
TBLPROPERTIES ("transactional"="true");

 

# /etc/profile

#配置java環境變量
export JAVA_HOME=/usr/local/jdk
export JAVA_BIN=$JAVA_HOME/bin
export JAVA_LIB=$JAVA_HOME/lib
export CLASSPATH=.:$JAVA_LIB/tools.jar:$JAVA_LIB/dt.jar

#配置HADOOP環境變量
export HADOOP_HOME=/usr/local/hadoop
export HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_HOME/lib/native
export HADOOP_OPTS="-Djava.library.path=$HADOOP_HOME/lib"
export HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_HOME/lib/native
export HADOOP_OPTS="-Djava.library.path=$HADOOP_HOME/lib"

#配置HIVE環境變量
export HIVE_HOME=/usr/local/hive

#配置zookeeper環境變量
export ZOOKEEPER_HOME=/usr/local/zookeeper

#配置hbase的環境變量
export HBASE_HOME=/usr/local/hbase

export PATH=.:$HBASE_HOME/bin:$ZOOKEEPER_HOME/bin:$HIVE_HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$JAVA_BIN:$PATH




#設置時區
TZ='Asia/Shanghai'
export TZ