數據倉庫 業務數倉 ODS層

ODS層數據不做任何處理，完全仿照業務數據庫中的表字段，一模一樣的創建ODS層對應表。

8張表建表語句：

①用sqoop把導入到HDFS的時候，加了參數--fields-terminated-by "\t"，因此這里ODS層建表的時候也注意相同的分隔符。

②不管是全量導入還是其他形式，都使用分區表，每天保存一次數據。

drop table if exists ods_order_info;
create external table ods_order_info (
    `id` string COMMENT '訂單編號',
    `total_amount` decimal(10,2) COMMENT '訂單金額',
    `order_status` string COMMENT '訂單狀態',
    `user_id` string COMMENT '用戶id',
    `payment_way` string COMMENT '支付方式',
    `out_trade_no` string COMMENT '支付流水號',
    `create_time` string COMMENT '創建時間',
    `operate_time` string COMMENT '操作時間'
) COMMENT '訂單表'
PARTITIONED BY (`dt` string)
row format delimited fields terminated by '\t'
location '/warehouse/gmall/ods/ods_order_info/'
;

drop table if exists ods_order_detail;
create external table ods_order_detail( 
    `id` string COMMENT '訂單編號',
    `order_id` string  COMMENT '訂單號', 
    `user_id` string COMMENT '用戶id',
    `sku_id` string COMMENT '商品id',
    `sku_name` string COMMENT '商品名稱',
    `order_price` string COMMENT '商品價格',
    `sku_num` string COMMENT '商品數量',
    `create_time` string COMMENT '創建時間'
) COMMENT '訂單明細表'
PARTITIONED BY (`dt` string)
row format delimited fields terminated by '\t' 
location '/warehouse/gmall/ods/ods_order_detail/'
;

drop table if exists ods_sku_info;
create external table ods_sku_info( 
    `id` string COMMENT 'skuId',
    `spu_id` string   COMMENT 'spuid', 
    `price` decimal(10,2) COMMENT '價格',
    `sku_name` string COMMENT '商品名稱',
    `sku_desc` string COMMENT '商品描述',
    `weight` string COMMENT '重量',
    `tm_id` string COMMENT '品牌id',
    `category3_id` string COMMENT '品類id',
    `create_time` string COMMENT '創建時間'
) COMMENT '商品表'
PARTITIONED BY (`dt` string)
row format delimited fields terminated by '\t'
location '/warehouse/gmall/ods/ods_sku_info/'
;

drop table if exists ods_user_info;
create external table ods_user_info( 
    `id` string COMMENT '用戶id',
    `name`  string COMMENT '姓名',
    `birthday` string COMMENT '生日',
    `gender` string COMMENT '性別',
    `email` string COMMENT '郵箱',
    `user_level` string COMMENT '用戶等級',
    `create_time` string COMMENT '創建時間'
) COMMENT '用戶信息'
PARTITIONED BY (`dt` string)
row format delimited fields terminated by '\t'
location '/warehouse/gmall/ods/ods_user_info/'
;

drop table if exists ods_base_category1;
create external table ods_base_category1( 
    `id` string COMMENT 'id',
    `name`  string COMMENT '名稱'
) COMMENT '商品一級分類'
PARTITIONED BY (`dt` string)
row format delimited fields terminated by '\t'
location '/warehouse/gmall/ods/ods_base_category1/'
;

drop table if exists ods_base_category2;
create external table ods_base_category2( 
    `id` string COMMENT ' id',
    `name` string COMMENT '名稱',
    category1_id string COMMENT '一級品類id'
) COMMENT '商品二級分類'
PARTITIONED BY (`dt` string)
row format delimited fields terminated by '\t'
location '/warehouse/gmall/ods/ods_base_category2/'
;

drop table if exists ods_base_category3;
create external table ods_base_category3(
    `id` string COMMENT ' id',
    `name`  string COMMENT '名稱',
    category2_id string COMMENT '二級品類id'
) COMMENT '商品三級分類'
PARTITIONED BY (`dt` string)
row format delimited fields terminated by '\t'
location '/warehouse/gmall/ods/ods_base_category3/'
;

drop table if exists ods_payment_info;
create external table ods_payment_info(
    `id`   bigint COMMENT '編號',
    `out_trade_no`    string COMMENT '對外業務編號',
    `order_id`        string COMMENT '訂單編號',
    `user_id`         string COMMENT '用戶編號',
    `alipay_trade_no` string COMMENT '支付寶交易流水編號',
    `total_amount`    decimal(16,2) COMMENT '支付金額',
    `subject`         string COMMENT '交易內容',
    `payment_type`    string COMMENT '支付類型',
    `payment_time`    string COMMENT '支付時間'
   )  COMMENT '支付流水表'
PARTITIONED BY (`dt` string)
row format delimited fields terminated by '\t'
location '/warehouse/gmall/ods/ods_payment_info/'
;

數據導入腳本：

#!/bin/bash

   APP=gmall
   hive=/opt/module/hive/bin/hive

# 如果是輸入的日期按照取輸入日期；如果沒輸入日期取當前時間的前一天
if [ -n "$1" ] ;then
    do_date=$1
else 
    do_date=`date -d "-1 day" +%F`
fi

sql=" 
load data inpath '/origin_data/$APP/db/order_info/$do_date' OVERWRITE into table "$APP".ods_order_info partition(dt='$do_date');

load data inpath '/origin_data/$APP/db/order_detail/$do_date' OVERWRITE into table "$APP".ods_order_detail partition(dt='$do_date');

load data inpath '/origin_data/$APP/db/sku_info/$do_date' OVERWRITE into table "$APP".ods_sku_info partition(dt='$do_date');

load data inpath '/origin_data/$APP/db/user_info/$do_date' OVERWRITE into table "$APP".ods_user_info partition(dt='$do_date');

load data inpath '/origin_data/$APP/db/payment_info/$do_date' OVERWRITE into table "$APP".ods_payment_info partition(dt='$do_date');

load data inpath '/origin_data/$APP/db/base_category1/$do_date' OVERWRITE into table "$APP".ods_base_category1 partition(dt='$do_date');

load data inpath '/origin_data/$APP/db/base_category2/$do_date' OVERWRITE into table "$APP".ods_base_category2 partition(dt='$do_date');

load data inpath '/origin_data/$APP/db/base_category3/$do_date' OVERWRITE into table "$APP".ods_base_category3 partition(dt='$do_date'); 
"
$hive -e "$sql"