set hive.merge.mapredfiles=true;
set hive.merge.mapfiles=true;
set mapred.max.split.size=5073741824;
set dfs.block.size=1073741824;
set mapreduce.map.memory.mb=8196;
set mapreduce.reduce.memory.mb=8196;
set hive.exec.compress.intermediate=true;
set hive.intermediate.compression.codec=org.apache.hadoop.io.compress.SnappyCodec;
set hive.exec.parallel=true;
set hive.exec.parallel.thread.number=8;
--压缩设置
set mapred.compress.map.output = true;
set mapred.output.compress = true;
set hive.exec.compress.output = true;
--输出设置
set hive.merge.mapfiles = true;
set hive.merge.mapredfiles = true;
set hive.merge.size.per.task = 256000000;
set hive.merge.smallfiles.avgsize = 128000000;
--输入设置
set hive.input.format = org.apache.hadoop.hive.ql.io.CombineHiveInputFormat;
set mapred.min.split.size = 256000000;
set mapred.min.split.size.per.node = 256000000;
set mapred.min.split.size.per.rack = 256000000;
set hive.exec.reducers.bytes.per.reducer = 256000000;