一、 HAWQ2.3.0環境准備【全部主機節點】:
1, vim /etc/sysctl.conf,編輯如下內容:
kernel.shmmax= 1000000000 kernel.shmmni= 4096 kernel.shmall= 4000000000 kernel.sem= 250 512000 100 2048 kernel.sysrq= 1 kernel.core_uses_pid= 1 kernel.msgmnb= 65536 kernel.msgmax= 65536 kernel.msgmni= 2048 net.ipv4.tcp_syncookies= 0 net.ipv4.ip_forward= 0 net.ipv4.conf.default.accept_source_route= 0 net.ipv4.tcp_tw_recycle= 1 net.ipv4.tcp_max_syn_backlog= 200000 net.ipv4.conf.all.arp_filter= 1 net.ipv4.ip_local_port_range= 1281 65535 net.core.netdev_max_backlog= 200000 vm.overcommit_memory= 2 fs.nr_open= 3000000 kernel.threads-max= 798720 kernel.pid_max= 798720 #increase network net.core.rmem_max=2097152 net.core.wmem_max=2097152
保存退出后,使用命令“sysctl -p” 使之生效:
sysctl -p
2, vim /etc/security/limits.conf 編輯如下內容:
* soft nofile 2900000 * hard nofile 2900000 * soft nproc 131072 * hard nproc 131072
重新登錄以使配置生效,或者使用命令臨時設置一下:
ulimit -n 10240
3, 添加gpadmin用戶(greeplum admin縮寫),使用/opt/gpadmin作為主目錄
useradd --home=/opt/gpadmin/ --no-create-home --comment "HAWQ admin" gpadmin echo gpadmin | passwd --stdin gpadmin mkdir /opt/gpadmin chown gpadmin:gpadmin /opt/gpadmin
2,添加gpadmin到/etc/sudoers,編輯該文件,添加如下行
gpadmin ALL=(ALL) NOPASSWD:ALL
3, 配置gpadmin用戶的ssh【免密互訪】
4, 准備軟件包依賴項:
libgsasl is needed by apache-hawq-2.3.0.0-el7.x86_64
protobuf >= 2.5.0 is needed by apache-hawq-2.3.0.0-el7.x86_64
net-snmp-libs is needed by apache-hawq-2.3.0.0-el7.x86_64
thrift >= 0.9.1 is needed by apache-hawq-2.3.0.0-el7.x86_64
boost >= 1.53.0 is needed by apache-hawq-2.3.0.0-el7.x86_64
其中gcc gcc-c++ protobuf net-snmp-libs boots可直接安裝。而安裝libgsasl需要添加一個包含libgsasl庫的源:新建/etc/yum.repos.d/fedora.repo文件,添加如下內容:
[epel-repo] name=epel baseurl=http://dl.fedoraproject.org/pub/epel/6/x86_64/ enabled=1 gpgcheck=0
執行安裝:
yum install gcc gcc-c++ -y yum install protobuf -y yum install net-snmp-libs -y yum install boost -y yum install libgsasl-devel -y
C)下載安裝thrift 0.9.1
wget http://dl.fedoraproject.org/pub/epel/7/x86_64/Packages/t/thrift-0.9.1-15.el7.x86_64.rpm rpm -ivh thrift-0.9.1-15.el7.x86_64.rpm
二、 下載安裝包apache-hawq-rpm-2.3.0.0-incubating.tar.gz,並解壓縮
cd downloads wget http://mirrors.tuna.tsinghua.edu.cn/apache/incubator/hawq/2.3.0.0-incubating/apache-hawq-rpm-2.3.0.0-incubating.tar.gz tar -zxvf apache-hawq-rpm-2.3.0.0-incubating.tar.gz
三、 安裝服務端
1, 安裝
rpm -ivh --prefix=/opt/gpadmin apache-hawq-2.3.0.0-el7.x86_64.rpm
#在其他主機中依次執行上面安裝
**移除安裝
yum remove apache-hawq -y
2 HAWQ的參數配置
0) 切換用戶gpadmin
[root@ep-bd01 apache-hawq]# su - gpadmin
編輯用戶gpadmin啟動文件.bash_proifile
source /opt/gpadmin/apache-hawq/greenplum_path.sh
執行命令
source /opt/gpadmin/apache-hawq/greenplum_path.sh
配置HAWQ的集群及其它參數
1) vi /opt/gpadmin/apache-hawq/etc/slaves
# 添加集群主機列表
ep-bd01 ep-bd02 ep-bd03 ep-bd04 ep-bd05 ep-bd06
2) vi hawq-site.xml
# 修改如下參數
參數名 值
hadoop.security.authentication simple hawq_master_address_host ep-bd01 hawq_dfs_url ep-bd01:8020/hawq hawq_rm_memory_limit_perseg 64GB hawq_rm_yarn_address ep-bd01:8032 hawq_dfs_url ep-bd01:8020/hawq hawq_global_rm_type yarn hawq_rm_yarn_scheduler_address ep-bd01:8030 hawq_acl_type standalone hawq_master_directory /opt/gpadmin/data/masterdd hawq_segment_directory /opt/gpadmin/data/segmentdd
vi yarn-site.xml
# 修改如下參數
參數名 值
hadoop.security.authentication simple
2, 建立dfs/hawq
sudo -u hdfs hdfs dfs -mkdir /hawq sudo -u hdfs hdfs dfs -chown gpadmin:gpadmin /hawq
3, 建立data目錄(全部節點)
sudo -u gpadmin mkdir -p /opt/gpadmin/data/masterdd /opt/gpadmin/data/segmentdd
4, 復制配置文件到其他節點
su - gpadmin gpscp -f slaves slaves =:/opt/gpadmin/apache-hawq/etc
gpscp -f slaves hawq-site.xml =:/opt/gpadmin/apache-hawq/etc
gpscp -f slaves hdfs-client.xml =:/opt/gpadmin/apache-hawq/etc
gpscp -f slaves yarn-client.xml =:/opt/gpadmin/apache-hawq/etc
5,初始化cluster【注意,這里必須進入/opt/gpadmin/apache-hawq/bin】
su - gpadmin cd /opt/gpadmin/apache-hawq/bin source ../greenplum_path.sh ./hawq init cluster
# 初始化cluster失敗,調整之后,再次執行初始化之前需要
#刪除dfs目錄中內容:
sudo -u hdfs hdfs dfs -rm -r -f /hawq/*
#刪除數據目錄中內容
ssh ep-bd01 rm -rf /opt/gpadmin/data/masterdd/* ssh ep-bd01 rm -rf /opt/gpadmin/data/segmentdd/* ssh ep-bd02 rm -rf /opt/gpadmin/data/masterdd/* ssh ep-bd02 rm -rf /opt/gpadmin/data/segmentdd/* ssh ep-bd03 rm -rf /opt/gpadmin/data/masterdd/* ssh ep-bd03 rm -rf /opt/gpadmin/data/segmentdd/* ssh ep-bd04 rm -rf /opt/gpadmin/data/masterdd/* ssh ep-bd04 rm -rf /opt/gpadmin/data/segmentdd/* ssh ep-bd05 rm -rf /opt/gpadmin/data/masterdd/* ssh ep-bd05 rm -rf /opt/gpadmin/data/segmentdd/* ssh ep-bd06 rm -rf /opt/gpadmin/data/masterdd/* ssh ep-bd06 rm -rf /opt/gpadmin/data/segmentdd/*
# 啟動停止重啟hawq
hawq stop/restart/start cluster