CentOS 7 安裝 CDH 5.16.2
1. 系統准備
1.1 鏡像模板
-
關閉 selinux
setenforce 0 # 須先禁用后面才能關閉防火牆 vi /etc/selinux/config SELINUX=disabled -
安裝 ntp
yum install -y ntp systemctl enable ntp -
禁用防火牆
systemctl diable firewalld -
安裝 JDK
# 刪除舊版 # 查詢已安裝jdk rpm -qa | grep jdk # 刪除舊版 rpm -e --no-deps package-name # 解壓至/usr/java/ mkdir /usr/java tar -zxvf cdh5.16.2/jdk-8u162-linux-x64.tar.gz -C /usr/java # 設置環境變量 vi /etc/profile.d/java.sh export JAVA_HOME=/usr/java/jdk1.8.0_162 export PATH=$PATH:$JAVA_HOME/bin
1.2 其他設置
-
ip
vi /etc/sysconfig/network-scripts/ifcfg-enp0s3 BOOTPROTO=static # 靜態ip ONBOOT=yes # 開機啟動 IPADDR=192.168.2.193 # ip 地址 PREFIX=24 # 子網掩碼 GATEWAY=192.168.2.1 # 網關 -
hostname
vi /etc/sysconfig/network HOSTNAME=cdh1.hnzst.com -
hosts
vi /etc/hosts 192.168.2.193 cdh1.hnzst.com cdh1 192.168.2.194 cdh2.hnzst.com cdh2 192.168.2.195 cdh3.hnzst.com cdh3 -
ssh （主節點）
# 生成 key ssh-keygen -t rsa # 拷貝到其他節點 ssh-copy-id cdh2 ssh-copy-id cdh3 -
ntp
-
主節點
vi /etc/ntp.conf #先注釋掉所有server # 192.168.2.0 和 255.255.255表示客戶端網段和子網掩碼 restrict 192.168.2.0 mask 255.255.255.0 nomodify notrap server 127.127.1.0 # 以本機作為服務器 fudge 127.127.1.0 stratum 10 systemctl start ntpd -
子節點
vi /etc/ntp.conf #先注釋掉所有server server cdh1.hnzst.com systemctl start ntpd
-
1.3 MySQL 安裝(主節點)
-
刪除舊版
rpm -qa | grep mysql rpm -e --no-deps packagename -
安裝依賴
mount /dev/cdrom /media/cdrom yum --disablerepo=\* --enablerepo=c7-media install -y libaio -
解壓
tar -zxvf cdh5.16.2/mysql-5.7.28-linux-glibc2.12-x86_64.tar.gz -C /usr/local/ -
軟鏈接
ln -s /usr/local/mysql-5.7.28-linux-glibc2.12-x86_64/ /usr/local/mysql -
創建用戶和組
groupadd mysql useradd -r -g mysql -s /bin/false mysql -
環境變量
vi /etc/profile.d/mysql.sh export PATH=$PATH:/usr/local/mysql/bin source /etc/profile.d/mysql.sh -
修改配置
vi /etc/my.conf[mysqld] # It is important that the datadir directory, which, by default, is /var/lib/mysql, # is on a partition that has sufficient free space. datadir=/var/lib/mysql socket=/var/lib/mysql/mysql.sock transaction-isolation = READ-COMMITTED # Disabling symbolic-links is recommended to prevent assorted security risks; # to do so, uncomment this line: symbolic-links = 0 key_buffer_size = 32M max_allowed_packet = 32M thread_stack = 256K thread_cache_size = 64 query_cache_limit = 8M query_cache_size = 64M query_cache_type = 1 max_connections = 550 #expire_logs_days = 10 #max_binlog_size = 100M #log_bin should be on a disk with enough free space. #Replace '/var/lib/mysql/mysql_binary_log' with an appropriate path for your #system and chown the specified folder to the mysql user. log_bin=/var/lib/mysql/mysql_binary_log #In later versions of MySQL, if you enable the binary log and do not set #a server_id, MySQL will not start. The server_id must be unique within #the replicating group. server_id=1 binlog_format = mixed read_buffer_size = 2M read_rnd_buffer_size = 16M sort_buffer_size = 8M join_buffer_size = 8M # InnoDB settings innodb_file_per_table = 1 innodb_flush_log_at_trx_commit = 2 innodb_log_buffer_size = 64M innodb_buffer_pool_size = 4G innodb_thread_concurrency = 8 innodb_flush_method = O_DIRECT innodb_log_file_size = 512M character-set-server=utf8mb4 collation-server=utf8mb4_general_ci [mysqld_safe] log-error=/var/log/mysqld.log pid-file=/var/run/mysqld/mysqld.pid sql_mode=STRICT_ALL_TABLES [mysql] default-character-set=utf8mb4 [client] socket=/var/lib/mysql/mysql.sock default-character-set=utf8mb4 -
數據目錄初始化
mysqld --initialize-insecure --user=mysql -
服務腳本
cp /usr/local/mysql/support-files/mysql.server /etc/init.d/mysql reboot -
設置開機自啟動
systemctl enable mysql -
啟動
systemctl start mysql -
修改root密碼
-
登錄 MySQL
mysql -u root --skip-password -
修改密碼
ALTER USER 'root'@'localhost' IDENTIFIED BY '123456';
-
-
設置root訪問權限
GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' IDENTIFIED BY '123456'; flush privileges; -
jdbc
mkdir /usr/share/java/ cp cdh5.16.2/mysql-connector-java-5.1.48.jar /usr/share/java/mysql-connector-java.jar
2. 集群安裝
2.1 安裝 CM
-
解壓文件
tar -zxvf cdh5.16.2/cloudera-manager-centos7-cm5.16.2_x86_64.tar.gz -C /opt/ -
添加用戶(所有節點)
useradd --system \ --home=/opt/cm-5.16.2/run/cloudera-scm-server \ --no-create-home \ --shell=/bin/false \ --comment "Cloudera SCM User" \ cloudera-scm -
創建 CM 服務本地數據存儲目錄
mkdir /var/lib/cloudera-scm-server chown cloudera-scm:cloudera-scm /var/lib/cloudera-scm-server -
配置 Agent
vi /opt/cm-5.16.2/etc/cloudera-scm-agent/config.ini server_host=cdh1.hnzst.com -
創建 Parcel 目錄
mkdir -p /opt/cloudera/parcel-repo/ chown cloudera-scm:cloudera-scm /opt/cloudera/parcel-repo/ mkdir -p /opt/cloudera/parcels chown cloudera-scm:cloudera-scm /opt/cloudera/parcels/ -
創建 CM 初始數據庫
/opt/cm-5.16.2/share/cmf/schema/scm_prepare_database.sh -uroot -p mysql scm scm scm -
同步 cm 到其他節點
scp -r /opt/cm-5.16.2 cdh2:/opt
2.2 安裝 CDH
-
安裝 perl
yum --disablerepo=\* --enablerepo=c7-media install -y perl -
拷貝 parcels
cp cdh5.16.2/CDH-5.16.2-1.cdh5.16.2.p0.8-el7.* cdh5.16.2/manifest.json /opt/cloudera/parcel-repo/ # xxx.sha1 須重命名為 xxx.sha mv /opt/cloudera/parcel-repo/CDH-5.16.2-1.cdh5.16.2.p0.8-el7.parcel.sha1 /opt/cloudera/parcel-repo/CDH-5.16.2-1.cdh5.16.2.p0.8-el7.parcel.sha -
啟動 server （主節點）
/opt/cm-5.16.2/etc/init.d/cloudera-scm-server start -
啟動 agents （所有節點）
/opt/cm-5.16.2/etc/init.d/cloudera-scm-agent start -
swappiness（所有節點）
sysctl vm.swappiness=0 echo "vm.swappiness=0" >> /etc/sysctl.conf -
最大透明（所有節點）
vi /etc/rc.d/rc.local echo never > /sys/kernel/mm/transparent_hugepage/enabled echo never > /sys/kernel/mm/transparent_hugepage/defrag chmod +x /etc/rc.d/rc.local -
服務化配置(可選)
-
主節點
cp /opt/cm-5.16.2/etc/init.d/cloudera-scm-server /etc/init.d/ vi /etc/init.d/cloudera-scm-server CMF_DEFAULTS=${CMF_DEFAULTS:-/opt/cm-5.16.2/etc/default} -
所有節點（包括主節點）
cp /opt/cm-5.16.2/etc/init.d/cloudera-scm-agent /etc/init.d/ vi /etc/init.d/cloudera-scm-agent CMF_DEFAULTS=${CMF_DEFAULTS:-/opt/cm-5.16.2/etc/default} systemctl daemon-reload -
server 啟動|停止|重啟
systemctl start|stop|restart cloudera-scm-server -
agent 啟動|停止|重啟
systemctl start|stop|restart cloudera-scm-agent
-