CentOS 7 安装 CDH 5.16.2
1. 系统准备
1.1 镜像模板
-
关闭 selinux
setenforce 0 # 须先禁用后面才能关闭防火墙 vi /etc/selinux/config SELINUX=disabled -
安装 ntp
yum install -y ntp systemctl enable ntp -
禁用防火墙
systemctl diable firewalld -
安装 JDK
# 删除旧版 # 查询已安装jdk rpm -qa | grep jdk # 删除旧版 rpm -e --no-deps package-name # 解压至/usr/java/ mkdir /usr/java tar -zxvf cdh5.16.2/jdk-8u162-linux-x64.tar.gz -C /usr/java # 设置环境变量 vi /etc/profile.d/java.sh export JAVA_HOME=/usr/java/jdk1.8.0_162 export PATH=$PATH:$JAVA_HOME/bin
1.2 其他设置
-
ip
vi /etc/sysconfig/network-scripts/ifcfg-enp0s3 BOOTPROTO=static # 静态ip ONBOOT=yes # 开机启动 IPADDR=192.168.2.193 # ip 地址 PREFIX=24 # 子网掩码 GATEWAY=192.168.2.1 # 网关 -
hostname
vi /etc/sysconfig/network HOSTNAME=cdh1.hnzst.com -
hosts
vi /etc/hosts 192.168.2.193 cdh1.hnzst.com cdh1 192.168.2.194 cdh2.hnzst.com cdh2 192.168.2.195 cdh3.hnzst.com cdh3 -
ssh （主节点）
# 生成 key ssh-keygen -t rsa # 拷贝到其他节点 ssh-copy-id cdh2 ssh-copy-id cdh3 -
ntp
-
主节点
vi /etc/ntp.conf #先注释掉所有server # 192.168.2.0 和 255.255.255表示客户端网段和子网掩码 restrict 192.168.2.0 mask 255.255.255.0 nomodify notrap server 127.127.1.0 # 以本机作为服务器 fudge 127.127.1.0 stratum 10 systemctl start ntpd -
子节点
vi /etc/ntp.conf #先注释掉所有server server cdh1.hnzst.com systemctl start ntpd
-
1.3 MySQL 安装(主节点)
-
删除旧版
rpm -qa | grep mysql rpm -e --no-deps packagename -
安装依赖
mount /dev/cdrom /media/cdrom yum --disablerepo=\* --enablerepo=c7-media install -y libaio -
解压
tar -zxvf cdh5.16.2/mysql-5.7.28-linux-glibc2.12-x86_64.tar.gz -C /usr/local/ -
软链接
ln -s /usr/local/mysql-5.7.28-linux-glibc2.12-x86_64/ /usr/local/mysql -
创建用户和组
groupadd mysql useradd -r -g mysql -s /bin/false mysql -
环境变量
vi /etc/profile.d/mysql.sh export PATH=$PATH:/usr/local/mysql/bin source /etc/profile.d/mysql.sh -
修改配置
vi /etc/my.conf[mysqld] # It is important that the datadir directory, which, by default, is /var/lib/mysql, # is on a partition that has sufficient free space. datadir=/var/lib/mysql socket=/var/lib/mysql/mysql.sock transaction-isolation = READ-COMMITTED # Disabling symbolic-links is recommended to prevent assorted security risks; # to do so, uncomment this line: symbolic-links = 0 key_buffer_size = 32M max_allowed_packet = 32M thread_stack = 256K thread_cache_size = 64 query_cache_limit = 8M query_cache_size = 64M query_cache_type = 1 max_connections = 550 #expire_logs_days = 10 #max_binlog_size = 100M #log_bin should be on a disk with enough free space. #Replace '/var/lib/mysql/mysql_binary_log' with an appropriate path for your #system and chown the specified folder to the mysql user. log_bin=/var/lib/mysql/mysql_binary_log #In later versions of MySQL, if you enable the binary log and do not set #a server_id, MySQL will not start. The server_id must be unique within #the replicating group. server_id=1 binlog_format = mixed read_buffer_size = 2M read_rnd_buffer_size = 16M sort_buffer_size = 8M join_buffer_size = 8M # InnoDB settings innodb_file_per_table = 1 innodb_flush_log_at_trx_commit = 2 innodb_log_buffer_size = 64M innodb_buffer_pool_size = 4G innodb_thread_concurrency = 8 innodb_flush_method = O_DIRECT innodb_log_file_size = 512M character-set-server=utf8mb4 collation-server=utf8mb4_general_ci [mysqld_safe] log-error=/var/log/mysqld.log pid-file=/var/run/mysqld/mysqld.pid sql_mode=STRICT_ALL_TABLES [mysql] default-character-set=utf8mb4 [client] socket=/var/lib/mysql/mysql.sock default-character-set=utf8mb4 -
数据目录初始化
mysqld --initialize-insecure --user=mysql -
服务脚本
cp /usr/local/mysql/support-files/mysql.server /etc/init.d/mysql reboot -
设置开机自启动
systemctl enable mysql -
启动
systemctl start mysql -
修改root密码
-
登录 MySQL
mysql -u root --skip-password -
修改密码
ALTER USER 'root'@'localhost' IDENTIFIED BY '123456';
-
-
设置root访问权限
GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' IDENTIFIED BY '123456'; flush privileges; -
jdbc
mkdir /usr/share/java/ cp cdh5.16.2/mysql-connector-java-5.1.48.jar /usr/share/java/mysql-connector-java.jar
2. 集群安装
2.1 安装 CM
-
解压文件
tar -zxvf cdh5.16.2/cloudera-manager-centos7-cm5.16.2_x86_64.tar.gz -C /opt/ -
添加用户(所有节点)
useradd --system \ --home=/opt/cm-5.16.2/run/cloudera-scm-server \ --no-create-home \ --shell=/bin/false \ --comment "Cloudera SCM User" \ cloudera-scm -
创建 CM 服务本地数据存储目录
mkdir /var/lib/cloudera-scm-server chown cloudera-scm:cloudera-scm /var/lib/cloudera-scm-server -
配置 Agent
vi /opt/cm-5.16.2/etc/cloudera-scm-agent/config.ini server_host=cdh1.hnzst.com -
创建 Parcel 目录
mkdir -p /opt/cloudera/parcel-repo/ chown cloudera-scm:cloudera-scm /opt/cloudera/parcel-repo/ mkdir -p /opt/cloudera/parcels chown cloudera-scm:cloudera-scm /opt/cloudera/parcels/ -
创建 CM 初始数据库
/opt/cm-5.16.2/share/cmf/schema/scm_prepare_database.sh -uroot -p mysql scm scm scm -
同步 cm 到其他节点
scp -r /opt/cm-5.16.2 cdh2:/opt
2.2 安装 CDH
-
安装 perl
yum --disablerepo=\* --enablerepo=c7-media install -y perl -
拷贝 parcels
cp cdh5.16.2/CDH-5.16.2-1.cdh5.16.2.p0.8-el7.* cdh5.16.2/manifest.json /opt/cloudera/parcel-repo/ # xxx.sha1 须重命名为 xxx.sha mv /opt/cloudera/parcel-repo/CDH-5.16.2-1.cdh5.16.2.p0.8-el7.parcel.sha1 /opt/cloudera/parcel-repo/CDH-5.16.2-1.cdh5.16.2.p0.8-el7.parcel.sha -
启动 server （主节点）
/opt/cm-5.16.2/etc/init.d/cloudera-scm-server start -
启动 agents （所有节点）
/opt/cm-5.16.2/etc/init.d/cloudera-scm-agent start -
swappiness（所有节点）
sysctl vm.swappiness=0 echo "vm.swappiness=0" >> /etc/sysctl.conf -
最大透明（所有节点）
vi /etc/rc.d/rc.local echo never > /sys/kernel/mm/transparent_hugepage/enabled echo never > /sys/kernel/mm/transparent_hugepage/defrag chmod +x /etc/rc.d/rc.local -
服务化配置(可选)
-
主节点
cp /opt/cm-5.16.2/etc/init.d/cloudera-scm-server /etc/init.d/ vi /etc/init.d/cloudera-scm-server CMF_DEFAULTS=${CMF_DEFAULTS:-/opt/cm-5.16.2/etc/default} -
所有节点（包括主节点）
cp /opt/cm-5.16.2/etc/init.d/cloudera-scm-agent /etc/init.d/ vi /etc/init.d/cloudera-scm-agent CMF_DEFAULTS=${CMF_DEFAULTS:-/opt/cm-5.16.2/etc/default} systemctl daemon-reload -
server 启动|停止|重启
systemctl start|stop|restart cloudera-scm-server -
agent 启动|停止|重启
systemctl start|stop|restart cloudera-scm-agent
-