安装cdh6.2.0

CDH至少3台服务器部署集群，规划如下：
ip1 node01.hadoop.com Hadoop Master, CM Server, dataNode, Mysql
ip2 node02.hadoop.com dataNode, CM Agent
ip3 node03.hadoop.com dataNode, CM Agent

一 修改主机名,更改Hosts映射文件,配置server到其他主机互信
1.1 修改主机名
hostnamectl set-hostname node01.hadoop.com
hostname node01.hadoop.com
hostnamectl set-hostname node02.hadoop.com
hostname node02.hadoop.com
hostnamectl set-hostname node03.hadoop.com
hostname node03.hadoop.com

1.2 配置主机名与ip地址的映射
cat <<EOF >> /etc/hosts
ip1 node01.hadoop.com node01
ip2 node02.hadoop.com node02
ip3 node03.hadoop.com node03
EOF

1.3 配置互信
ssh-keygen
for host in node01 node02 node03
do
ssh-copy-id ${host}.hadoop.com -o StrictHostKeyChecking=no
done

二 关闭防火墙，停掉selinux
systemctl disable firewalld
systemctl stop firewalld
setenforce 0
sed -i 's/SELINUX=enforcing/SELINUX=disabled' /etc/sysconfig/SELINUX

三 时间同步

3.1 安装ntp
yum install ntp -y

3.2 立即同步时间
ntpdate ntp1.aliyun.com

3.3 设置同步时间池
sed -i 's/server 0.centos.pool.ntp.org iburst/server ntp1.aliyun.com/g' /etc/ntp.conf
sed -i 's/server 1.centos.pool.ntp.org iburst/server ntp2.aliyun.com/g' /etc/ntp.conf
sed -i 's/server 2.centos.pool.ntp.org iburst/server ntp3.aliyun.com/g' /etc/ntp.conf
sed -i 's/server 3.centos.pool.ntp.org iburst/server ntp4.aliyun.com/g' /etc/ntp.conf

3.4 开机自启
systemctl enable ntpd

3.5 启动ntp服务
systemctl start ntpd

四 设置主机参数

4.1 修改虚拟内存交换页大小（机器-all）
// 临时修改
sysctl -w vm.swappiness=0
// 永久修改
echo "vm.swappiness=0" > /etc/sysctl.conf
sysctl -p

4.2 禁用透明页（kernel优化）（机器-all）
echo never > /sys/kernel/mm/transparent_hugepage/defrag
echo never > /sys/kernel/mm/transparent_hugepage/enabled
# 开机自启
echo "echo never > /sys/kernel/mm/transparent_hugepage/enabled" >> /etc/rc.local
echo "echo never > /sys/kernel/mm/transparent_hugepage/defrag" >> /etc/rc.local
chmod u+x /etc/rc.local

4.3 修改linux系统打开文件最大数量的限制
cat <<EOF >> /etc/security/limits.conf
* soft nproc 11000
* hard nproc 11000
* soft nofile 65535
* hard nofile 65535
EOF
cat <<EOF >> /etc/security/limits.conf
* soft nproc 11000
* hard nproc 11000
EOF

五 安装java,jdbc驱动(所有主机)
mkdir -pv /data/apps
tar -xf jdk-8u221-linux-x64.tar.gz -C /data/apps
cat <<EOF > /etc/profile.d/jdk.sh
export JAVA_HOME=/data/apps/jdk1.8.0_221
export PATH=:\$JAVA_HOME/bin:$PATH
EOF
source /etc/profile

wget http://central.maven.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar
mkdir -pv /usr/share/java # 必须是此目录
mv mysql-connector-java-8.0.16.jar /usr/share/java/mysql-connector-java.jar
scp /usr/share/java/mysql-connector-java.jar node02:/usr/share/java/
scp /usr/share/java/mysql-connector-java.jar node03:/usr/share/java/

#useradd --system --home=/var/lib/cloudera-scm-server --no-create-home --shell=/sbin/nologin --comment "Cloudera Manager" cloudera-scm

六 安装配置mysql

6.1 检查和卸载自带mysql、mariadb
rpm -qa | grep mysql
#rpm -e --nodeps mysql
rpm -qa | grep mysql |xargs rpm -e --nodeps
rpm -qa | grep mariadb
rpm -e --nodeps mariadb

6.2 安装配置mysql
# wget http://repo.mysql.com/mysql-community-release-el7-5.noarch.rpm
# rpm -ivh mysql-community-release-el7-5.noarch.rpm
# yum -y install mysql-server
# systemctl start mysqld
# systemctl enable mysqld

6.3 配置
cat <<EOF > /etc/my.cnf
[mysqld]
basedir = /data/apps/mysql
datadir = /data/apps/mysql/data
character_set_server = utf8
skip_name_resolve = on
innodb_buffer_pool_size =8G
innodb_write_io_threads = 8
innodb_read_io_threads= 8
innodb_log_file_size = 128M
innodb_log_buffer_size = 32M
innodb_page_cleaners = 4
innodb_io_capacity = 5000
innodb_flush_log_at_trx_commit = 2
innodb_flush_method = O_DIRECT
max_allowed_packet = 32M
max_connections = 4000
key_buffer_size = 32M
read_buffer_size = 2M
read_rnd_buffer_size = 16M
sort_buffer_size = 8M
join_buffer_size = 8M
thread_stack = 256K
thread_cache_size = 64
query_cache_limit = 8M
query_cache_size = 64M
query_cache_type = 1
server_id = 180
log_bin = mysql-bin
binlog_format = row
relay_log = relay-log
#gtid_mode = ON
#enforce_gtid_consistency = true
gtid_mode = OFF
enforce_gtid_consistency = OFF
master_info_repository = TABLE
relay_log_info_repository = TABLE
slave_parallel_type = LOGICAL_CLOCK
slave_parallel_workers = 8
relay_log_recovery = ON
slow_query_log = on
long_query_time = 3
slow_query_log_file = mysql-slow.log
expire_logs_days = 40
log_error = mysql.log
log_bin_trust_function_creators = on
sql_mode = "STRICT_TRANS_TABLES,NO_ENGINE_SUBSTITUTION"
lower_case_table_names = 1
log_timestamps = SYSTEM
EOF

mv /var/lib/mysql/ib_logfile0 /var/lib/mysql/ib_logfile1 /var/lib/mysql/back

6.4 防止mysql默认账号被人乱用，执行以下命令并一直回车
/usr/bin/mysql_secure_installation

6.5 配置CM及其他服务的数据库基于mysql, 增加mysql-connector包
#CREATE DATABASE scm DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE amon DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE rman DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE hue DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE metastore DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE sentry DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE nav DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE navms DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE oozie DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE hive DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;

#GRANT ALL ON scm.* TO 'scm'@'%' IDENTIFIED BY 'password';
GRANT ALL ON amon.* TO 'amon'@'%' IDENTIFIED BY 'password';
GRANT ALL ON rman.* TO 'rman'@'%' IDENTIFIED BY 'password';
GRANT ALL ON hue.* TO 'hue'@'%' IDENTIFIED BY 'password';
GRANT ALL ON metastore.* TO 'metastore'@'%' IDENTIFIED BY 'password';
GRANT ALL ON sentry.* TO 'sentry'@'%' IDENTIFIED BY 'password';
GRANT ALL ON nav.* TO 'nav'@'%' IDENTIFIED BY 'password';
GRANT ALL ON navms.* TO 'navms'@'%' IDENTIFIED BY 'password';
GRANT ALL ON oozie.* TO 'oozie'@'%' IDENTIFIED BY 'password';
GRANT ALL ON hive.* TO 'hive'@'%' IDENTIFIED BY 'password';

create user 'cdh'@'112.198.9.11' identified by 'password';
grant all on *.* to 'cdh'@'%';
FLUSH PRIVILEGES;
SHOW DATABASES;
SHOW GRANTS FOR '<user>'@'%';

 

七 准备CM yum源
yum -y install httpd
systemctl start httpd
systemctl enable httpd
mkdir -pv /var/www/html/cloudera-repos

wget --recursive --no-parent --no-host-directories https://archive.cloudera.com/cm6/6.2.0/redhat7/ -P /var/www/html/cloudera-repos
wget https://archive.cloudera.com/cm6/6.2.0/allkeys.asc -P /var/www/html/cloudera-repos/cm6/6.2.0/
chmod -R ugo+rX /var/www/html/cloudera-repos/cm6

#https://archive.cloudera.com/cm6/6.2.0/redhat7/yum/RPMS/x86_64/cloudera-manager-agent-6.2.0-968826.el7.x86_64.rpm
#https://archive.cloudera.com/cm6/6.2.0/redhat7/yum/RPMS/x86_64/cloudera-manager-daemons-6.2.0-968826.el7.x86_64.rpm
#https://archive.cloudera.com/cm6/6.2.0/redhat7/yum/RPMS/x86_64/cloudera-manager-server-6.2.0-968826.el7.x86_64.rpm
#https://archive.cloudera.com/cm6/6.2.0/redhat7/yum/RPMS/x86_64/cloudera-manager-server-db-2-6.2.0-968826.el7.x86_64.rpm
#https://archive.cloudera.com/cm6/6.2.0/redhat7/yum/RPMS/x86_64/enterprise-debuginfo-6.2.0-968826.el7.x86_64.rpm
#https://archive.cloudera.com/cm6/6.2.0/redhat7/yum/RPMS/x86_64/oracle-j2sdk1.8-1.8.0+update181-1.x86_64.rpm

yum -y install screen
screen -S simpeServer
cd /var/www/html
python -m SimpleHTTPServer 8900

cat <<EOF > /etc/yun.repos.d/cloudera-repo.repo
[cloudera-repo]
name=cloudera-repo
baseurl=http://ip:8900/cloudera-repos/cm6/6.2.0/redhat7/yum
enabled=1
gpgcheck=0
EOF

八 安装cdh

8.1 安装cloudera manager server
# master
yum -y install cloudera-manager-daemons cloudera-manager-agent cloudera-manager-server

# slave
yum -y install cloudera-manager-daemons cloudera-manager-agent

8.2 设置 Cloudera Manager 数据库
#/opt/cloudera/cm/schema/scm_prepare_database.sh mysql db_cdh6_scm -hnode01 -uroot -p't' --scm-host node01 db_cdh6_scm scm scm
/opt/cloudera/cm/schema/scm_prepare_database.sh mysql -hnode01 -uroot -p'' --scm-host node01 scm root ''
# 前一个是数据库主机 后一个是cm主机

8.3 配置
sed -i 's/server_host=/server_host=node01.hadoop.com' /etc/cloudera-scm-agent/config.ini
scp /etc/cloudera-scm-agent/config.ini node02:/etc/cloudera-scm-agent/
scp /etc/cloudera-scm-agent/config.ini node03:/etc/cloudera-scm-agent/

8.3 启动

# master 上
systemctl start cloudera-scm-server
systemctl enable cloudera-scm-server
# 所有主机
systemctl start cloudera-scm-agent
systemctl enable cloudera-scm-agent

# 列出重复的包
package-cleanup --dupes
# 删除重复的包
package-cleanup --cleandupes --removenewestdupes

 

namenode用来部署mysql,hdfs.namenode,Hbase.master,Yarn,ResourceManager
datenode用来做worker节点，承担zk,kafka等其他服务的部署

 

# 安装StreamSets3.9 网址：https://archives.streamsets.com/index.html

wget -P /var/www/html/streamsets3.9.0/ https://archives.streamsets.com/datacollector/3.9.0/parcel/manifest.json
wget -P /var/www/html/streamsets3.9.0/ https://archives.streamsets.com/datacollector/3.9.0/parcel/STREAMSETS_DATACOLLECTOR-3.9.0-el7.parcel.sha
wget -P /var/www/html/streamsets3.9.0/ https://archives.streamsets.com/datacollector/3.9.0/parcel/STREAMSETS_DATACOLLECTOR-3.9.0-el7.parcel

# 配置csd 网址: https://streamsets.com/opensource
wget -P /opt/cloudera/csd/ https://archives.streamsets.com/datacollector/3.9.0/csd/STREAMSETS-3.9.0.jar
cd /opt/cloudera/csd/
chown cloudera-scm:cloudera-scm STREAMSETS-3.9.0.jar && chmod 644 STREAMSETS-3.9.0.jar
systemctl restart cloudera-scm-server