CDH6.2.0 搭建大数据集群
1. 资料准备
现在官网https://www.cloudera.com 需要注册账号,未来可能会收费等问题,十分麻烦,这里有一份我自己百度云的备份
链接: https://pan.baidu.com/s/1kh50Z2f9S09Ln8E8uOow8Q
提取码: b9jf
里面有一整套下载内容
比如: mysql 的包 和 mysql-connector-java-5.1.47.tar.gz
JDK1.8下载地址
https://www.oracle.com/java/technologies/javase/javase8-archive-downloads.html
jdk-8u241-linux-x64.tar.gz
准备 三台服务器:2核 16G内存(这里是在腾讯云申请的三台)
8G 内存也可以,但作为自己的实验环境会不大好用,仅仅是刚刚能把环境搭建起来而已
2. 环境配置
- 配置静态IP (腾讯云自带内网静态IP,不用搞了)
- 配置主机名映射(三台机器都需要配置)
添加内网IP 和对应的主机名称,三台都设置为腾讯云内网IP
vim /etc/hosts
10.206.16.14 hadoop1
10.206.16.2 hadoop2
10.206.16.8 hadoop3
- 关闭防火墙(三台机器都需要配置)
#停止firewall
systemctl stop firewalld.service
#禁止firewall开机启动
systemctl disable firewalld.service
#查看默认防火墙状态(关闭后显示notrunning,开启后显示running)
firewall-cmd --state
- 关闭SELINUX(三台都要配置)
vim /etc/selinux/config
修改
SELINUX=disabled
- 配置时钟同步(三台都要配置)
## 时区
# 所有节点保存时区一致 (由于腾讯云默认上海东八区,可以不搞)
timedatectl --help # 查看帮助命令
timedatectl list-timezones # 列出时区
timedatectl set-timezone # 设置时区
# 所有节点安装ntp
yum install -y ntp
# 所有节点停止禁用ntpd服务
systemctl stop ntpd
systemctl disable ntpd
# 每个节点使用crontab每天同步时间 从腾讯ntp服务器同步时间
crontab –e
#追加这一行定时任务
*/20 * * * * /usr/sbin/ntpdate ntpupdate.tencentyun.com > /dev/null &
- 配置免密登录(每个节点)
1. 创建用户hadoop
useradd hadoop
passwd hadoop ##设置其密码
visudo
## Allow root to run any commands anywhere
root ALL=(ALL) ALL
hadoop ALL=(ALL) ALL ## 自主添加的一行
修改完毕 :wq!
1.2 /opt 下创建目录
使用hadoop 登录
mkdir module
mkdir software
2. 在其hadoop和root用户目录下执行(都需要配置免密登录,或者仅配置root也可以)
ssh-keygen -t rsa
3. 三个节点执行完毕后,再每个节点执行(hadoop1 等为主机名)
ssh-copy-id hadoop1
ssh-copy-id hadoop2
ssh-copy-id hadoop3
- hadoop1 配置文件分发脚本 xsync
创建文件
vim /usr/local/bin/xsync
插入内容
#!/bin/bash
#1 获取输入参数个数,如果没有参数,直接退出
pcount=$#
if((pcount==0)); then
echo no args;
exit;
fi
#2 获取文件名称
p1=$1
fname=`basename $p1`
echo fname=$fname
#3 获取上级目录到绝对路径
pdir=`cd -P $(dirname $p1); pwd`
echo pdir=$pdir
#4 获取当前用户名称
user=`whoami`
#5 循环
for((host=1; host<4; host++)); do
echo ------------------- hadoop$host --------------
rsync -rvl $pdir/$fname $user@hadoop$host:$pdir
done
添加脚本权限
chmod 777 /usr/local/bin/xsync
- 安装jdk (三台机器)
上传jdk的压缩文件
解压缩到指定目录下
tar -zxvf jdk-8u141-linux-x64.tar.gz -C /opt/module/
配置配置文件
vim /etc/profile
##JAVA_HOME
export JAVA_HOME=/opt/module/jdk1.8.0_141
export PATH=$JAVA_HOME/bin:$PATH
source /etc/profile
## 验证
which java
java -version
- 上传 mysql-connector-java (三台机器)
通过rz 上传到 /opt/software
并mysql-connector-java-5.1.47-bin.jar去版本号 改名为 mysql-connector-java.jar
- 节点修改linux swappiness参数(三台机器)
## 设置为0,重启后又还原
echo 0 > /proc/sys/vm/swappiness
## 修改配置文件, 添加参数
vi /etc/sysctl.conf
vm.swappiness=0
## 上述在el6中有效,在el7中tuned服务会动态调整参数
## 查找tuned配置,直接修改,进入tuned目录
cd /usr/lib/tuned/
## 查找包含的所在文件路径
grep "vm.swappiness" * -R
## 逐个修改参数vm.swappiness=0
vi latency-performance/tuned.conf
## 修改后确认
grep "vm.swappiness" * -R
## sysctl修改内核参数 重启不生效 参考:
https://blog.csdn.net/ygtlovezf/article/details/79014299
- 节点禁用透明页
## 立刻生效
echo never > /sys/kernel/mm/transparent_hugepage/defrag
echo never > /sys/kernel/mm/transparent_hugepage/enabled
## 永久生效 在/etc/rc.local末尾添加两行
vi /etc/rc.local
echo never > /sys/kernel/mm/transparent_hugepage/defrag
echo never > /sys/kernel/mm/transparent_hugepage/enabled
## 给与可执行权限
chmod +x /etc/rc.d/rc.local
3. mysql 安装
任选一节点,比如 hadoop1 上安装mysql
## 先卸载自带的数据库
rpm -qa | grep mariadb
rpm -e --nodeps mariadb-libs-5.5.64-1.el7.x86_64
## 查找并卸载老版本mysql
find / -name mysql|xargs rm -rf
## 安装
rpm -ivh mysql-community-common-5.7.31-1.el7.x86_64.rpm
rpm -ivh mysql-community-libs-5.7.31-1.el7.x86_64.rpm
rpm -ivh mysql-community-libs-compat-5.7.31-1.el7.x86_64.rpm
rpm -ivh mysql-community-client-5.7.31-1.el7.x86_64.rpm
rpm -ivh mysql-community-server-5.7.31-1.el7.x86_64.rpm
如果报错 执行
yum -y install numactl
# 查看状态及启动
systemctl status mysqld
systemctl start mysql
systemctl disable mysqld # 关闭开机自启动,不用的时候关掉,占资源
## 查看临时密码
grep 'temporary password' /var/log/mysqld.log
mysql -uroot -p临时密码
## 直接设置密码会出现
ERROR 1819 (HY000): Your password does not satisfy the current policy requirements
set global validate_password_policy=LOW;
set global validate_password_length=6;
set password=password('999999');
use mysql;
select host, user, authentication_string from user;
update user set host='%' where host="localhost";
## 登录
mysql -uroot -p999999
## 注意
# mysql5.6需要安装以下两个组件,否则无法安装agent
MySQL-shared-5.6.49-1.el7.x86_64.rpm
MySQL-shared-compat-5.6.49-1.el7.x86_64.rpm
# mysql5.7需要安装
mysql-community-libs-compat-5.7.31-1.el7.x86_64.rpm
创建元数据库和用户
DROP DATABASE IF EXISTS cmf;
DROP DATABASE IF EXISTS amon;
DROP DATABASE IF EXISTS cmserver;
DROP DATABASE IF EXISTS metastore;
DROP DATABASE IF EXISTS amon;
DROP DATABASE IF EXISTS rman;
DROP DATABASE IF EXISTS oozie;
DROP DATABASE IF EXISTS hue;
DROP DATABASE IF EXISTS hive;
CREATE DATABASE scm DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON scm.* TO 'scm'@'%' IDENTIFIED BY 'scm';
CREATE DATABASE amon DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON amon.* TO 'amon'@'%' IDENTIFIED BY 'amon';
CREATE DATABASE rman DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON rman.* TO 'rman'@'%' IDENTIFIED BY 'rman';
CREATE DATABASE hue DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON hue.* TO 'hue'@'%' IDENTIFIED BY 'hue';
CREATE DATABASE metastore DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON metastore.* TO 'hive'@'%' IDENTIFIED BY 'hive';
CREATE DATABASE sentry DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON sentry.* TO 'sentry'@'%' IDENTIFIED BY 'sentry';
CREATE DATABASE nav DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON nav.* TO 'nav'@'%' IDENTIFIED BY 'nav';
CREATE DATABASE navms DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON navms.* TO 'navms'@'%' IDENTIFIED BY 'navms';
CREATE DATABASE oozie DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON oozie.* TO 'oozie'@'%' IDENTIFIED BY 'oozie';
CREATE DATABASE hive default charset utf8 collate utf8_general_ci;
GRANT ALL PRIVILEGES ON hive.* TO 'hive'@'%' IDENTIFIED BY '999999';
flush privileges;
4. CDH 安装
- 离线安装cm server及agent
## 先安装agent需要的依赖
yum install -y redhat-lsb httpd mod_ssl openssl-devel python-psycopg2 MySQL-python libpq.so
yum -y install chkconfig python bind-utils psmisc libxslt zlib sqlite cyrus-sasl-plain cyrus-sasl-gssapi fuse fuse-libs redhat-lsb portmap
## 主节点为hadoop1,其他节点为从节点
## 主节点安装daemons, server, agent
rpm -ivh cloudera-manager-daemons-6.2.0-968826.el7.x86_64.rpm # 必要包
rpm -ivh cloudera-manager-server-6.2.0-968826.el7.x86_64.rpm # server
rpm -ivh cloudera-manager-agent-6.2.0-968826.el7.x86_64.rpm # agent
## 从节点安装daemons, agent
rpm -ivh cloudera-manager-daemons-6.2.0-968826.el7.x86_64.rpm
rpm -ivh cloudera-manager-agent-6.2.0-968826.el7.x86_64.rpm
## 从节点修改agent的配置,指向server的节点
vi /etc/cloudera-scm-agent/config.ini
server_host=hadoop1
## 主节点修改server的配置,确定以下项与之前创建库时一致
vi /etc/cloudera-scm-server/db.properties
com.cloudera.cmf.db.type=mysql
com.cloudera.cmf.db.host=hadoop1
com.cloudera.cmf.db.name=cmf
com.cloudera.cmf.db.user=cmf
com.cloudera.cmf.db.password=999999
com.cloudera.cmf.db.setupType=EXTERNAL
- 部署离线parcel源
## hadoop2安装httpd服务(前面已经安装),web查看http://hadoop2/
systemctl status httpd # 查看状态
systemctl start httpd # 启动
systemctl enable httpd.service #设置httpd服务开机自启
## 部署离线parcel源,将parcel相关的三个文件拷贝进去, .sha1将1去掉
## web查看http://hadoop2/cdh6_parcel
mkdir -p /var/www/html/cdh6_parcel
mv CDH-6.2.0-1.cdh6.2.0.p0.967373-el7.parcel /var/www/html/cdh6_parcel
mv CDH-6.2.0-1.cdh6.2.0.p0.967373-el7.parcel.sha /var/www/html/cdh6_parcel
mv manifest.json /var/www/html/cdh6_parcel
#如果没下载sha1,可以生成
sha1sum CDH-6.2.0-1.cdh6.2.0.p0.967373-el7.parcel | awk '{ print $1 }' > CDH-6.2.0-1.cdh6.2.0.p0.967373-el7.parcel.sha
- 启动server和agent
## 主节点hadoop1启动server
systemctl start cloudera-scm-server
# 查看日志,出现7180说明启动成功,,有错误解决错误
tail -F /var/log/cloudera-scm-server/cloudera-scm-server.log
# 使用hadoop1外网ip:7180可以进入web页面表示成功,接下来启动agent
## 从节点启动agent
systemctl start cloudera-scm-agent
## 停止
systemctl stop cloudera-scm-server
systemctl stop cloudera-scm-agent
systemctl disable cloudera-scm-server # 关闭开机启动
systemctl disable cloudera-scm-agent # 关闭开机启动
## 查看端口 7180
netstat -nltp
-
web界面操作,登录http://hadoop1外网ip:7180,默认用户名密码为admin
八成进不去,需要在腾讯云配置安全组,打开7180端口才能访问(默认没有打开7180) -
web 操作
参考博文:
https://blog.csdn.net/weixin_37680513/article/details/111713387
https://zhuanlan.zhihu.com/p/254742441