bingmous

欢迎交流，不吝赐教~

导航

CDH6.2.0离线安装（详细）

目录

01 准备工作

02 环境配置

03 CDH安装

报错

01 准备工作

官网地址下载页面：https://www.cloudera.com/downloads/cdh.html，现在下载好像需要注册一个账号，下面的链接进入后需要登录，也可以自行在上面的下载页面找。

部署主要分为三部分：MySQL离线部署、CM离线部署、Parcel文件离线源部署

02 环境配置

## 时区
# 所有节点保存时区一致
timedatectl --help  # 查看帮助命令
timedatectl  list-timezones  # 列出时区
timedatectl  set-timezone    # 设置时区

## 时间
# 所有节点安装ntp
yum install -y ntp

# 选取一个节点为主节点
vi /etc/ntp.conf
server 127.127.1.0 iburst local clock  # 当外部时间不可用时，可使用本地硬件时间
restrict 192.168.33.0 mask 255.255.255.0 nomodify notrap #允许哪些网段的机器来同步时间

# 开启ntpd及查看状态
systemctl start ntpd
systemctl status ntpd

# 验证
ntpq -p

# 其他从节点停止禁用ntpd服务
systemctl stop ntpd
systemctl disable ntpd

# 其他节点同步时间
/usr/sbin/ntpdate 主节点

# 使用crontab每天同步时间
crontab -e
00 00 * * * /usr/sbin/ntpdate 主节点  # 每天0分0时执行命令
  • 节点安装JDK
## 创建目录，解压，修改所有者和组
mkdir /usr/java
tar -xvzf jdk-8u241-linux-x64.tar.gz -C /usr/java
chown -R root:root /usr/java/jdk1.8.0_241

## 配置环境变量，在/etc/profile.d/下创建env.sh
vi /etc/profile.d/env.sh
export JAVA_HOME=/usr/java/jdk1.8.0_241
export PATH=$JAVA_HOME/bin:PATH

source /etc/profile

## 验证
which java
java -version
## 先卸载自带的数据库
rpm -qa | grep mariadb  
rpm -e --nodeps mariadb-libs-5.5.64-1.el7.x86_64

## 查找并卸载老版本mysql
find / -name mysql|xargs rm -rf

## 安装
rpm -ivh mysql-community-common-5.7.31-1.el7.x86_64.rpm
rpm -ivh mysql-community-libs-5.7.31-1.el7.x86_64.rpm
rpm -ivh mysql-community-libs-compat-5.7.31-1.el7.x86_64.rpm
rpm -ivh mysql-community-client-5.7.31-1.el7.x86_64.rpm
rpm -ivh mysql-community-server-5.7.31-1.el7.x86_64.rpm

# 查看状态及启动
systemctl status mysqld 
systemctl start mysql
systemctl disable mysqld # 关闭开机自启动，不用的时候关掉，占资源

## 查看临时密码
grep 'temporary password' /var/log/mysqld.log
mysql -uroot -p临时密码

## 直接设置密码会出现
ERROR 1819 (HY000): Your password does not satisfy the current policy requirements
set global validate_password_policy=LOW;
set global validate_password_length=6;
set password=password('999999');
use mysql;
select host, user, authentication_string from user;
update user set host='%' where host="localhost";

## 登录
mysql -uroot -p999999

## 注意
# mysql5.6需要安装以下两个组件，否则无法安装agent
MySQL-shared-5.6.49-1.el7.x86_64.rpm
MySQL-shared-compat-5.6.49-1.el7.x86_64.rpm

# mysql5.7需要安装
mysql-community-libs-compat-5.7.31-1.el7.x86_64.rpm
  • 创建元数据库和用户
## 先创建cmf, amon
create database cmf default charset utf8 collate utf8_general_ci;
grant all on cmf.* to 'cmf'@'%' identified by '999999';

create database amon default charset utf8 collate utf8_general_ci;
grant all on amon.* to 'amonuser'@'%' identified by '999999';

flush privileges;

------------------------------------------
create database cmserver default charset utf8 collate utf8_general_ci;
grant all on cmserver.* to 'cmserveruser'@'%' identified by 'root';

create database metastore default charset utf8 collate utf8_general_ci;
grant all on metastore.* to 'hiveuser'@'%' identified by 'root';

create database amon default charset utf8 collate utf8_general_ci;
grant all on amon.* to 'amonuser'@'%' identified by '999999';

create database rman default charset utf8 collate utf8_general_ci;
grant all on rman.* to 'rmanuser'@'%' identified by 'root';

create database oozie default charset utf8 collate utf8_general_ci;
grant all on oozie.* to 'oozieuser'@'%' identified by 'root';

create database hue default charset utf8 collate utf8_general_ci;
grant all on hue.* to 'hueuser'@'%' identified by 'root';

flush privileges;
  • MySQL所在节点安装MySQL JDBC jar
## 创建/usr/share/java目录，将jdbc的jar包放进去
mkdir -p /usr/share/java  # cdh默认搜索目录
cp mysql-connector-java-5.1.48-bin.jar mysql-connector-java.jar  # 去掉版本号
  • 节点修改linux swappiness参数
## 设置为0，重启后又还原
echo 0 > /proc/sys/vm/swappiness

## 修改配置文件, 添加参数
vi /etc/sysctl.conf
vm.swappiness=0

## 上述在el6中有效，在el7中tuned服务会动态调整参数

## 查找tuned配置，直接修改，进入tuned目录
cd /usr/lib/tuned/
## 查找包含的所在文件路径
grep "vm.swappiness" * -R

## 逐个修改参数vm.swappiness=0
vi latency-performance/tuned.conf

## 修改后确认
grep "vm.swappiness" * -R

## sysctl修改内核参数 重启不生效 参考：
https://blog.csdn.net/ygtlovezf/article/details/79014299
  • 节点禁用透明页
## 立刻生效
echo never > /sys/kernel/mm/transparent_hugepage/defrag
echo never > /sys/kernel/mm/transparent_hugepage/enabled

## 永久生效 在/etc/rc.local末尾添加两行
vi /etc/rc.local
echo never > /sys/kernel/mm/transparent_hugepage/defrag
echo never > /sys/kernel/mm/transparent_hugepage/enabled

## 给与可执行权限
chmod +x /etc/rc.d/rc.local

03 CDH安装

  • 离线安装cm server及agent
## 先安装agent需要的依赖
yum install -y redhat-lsb httpd mod_ssl openssl-devel python-psycopg2 MySQL-python libpq.so


## 主节点安装daemons, server, agent
rpm -ivh cloudera-manager-daemons-6.2.0-968826.el7.x86_64.rpm  # 必要包
rpm -ivh cloudera-manager-server-6.2.0-968826.el7.x86_64.rpm  # server
rpm -ivh cloudera-manager-agent-6.2.0-968826.el7.x86_64.rpm  # agent

## 从节点安装daemons, agent
rpm -ivh cloudera-manager-daemons-6.2.0-968826.el7.x86_64.rpm
rpm -ivh cloudera-manager-agent-6.2.0-968826.el7.x86_64.rpm

## 从节点修改agent的配置，指向server的节点
vi /etc/cloudera-scm-agent/config.ini
server_host=localhost102

## 主节点修改server的配置，确定以下项与之前创建库时一致
vi /etc/cloudera-scm-server/db.properties
com.cloudera.cmf.db.type=mysql
com.cloudera.cmf.db.host=localhost102
com.cloudera.cmf.db.name=cmf
com.cloudera.cmf.db.user=cmf
com.cloudera.cmf.db.password=999999
com.cloudera.cmf.db.setupType=EXTERNAL
  • 部署离线parcel源
## 安装httpd服务(前面已经安装)，web查看http://localhost102/
systemctl status httpd  # 查看状态
systemctl start httpd  # 启动
systemctl enable httpd.service  #设置httpd服务开机自启

## 部署离线parcel源，将parcel相关的三个文件拷贝进去, .sha1将1去掉
## web查看http://localhost102/cdh6_parcel
mkdir -p /var/www/html/cdh6_parcel
mv CDH-6.2.0-1.cdh6.2.0.p0.967373-el7.parcel /var/www/html/cdh6_parcel
mv CDH-6.2.0-1.cdh6.2.0.p0.967373-el7.parcel.sha1 /var/www/html/cdh6_parcel
mv manifest.json /var/www/html/cdh6_parcel

#如果没下载sha1，可以生成
sha1sum CDH-6.2.0-1.cdh6.2.0.p0.967373-el7.parcel | awk '{ print $1 }' > CDH-6.2.0-1.cdh6.2.0.p0.967373-el7.parcel.sha
  • 启动server和agent
## 主节点启动server
systemctl start cloudera-scm-server

# 查看日志，出现7180说明启动成功,，有错误解决错误
tail -F /var/log/cloudera-scm-server/cloudera-scm-server.log

# 使用localhost102:7180可以进入web页面表示成功，接下来启动agent

## 从节点启动agent
systemctl start cloudera-scm-agent

## 停止
systemctl stop cloudera-scm-server
systemctl stop cloudera-scm-agent
systemctl disable cloudera-scm-server # 关闭开机启动
systemctl disable cloudera-scm-agent # 关闭开机启动

## 查看端口 7180
netstat -nltp

报错

  • Src file /opt/cloudera/parcels/.flood/CDH-6.2.0-1.cdh6.2.0.p0.967373-el7.parcel/CDH-6.2.0-1.cdh6.2.0.p0.967373-el7.parcel does not exist
    • 硬盘容量不足

posted on 2020-12-29 01:03  Bingmous  阅读(417)  评论(0)  编辑  收藏  举报