CDH6.3.2搭建部署

CDH6.3.2

启动命令

启动mysql
[mysqladmin@node1 ~]$ service mysql start

#查看日志
cd /var/log/cloudera-scm-server
#启动 【启动后可以查看日志】 启动大概需要1分钟
[root@node1 ~]# service cloudera-scm-server start
[root@node1 cloudera-scm-server]# tail -F cloudera-scm-server.log
http://node1:7180/cmf/login

[root@node1 x86_64]# service cloudera-scm-agent start
[root@node2 CDH6.3.1]# service cloudera-scm-agent start
[root@node3 CDH6.3.1]# service cloudera-scm-agent start

[root@node1 x86_64]#service httpd start
http://node1/cdh6_parcel/

一、准备工作

1.离线部署主要分为三块：

a.Mysq离线部署

b.CM离线部署

c.Parcel文件离线部署

2.规划

3.下载源

支持的版本：requirements supported versions

• CM【这个下载地址已经过时了，不过我找到了安装包】

cm6.3.1-redhat7.tar.gz

• Parcel

• JDK 下载jdk-8u181-linux-x64.tar.gz

https://www.oracle.com/technetwork/java/javase/downloads/java-archive-javase8-2177648.html

• MySQL 下载mysql-5.7.26-el7-x86_64.tar.gz

https://dev.mysql.com/downloads/mysql/5.7.html#downloads

• MySQL jdbc jar

mysql-connector-java-5.1.47.jar

下载完成后要重命名去掉版本号:

mv mysql-connector-java-5.1.47.jar mysql-connector-java.jar

二、集群节点初始化

1.三台虚拟机

我的测试虚拟机是三台 2核8G

2.当前笔记本或台式机配置hosts⽂件

• MAC: /etc/hosts

• Window: C:\windows\system32\drivers\etc\hosts

需要把hosts文件复制出来，修改后再粘贴进去
#cdh
node1 192.168.88.100
node2 192.168.88.101
node3 192.168.88.102

3.设置所有节点的hosts⽂件

修改主机名：hostnamectl set-hostname node2

vim /etc/hosts
#cdh
192.168.88.100 node1
192.168.88.101 node2
192.168.88.102 node3

4.关闭所有节点的防⽕墙及清空规则

systemctl stop firewalld 
systemctl disable firewalld
iptables -F

5.关闭所有节点的selinux

vi /etc/selinux/config
将SELINUX=enforcing改为SELINUX=disabled 
设置后需要重启才能⽣效

6.设置所有节点的时区⼀致及时钟同步

[root@node1 ~]# date
2021年 09月 15日 星期三 14:25:28 CST
[root@node1 ~]# timedatectl
      Local time: 三 2021-09-15 14:25:40 CST
  Universal time: 三 2021-09-15 06:25:40 UTC
        RTC time: 三 2021-09-15 06:25:40
       Time zone: Asia/Shanghai (CST, +0800)
     NTP enabled: yes
NTP synchronized: yes
 RTC in local TZ: no
      DST active: n/a

#查看哪些时区
[root@node1 ~]#  timedatectl list-timezones
Africa/Abidjan
Africa/Accra
Africa/Addis_Ababa
Africa/Algiers
Africa/Asmara
Africa/Bamako
....
#所有节点设置亚洲上海时区
[root@node1 ~]# timedatectl set-timezone Asia/Shanghai
[root@node2 ~]# timedatectl set-timezone Asia/Shanghai
[root@node3 ~]# timedatectl set-timezone Asia/Shanghai

• 安装ntp服务

  • 三台机器都要执行

    yum -y install ntp
    

• 配置本地时钟同步

  • 三台机器都要执行

    vim /etc/sysconfig/ntpd
    #添加以下内容：
    OPTIONS="-g"
    SYNC_HWCLOCK=yes
    

• 第一台机器与外网同步【我第一台机器IP192.168.88.100】

  • 修改配置文件

    vim /etc/ntp.conf
    #删除或者注释所有restrict以及server开头的行
    
    #添加以下内容
    #允许哪个网段的机器跟我进行同步
    restrict 192.168.88.0 mask 255.255.255.0 nomodify notrap
    #aliyun的时钟服务器有问题，就使用第二个
    server ntp4.aliyun.com
    server 202.112.10.36
    server 127.127.1.0 # localclock
    fudge 127.127.1.0 stratum 10
    
    

• 启动nty服务

  systemctl start ntpd.service 
  systemctl enable ntpd.service
  

• 检查同步状态

  ntpq -p：列举当前所有可用的同步服务器
  ntpstat：查看当前的同步状态
  

• 配置另外两台机器

  • 修改配置文件

    vim /etc/ntp.conf
    #删除所有restrict以及server开头的行
    #添加以下内容：
    server 192.168.88.100
    

  • 启动两台机器的ntp服务

    systemctl start ntpd.service
    systemctl enable ntpd.service
    

  • 手动同步

    ntpdate -u 192.168.88.100
    

  • 配置定时任务

    crontab -e
    */5 * * * * /usr/sbin/ntpdate -u 192.168.88.100 > /dev/null 2>&1
    

7.每个节点部署jdk

mkdir /usr/java
[root@node1 ~]# tar -xvf jdk-8u181-linux-x64.tar.gz -C /usr/java/
#切记必须修正所属⽤户及⽤户组
chown -R root:root /usr/java/jdk1.8.0_181
echo "export JAVA_HOME=/usr/java/jdk1.8.0_181" >> /etc/profile
echo "export PATH=/usr/java/jdk1.8.0_181/bin:${PATH}" >> /etc/profile
source /etc/profile
which java

8.node1节点离线部署Mysql5.7

1.解压及创建目录 
[root@node1 ~]# tar -xvf mysql-5.7.11-linux-glibc2.5-x86_64.tar.gz -C /usr/local/
[root@node1 local]# mv mysql-5.7.11-linux-glibc2.5-x86_64 mysql
[root@node1 local]# mkdir mysql/arch mysql/data mysql/tmp

2.创建my.cnf(见文件)
[root@node1 local]# vi /etc/my.cnf
[client]
port            = 3306
socket          = /usr/local/mysql/data/mysql.sock
default-character-set=utf8mb4

[mysqld]
port            = 3306
socket          = /usr/local/mysql/data/mysql.sock

skip-slave-start

skip-external-locking
key_buffer_size = 256M
sort_buffer_size = 2M
read_buffer_size = 2M
read_rnd_buffer_size = 4M
query_cache_size= 32M
max_allowed_packet = 16M
myisam_sort_buffer_size=128M
tmp_table_size=32M

table_open_cache = 512
thread_cache_size = 8
wait_timeout = 86400
interactive_timeout = 86400
max_connections = 600

# Try number of CPU's*2 for thread_concurrency
#thread_concurrency = 32 

#isolation level and default engine 
default-storage-engine = INNODB
transaction-isolation = READ-COMMITTED

server-id  = 1739
basedir     = /usr/local/mysql
datadir     = /usr/local/mysql/data
pid-file     = /usr/local/mysql/data/hostname.pid

#open performance schema
log-warnings
sysdate-is-now

binlog_format = ROW
log_bin_trust_function_creators=1
log-error  = /usr/local/mysql/data/hostname.err
log-bin = /usr/local/mysql/arch/mysql-bin
expire_logs_days = 7

innodb_write_io_threads=16

relay-log  = /usr/local/mysql/relay_log/relay-log
relay-log-index = /usr/local/mysql/relay_log/relay-log.index
relay_log_info_file= /usr/local/mysql/relay_log/relay-log.info

log_slave_updates=1
gtid_mode=OFF
enforce_gtid_consistency=OFF

# slave
slave-parallel-type=LOGICAL_CLOCK
slave-parallel-workers=4
master_info_repository=TABLE
relay_log_info_repository=TABLE
relay_log_recovery=ON

#other logs
#general_log =1
#general_log_file  = /usr/local/mysql/data/general_log.err
#slow_query_log=1
#slow_query_log_file=/usr/local/mysql/data/slow_log.err

#for replication slave
sync_binlog = 500


#for innodb options 
innodb_data_home_dir = /usr/local/mysql/data/
innodb_data_file_path = ibdata1:1G;ibdata2:1G:autoextend

innodb_log_group_home_dir = /usr/local/mysql/arch
innodb_log_files_in_group = 4
innodb_log_file_size = 1G
innodb_log_buffer_size = 200M

#根据生产需要，调整pool size 
innodb_buffer_pool_size = 2G
#innodb_additional_mem_pool_size = 50M #deprecated in 5.6
tmpdir = /usr/local/mysql/tmp

innodb_lock_wait_timeout = 1000
#innodb_thread_concurrency = 0
innodb_flush_log_at_trx_commit = 2

innodb_locks_unsafe_for_binlog=1

#innodb io features: add for mysql5.5.8
performance_schema
innodb_read_io_threads=4
innodb-write-io-threads=4
innodb-io-capacity=200
#purge threads change default(0) to 1 for purge
innodb_purge_threads=1
innodb_use_native_aio=on

#case-sensitive file names and separate tablespace
innodb_file_per_table = 1
lower_case_table_names=1

[mysqldump]
quick
max_allowed_packet = 128M

[mysql]
no-auto-rehash
default-character-set=utf8mb4

[mysqlhotcopy]
interactive-timeout

[myisamchk]
key_buffer_size = 256M
sort_buffer_size = 256M
read_buffer = 2M
write_buffer = 2M


3.创建用户组及用户
[root@node1 local]# groupadd -g 101 dba
[root@node1 local]# useradd -u 514 -g dba -G root -d /usr/local/mysql mysqladmin
[root@node1 local]# id mysqladmin
uid=514(mysqladmin) gid=101(dba) 组=101(dba),0(root)

4.copy 环境变量配置文件至mysqladmin用户的home目录中,为了以下步骤配置个人环境变量
[root@node1 local]# cp /etc/skel/.* /usr/local/mysql  ###important


5.配置环境变量
[root@node1 local]# vi mysql/.bashrc

export MYSQL_BASE=/usr/local/mysql
export PATH=${MYSQL_BASE}/bin:$PATH


6.赋权限和用户组，切换用户mysqladmin，安装
[root@node1 local]# chown  mysqladmin:dba /etc/my.cnf 
[root@node1 local]# chmod  640 /etc/my.cnf 


[root@node1 local]# chown -R mysqladmin:dba /usr/local/mysql
[root@node1 local]# chmod -R 755 /usr/local/mysql


7.配置服务及开机自启动
[root@node1 local]# cd /usr/local/mysql
#将服务文件拷贝到init.d下，并重命名为mysql
[root@node1 mysql]# cp support-files/mysql.server /etc/rc.d/init.d/mysql 
#赋予可执行权限
[root@node1 mysql]# chmod +x /etc/rc.d/init.d/mysql
#删除服务
[root@node1 mysql]# chkconfig --del mysql
#添加服务
[root@node1 mysql]# chkconfig --add mysql


8.安装libaio及安装mysql的初始db
[root@node1 mysql]# yum -y install libaio
[root@node1 mysql]# sudo su - mysqladmin
#执行
bin/mysqld \
--defaults-file=/etc/my.cnf \
--user=mysqladmin \
--basedir=/usr/local/mysql/ \
--datadir=/usr/local/mysql/data/ \
--initialize

在初始化时如果加上 –initial-insecure，则会创建空密码的 root@localhost 账号，否则会创建带密码的 root@localhost 账号，密码直接写在 log-error 日志文件中
（在5.6版本中是放在 ~/.mysql_secret 文件里，更加隐蔽，不熟悉的话可能会无所适从）

9.查看临时密码
[mysqladmin@node1 ~]$ cd /usr/local/mysql/data
[mysqladmin@node1 data]$ cat hostname.err |grep password
2021-09-15T16:02:11.119910Z 1 [Note] A temporary password is generated for root@localhost: l03U1yFwRi;G


10.启动
[mysqladmin@node1 ~]$ service mysql start
Starting MySQL. SUCCESS! 

11.登录及修改用户密码
[mysqladmin@node1 ~]$ mysql -uroot -p'l03U1yFwRi;G'

mysql> alter user root@localhost identified by '123456';
Query OK, 0 rows affected (0.05 sec)

mysql> GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' IDENTIFIED BY '123456' ;
Query OK, 0 rows affected, 1 warning (0.02 sec)


mysql> flush privileges;
Query OK, 0 rows affected (0.00 sec)

mysql> exit;
Bye

12.重启
[mysqladmin@node1 ~]$ service mysql restart
Shutting down MySQL.. SUCCESS! 
Starting MySQL. SUCCESS

[mysqladmin@node1 ~]$ mysql -uroot -p123456

9.创建CDH的元数据库和⽤户、amon服务的数据库及⽤户

create database cmf DEFAULT CHARACTER SET utf8;
create database amon DEFAULT CHARACTER SET utf8;
grant all on cmf.* TO 'cmf'@'%' IDENTIFIED BY 'www.wangshuangying.com';
grant all on amon.* TO 'amon'@'%' IDENTIFIED BY 'www.wangshuangying.com';
flush privileges;

10.node1节点部署mysql jdbc jar

mkdir -p /usr/share/java/
重命名不能带版本号
[root@node1 ~]# cp mysql-connector-java-5.1.47.jar /usr/share/java/mysql-connector-java.jar

三、离线部署cm、server、agent

1.离线部署cm server及agent

1.1.所有节点创建⽬录及解压

mkdir /opt/cloudera-manager
tar -xzvf cm6.3.1-redhat7.tar.gz -C /opt/cloudera-manager/

1.2.选择node1为cm server，不下载依赖包直接部署

cd /opt/cloudera-manager/cm6.3.1/RPMS/x86_64
rpm -ivh cloudera-manager-daemons-6.3.1-1466458.el7.x86_64.rpm --nodeps --force
rpm -ivh cloudera-manager-server-6.3.1-1466458.el7.x86_64.rpm --nodeps --force

[root@node1 x86_64]# cd /etc/cloudera-scm-server/
[root@node1 cloudera-scm-server]# vim db.properties
#添加
com.cloudera.cmf.db.type=mysql
com.cloudera.cmf.db.host=node1:3306
com.cloudera.cmf.db.name=cmf
com.cloudera.cmf.db.user=cmf
com.cloudera.cmf.db.password=www.wangshuangying.com
#修改
com.cloudera.cmf.db.setupType=EXTERNAL

#查看日志
cd /var/log/cloudera-scm-server
#启动 【启动后可以查看日志】
[root@node1 ~]# service cloudera-scm-server start
[root@node1 cloudera-scm-server]# tail -F cloudera-scm-server.log

打开7180web界面不要动

http://node1:7180/cmf/login

1.3.所有节点为cm agent，不下载依赖包直接部署分发

cd /opt/cloudera-manager/cm6.3.1/RPMS/x86_64
[root@node1 x86_64]# scp  cloudera-manager-agent-6.3.1-1466458.el7.x86_64.rpm node2:/root/CDH6.3.1
[root@node1 x86_64]# scp  cloudera-manager-agent-6.3.1-1466458.el7.x86_64.rpm node3:/root/CDH6.3.1

[root@node1 x86_64]# scp cloudera-manager-daemons-6.3.1-1466458.el7.x86_64.rpm node2:/root/CDH6.3.1
[root@node1 x86_64]# scp cloudera-manager-daemons-6.3.1-1466458.el7.x86_64.rpm node3:/root/CDH6.3.1

所有节点为cm agent

cd /opt/cloudera-manager/cm6.3.1/RPMS/x86_64
rpm -ivh cloudera-manager-daemons-6.3.1-1466458.el7.x86_64.rpm --nodeps --force
rpm -ivh cloudera-manager-agent-6.3.1-1466458.el7.x86_64.rpm --nodeps --force

#所有节点
[root@node1 x86_64]# vim /etc/cloudera-scm-agent/config.ini
server_host=node1

• 启动

[root@node1 x86_64]# service cloudera-scm-agent start
[root@node2 CDH6.3.1]# service cloudera-scm-agent start
[root@node3 CDH6.3.1]# service cloudera-scm-agent start

账号密码均为： admin

页面操作

四、部署离线parcel源

2.1.安装httpd服务

yum install -y httpd

2.2.部署离线parcel源

[root@node1 cloudera-scm-server]#  mkdir -p /var/www/html/cdh6_parcel

[root@node1 ~]# mv CDH-6.3.2-1.cdh6.3.2.p0.1605554-el7.parcel /var/www/html/cdh6_parcel
#需要先去掉 sha1 的1
[root@node1 ~]# mv CDH-6.3.2-1.cdh6.3.2.p0.1605554-el7.parcel.sha1 CDH-6.3.2-1.cdh6.3.2.p0.1605554-el7.parcel.sha
[root@node1 ~]# mv CDH-6.3.2-1.cdh6.3.2.p0.1605554-el7.parcel.sha /var/www/html/cdh6_parcel
[root@node1 ~]# mv manifest.json  /var/www/html/cdh6_parcel

[root@node1 ~]# ll /var/www/html/cdh6_parcel
总用量 2033428
-rw-r--r--. 1 root root 2082186246 9月  15 12:28 CDH-6.3.2-1.cdh6.3.2.p0.1605554-el7.parcel
-rw-r--r--. 1 root root         40 9月  15 12:28 CDH-6.3.2-1.cdh6.3.2.p0.1605554-el7.parcel.sha
-rw-r--r--. 1 root root      33887 9月  15 12:28 manifest.json

[root@node1 ~]# service httpd start

http://node1/cdh6_parcel/

2.3三台机器

echo never > /sys/kernel/mm/transparent_hugepage/defrag
echo never > /sys/kernel/mm/transparent_hugepage/enabled

__EOF__

本文作者：王双颖
本文链接：https://www.cnblogs.com/ChloeAilian/p/15513450.html
关于博主：评论和私信会在第一时间回复。或者直接私信我。
版权声明：本博客所有文章除特别声明外，均采用 BY-NC-SA 许可协议。转载请注明出处！
声援博主：如果您觉得文章对您有帮助，可以点击文章右下角【推荐】一下。您的鼓励是博主的最大动力！