公告

昵称： a120608yby
园龄： 7年
粉丝： 4
关注： 10

+加关注

2025年3月

日

一

二

三

四

五

六

随笔分类

随笔档案

阅读排行榜

推荐排行榜

CentOS 7 部署 CDH 6.3.2

一、服务器准备

　　至少4台服务器（CentOS 7.9 最小版本安装）

服务器名称	配置	IP地址	备注
cdh-110-101	8核/16G内存/300G SSD硬盘	192.168.110.101	CDH MGR/Agent
cdh-110-102	8核/16G内存/300G SSD硬盘	192.168.110.102	Agent
cdh-110-103	8核/16G内存/300G SSD硬盘	192.168.110.103	Agent
cdh-110-104	8核/16G内存/300G SSD硬盘	192.168.110.104	Agent

二、服务器初始化

　　1、配置服务器之间免密登录

ssh-keygen -q -N ''
ssh-copy-id 192.168.110.101
ssh-copy-id 192.168.110.102
ssh-copy-id 192.168.110.103
ssh-copy-id 192.168.110.104
#不知道root密码的情况
复制当前主机/root/.ssh/id_rsa.pub的内容到其他主机的authorized_keys文件

　　2、主机名配置

hostnamectl set-hostname vm-110-101-centos    #其他主机参照设置，注意主机名称
# vim /etc/hosts
192.168.110.101 vm-110-101-centos
192.168.110.102 vm-110-102-centos
192.168.110.103 vm-110-103-centos
192.168.110.104 vm-110-104-centos

　　3、优化

参考：https://docs.cloudera.com/documentation/enterprise/latest/topics/cdh_admin_performance.html
 
# vim /etc/rc.local
echo never > /sys/kernel/mm/transparent_hugepage/enabled
echo never > /sys/kernel/mm/transparent_hugepage/defrag
# vim /etc/sysctl.conf
vm.swappiness = 10

三、数据库准备

　　1、部署数据库

　　参考：https://docs.cloudera.com/documentation/enterprise/latest/topics/cm_ig_mysql.html#cmig_topic_5_5

　　离线安装包地址：https://mirrors.ustc.edu.cn/mysql-ftp/Downloads/MySQL-5.7/mysql-5.7.38-1.el7.x86_64.rpm-bundle.tar

　　离线安装参考：https://www.cnblogs.com/a120608yby/p/9888437.html

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

设置repo源
yum -y install http://mirrors.ustc.edu.cn/mysql-repo/mysql57-community-release-el7.rpm
----------------------------------------
安装MySQL
yum install -y mysql-community-server
----------------------------------------
配置MySQL
if [ ! "$(cat /usr/bin/mysqld_pre_systemd | grep -v ^\# | grep initialize-insecure )" ]; then
    sed -i "s@--initialize @--initialize-insecure @g" /usr/bin/mysqld_pre_systemd
fi
----------------------------------------
配置文件修改
# vim /etc/my.cnf
[client]
port = 3306
socket = /var/lib/mysql/mysql.sock
default-character-set = utf8mb4
 
[mysql]
prompt="MySQL [\d]> "
no-auto-rehash
 
[mysqld]
port = 3306
socket = /var/lib/mysql/mysql.sock
 
datadir = /var/lib/mysql/
pid-file = /var/run/mysqld/mysqld.pid
user = mysql
bind-address = 0.0.0.0
server-id = 1
 
init-connect = 'SET NAMES utf8mb4'
character-set-server = utf8mb4
 
skip-name-resolve
#skip-networking
back_log = 300
 
max_connections = 3783
max_connect_errors = 6000
open_files_limit = 65535
table_open_cache = 1024
max_allowed_packet = 500M
binlog_cache_size = 1M
max_heap_table_size = 8M
tmp_table_size = 128M
 
read_buffer_size = 2M
read_rnd_buffer_size = 8M
sort_buffer_size = 8M
join_buffer_size = 8M
key_buffer_size = 256M
 
thread_cache_size = 64
 
query_cache_type = 1
query_cache_size = 64M
query_cache_limit = 2M
 
ft_min_word_len = 4
 
log_bin = mysql-bin
binlog_format = row
expire_logs_days = 7
 
log_error = /var/log/mysqld.log
slow_query_log = 1
long_query_time = 1
slow_query_log_file = /var/lib/mysql/slow.log
 
performance_schema = 0
explicit_defaults_for_timestamp
 
lower_case_table_names = 1
 
skip-external-locking
 
default_storage_engine = InnoDB
#default-storage-engine = MyISAM
innodb_file_per_table = 1
innodb_open_files = 500
innodb_buffer_pool_size = 1024M
innodb_write_io_threads = 4
innodb_read_io_threads = 4
innodb_thread_concurrency = 0
innodb_purge_threads = 1
innodb_flush_log_at_trx_commit = 2
innodb_log_buffer_size = 2M
innodb_log_file_size = 32M
innodb_log_files_in_group = 3
innodb_max_dirty_pages_pct = 90
innodb_lock_wait_timeout = 120
 
bulk_insert_buffer_size = 8M
myisam_sort_buffer_size = 64M
myisam_max_sort_file_size = 10G
myisam_repair_threads = 1
 
interactive_timeout = 28800
wait_timeout = 28800
 
binlog_rows_query_log_events = 1
log_bin_trust_function_creators = 1
log_timestamps = SYSTEM
 
sql_mode = NO_AUTO_VALUE_ON_ZERO
 
[mysqldump]
quick
max_allowed_packet = 500M
 
[myisamchk]
key_buffer_size = 256M
sort_buffer_size = 8M
read_buffer = 4M
write_buffer = 4M
----------------------------------------
启动MySQL
systemctl enable mysqld --now
----------------------------------------
配置root密码
> update mysql.user set authentication_string=password('Demo2@22-#') where User='root';
> flush privileges;
 
或者可以通过自动的安全配置工具设置
# mysql_secure_installation

　　2、创建数据库及用户和权限

CREATE DATABASE scm DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON scm.* TO 'scm'@'%' IDENTIFIED BY 'Demo2@22-#';
CREATE DATABASE amon DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON amon.* TO 'amon'@'%' IDENTIFIED BY 'Demo2@22-#';
CREATE DATABASE rman DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON rman.* TO 'rman'@'%' IDENTIFIED BY 'Demo2@22-#';
CREATE DATABASE hue DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON hue.* TO 'hue'@'%' IDENTIFIED BY 'Demo2@22-#';
CREATE DATABASE metastore DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON metastore.* TO 'hive'@'%' IDENTIFIED BY 'Demo2@22-#';
CREATE DATABASE sentry DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON sentry.* TO 'sentry'@'%' IDENTIFIED BY 'Demo2@22-#';
CREATE DATABASE nav DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON nav.* TO 'nav'@'%' IDENTIFIED BY 'Demo2@22-#';
CREATE DATABASE navms DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON navms.* TO 'navms'@'%' IDENTIFIED BY 'Demo2@22-#';
CREATE DATABASE oozie DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON oozie.* TO 'oozie'@'%' IDENTIFIED BY 'Demo2@22-#';
 
GRANT ALL ON scm.* TO 'scm'@'192.168.110.%' IDENTIFIED BY 'Demo2@22-#';
GRANT ALL ON amon.* TO 'amon'@'192.168.110.%' IDENTIFIED BY 'Demo2@22-#';
GRANT ALL ON rman.* TO 'rman'@'192.168.110.%' IDENTIFIED BY 'Demo2@22-#';
GRANT ALL ON hue.* TO 'hue'@'192.168.110.%' IDENTIFIED BY 'Demo2@22-#';
GRANT ALL ON metastore.* TO 'hive'@'192.168.110.%' IDENTIFIED BY 'Demo2@22-#';
GRANT ALL ON sentry.* TO 'sentry'@'192.168.110.%' IDENTIFIED BY 'Demo2@22-#';
GRANT ALL ON nav.* TO 'nav'@'192.168.110.%' IDENTIFIED BY 'Demo2@22-#';
GRANT ALL ON navms.* TO 'navms'@'192.168.110.%' IDENTIFIED BY 'Demo2@22-#';
GRANT ALL ON oozie.* TO 'oozie'@'192.168.110.%' IDENTIFIED BY 'Demo2@22-#';

四、准备CDH程序

　　1、安装httpd

1 2	`yum -y` `install` `httpd createrepo` `rm` `-f` `/etc/httpd/conf.d/welcome.conf`

　　2、修改/etc/httpd/conf/httpd.conf

vim /etc/httpd/conf/httpd.conf
ServerName localhost:80    #95行
AllowOverride All    #151行
DirectoryIndex index.html index.cgi index.php    #164行
AddType application/x-gzip .gz .tgz .parcel    #284行
ServerTokens Prod    #最后添加

　　3、启动服务

1	`systemctl` `enable` `httpd --now`

　　4、上传CDH压缩包至/var/www/html目录并解压

1 2	`tar` `xf cdh.tar.gz -C` `/var/www/html` `chown` `-R apache.apache` `/var/www/html`

五、安装Cloudera Manager

　　1、配置安装源

# vim /etc/yum.repos.d/cloudera-repo.repo 
[cloudera-repo]
name=cloudera-repo
baseurl=http://192.168.110.150/cloudera-repos/
enabled=1
gpgcheck=0

　　2、安装jdk及Cloudera Manager

yum localinstall /usr/local/src/jdk-8u181-linux-x64.rpm -y    #所有主机都需要安装
yum install cloudera-manager-server -y
 
tar xf /usr/local/src/parcel-repo.tar.gz -C /opt/cloudera/
tar xf /usr/local/src/csd.tar.gz -C /opt/cloudera/

　　3、初始化并启动服务

tar xf /usr/local/src/java.tar.gz -C /usr/share/    #所有主机
/opt/cloudera/cm/schema/scm_prepare_database.sh mysql -h 192.168.110.101 scm scm
输入scm数据库用户的密码"Demo2@22-#"
 
systemctl enable cloudera-scm-server --now
tail -f /var/log/cloudera-scm-server/cloudera-scm-server.log

六、服务集群部署

　　1、控制台

http://192.168.110.101:7180/
默认账号：admin
默认密码：admin

　　2、控制界面操作

1 2	`参考：https://blog.csdn.net/qq_40180229/article/details/108756561` `https://blog.51cto.com/wzlinux/2529226`

七、问题处理

　　1、kafka需要配置hosts问题

　　分别修改三台实例主机的配置，然后重启kafka即可

listeners=PLAINTEXT://0.0.0.0:9092
advertised.listeners=PLAINTEXT://192.168.110.102:9092
 
listeners=PLAINTEXT://0.0.0.0:9092
advertised.listeners=PLAINTEXT://192.168.110.103:9092
 
listeners=PLAINTEXT://0.0.0.0:9092
advertised.listeners=PLAINTEXT://192.168.110.104:9092