CentOS7下的CDH 6.2.0 安装过程

#install OS centos 7.5
#install lsb package

#install epel, or you'll receive error msg when you install hue for  "Requires: libtidy"

#install net-tools package
yum install -y redhat-lsb  net-tools

#如果在国内建议安装阿里云的epel：

wget -O /etc/yum.repos.d/epel-7.repo http://mirrors.aliyun.com/repo/epel-7.repo
yum clean all && yum makecache

 

#Set the hostname to a unique name (not localhost).

sudo hostnamectl set-hostname cdh-1.test.hugedata.com.cn
sudo hostnamectl set-hostname cdh-2.test.hugedata.com.cn
sudo hostnamectl set-hostname cdh-3.test.hugedata.com.cn
sudo hostnamectl set-hostname steamsets.test.hugedata.com.cn


#Edit /etc/hosts with the IP address and fully qualified domain name (FQDN) of each host in the cluster.
192.168.88.35 cdh-1.test.hugedata.com.cn cdh-1
192.168.88.36 cdh-2.test.hugedata.com.cn cdh-2
192.168.88.37 cdh-3.test.hugedata.com.cn cdh-3
192.168.88.38 steamsets.test.hugedata.com.cn steamsets


#Edit /etc/sysconfig/network with the FQDN of this host only:

HOSTNAME=cdh-1.test.hugedata.com.cn

HOSTNAME=cdh-2.test.hugedata.com.cn

HOSTNAME=cdh-3.test.hugedata.com.cn

HOSTNAME=steamsets.test.hugedata.com.cn


#stop and disable firewalld service
systemctl stop firewalld
systemctl disable firewalld


#disable selinux
sed -i 's/SELINUX=enforcing/SELINUX=disabled/'  /etc/selinux/config
setenforce 0

#优化配置

sysctl vm.swappiness=10

echo 'vm.swappiness=10' >> /etc/sysctl.conf

echo never > /sys/kernel/mm/transparent_hugepage/defrag

echo never > /sys/kernel/mm/transparent_hugepage/enabled

echo 'echo never > /sys/kernel/mm/transparent_hugepage/defrag'  >> /etc/rc.local

echo 'echo never > /sys/kernel/mm/transparent_hugepage/enabled'  >> /etc/rc.local

chmod +x /etc/rc.d/rc.local

#set ntp
yum install -y ntp

#Edit the /etc/ntp.conf file  master

# 同意内网其它机器同步时间，下面这句要改成你的子网的网络号和子网掩码

restrict 192.168.1.0 mask 255.255.255.0 nomodify notrap

server 210.72.145.44 perfer   # 中国国家受时中心

server 202.112.10.36             # 1.cn.pool.ntp.org

server 59.124.196.83             # 0.asia.pool.ntp.org

server  127.127.1.0     # local clock

fudge   127.127.1.0 stratum 10

# allow update time by the upper server 

# 同意上层时间server主动改动本机时间

restrict 210.72.145.44 nomodify notrap noquery

restrict 202.112.10.36 nomodify notrap noquery

restrict 59.124.196.83 nomodify notrap noquery

#other server

vim /etc/ntp.conf
#改成上面配置的ntp的server
server 192.168.88.35
fudge 192.168.88.35 stratum 10


#在ntp的server上check
ntpdate -u 120.25.108.11

sudo systemctl start ntpd
sudo systemctl enable ntpd
hwclock --systohc

# all server
systemctl stop chronyd.service
systemctl disable chronyd.service

#Installing Cloudera Manager, CDH, and Managed Services on manager-server
#enable httpd
sudo yum -y install httpd
sudo systemctl start httpd

#Downloading and Publishing the Package Repository
sudo mkdir -p /var/www/html/cloudera-repos
sudo wget --recursive --no-parent --no-host-directories https://archive.cloudera.com/cm6/6.2.0/redhat7/ -P /var/www/html/cloudera-repos
sudo chmod -R ugo+rX /var/www/html/cloudera-repos/cm6

#download allkeys.asc
cd /var/www/html/cloudera-repos/cm6/6.2.0
wget https://archive.cloudera.com/cm6/6.2.0/allkeys.asc

# CDH 6
sudo mkdir -p /var/www/html/cloudera-repos
sudo wget --recursive --no-parent --no-host-directories https://archive.cloudera.com/cdh6/6.2.0/redhat7/ -P /var/www/html/cloudera-repos
sudo wget --recursive --no-parent --no-host-directories https://archive.cloudera.com/gplextras6/6.2.0/redhat7/ -P /var/www/html/cloudera-repos
sudo chmod -R ugo+rX /var/www/html/cloudera-repos/cdh6
sudo chmod -R ugo+rX /var/www/html/cloudera-repos/gplextras6

#install java sdk
#uninstall open jdk1
rpm -qa | grep jdk | xargs rpm -e --nodeps
#install jdk
rpm -ivh /var/www/html/cloudera-repos/cm6/6.2.0/redhat7/yum/RPMS/x86_64/oracle-j2sdk1.8-1.8.0+update181-1.x86_64.rpm
#update java_home
vim /etc/profile
JAVA_HOME=/usr/java/jdk1.8.0_181-cloudera
CLASSPATH=$JAVA_HOME/lib/
PATH=$PATH:$JAVA_HOME/bin
export PATH JAVA_HOME CLASSPATH

#refresh profile
source /etc/profile

#install postgresql-server（此步不做）
#yum install -y postgresql-server.x86_64
#初始化postgresql数据库 （此步不做）
#sudo service postgresql initdb


#yum 源
#create repo
cd /var/www/html/cloudera-repos/
createrepo .
#所有机器上配置源
vim /etc/yum.repos.d/cloudera-repo.repo

[cloudera-repo]
name=cloudera-repo
baseurl=http://192.168.88.35/cloudera-repos/
enabled=1
gpgcheck=0

#创建缓存
yum makecache

#检查是否成功
yum search cloudera-manager-daemons cloudera-manager-agent cloudera-manager-server

#manager上安装
sudo yum install -y cloudera-manager-daemons cloudera-manager-server

#auto-TLS

#此步骤不要操作，否则可能导致安装失败
#sudo JAVA_HOME=/usr/java/jdk1.8.0_181-cloudera /opt/cloudera/cm-agent/bin/certmanager setup --configure-services

#查看是否创建成功
more /var/log/cloudera-scm-agent/certmanager.log

#Install and Configure Databases

#目前MariaDB官方已经停止支持10.1及之前的官方源了，建议改为试用pg，请参考：https://docs.cloudera.com/documentation/enterprise/6/latest/topics/cm_ig_extrnl_pstgrs.html#cmig_topic_5_6

#pg

CREATE ROLE scm LOGIN PASSWORD 'scm';
CREATE DATABASE scm OWNER scm ENCODING 'UTF8';

CREATE ROLE amon LOGIN PASSWORD 'amon';
CREATE DATABASE amon OWNER amon ENCODING 'UTF8';

CREATE ROLE rman LOGIN PASSWORD 'rman';
CREATE DATABASE rman OWNER rman ENCODING 'UTF8';

CREATE ROLE hue LOGIN PASSWORD 'hue';
CREATE DATABASE hue OWNER hue ENCODING 'UTF8';

CREATE ROLE hive LOGIN PASSWORD 'hive';
CREATE DATABASE hive OWNER hive ENCODING 'UTF8';

CREATE ROLE sentry LOGIN PASSWORD 'sentry';
CREATE DATABASE sentry OWNER sentry ENCODING 'UTF8';

CREATE ROLE nav LOGIN PASSWORD 'nav';
CREATE DATABASE nav OWNER nav ENCODING 'UTF8';

CREATE ROLE navms LOGIN PASSWORD 'navms';
CREATE DATABASE navms OWNER navms ENCODING 'UTF8';

CREATE ROLE oozie LOGIN PASSWORD 'oozie';
CREATE DATABASE oozie OWNER oozie ENCODING 'UTF8';

ALTER DATABASE scm SET standard_conforming_strings=off;

ALTER DATABASE amon SET standard_conforming_strings=off;

ALTER DATABASE rman SET standard_conforming_strings=off;

ALTER DATABASE hue SET standard_conforming_strings=off;

ALTER DATABASE hive SET standard_conforming_strings=off;

ALTER DATABASE sentry SET standard_conforming_strings=off;

ALTER DATABASE nav SET standard_conforming_strings=off;

ALTER DATABASE navms SET standard_conforming_strings=off;

ALTER DATABASE oozie SET standard_conforming_strings=off;

 

#MariaDB-server
vim /etc/yum.repos.d/MariaDB.repo

使用阿里云的源（推荐）：

[mariadb]
name = MariaDB
baseurl = http://mirrors.aliyun.com/mariadb/yum/10.1/centos7-amd64
gpgkey=http://mirrors.aliyun.com/mariadb/yum/RPM-GPG-KEY-MariaDB
gpgcheck=1

或者官方的源（速度可能较慢）：

[mariadb]
name = MariaDB
baseurl = http://yum.mariadb.org/10.1/centos7-amd64
gpgkey=https://yum.mariadb.org/RPM-GPG-KEY-MariaDB
gpgcheck=1

 

#安装新版本mariadb
sudo yum install -y MariaDB-server MariaDB-client

#Configuring and Starting the MariaDB Server
sudo systemctl stop mariadb
mkdir /root/innoDBLogFiles
mv /var/lib/mysql/ib_logfile0 /root/innoDBLogFiles/
mv /var/lib/mysql/ib_logfile1 /root/innoDBLogFiles/

#config mariadb
#backup config
mv /etc/my.cnf /etc/my.cnf.bak
#example
vim /etc/my.cnf
输入下面的配置文件的内容：
###########################配置文件开始################################################
[mysqld]
datadir=/var/lib/mysql
socket=/var/lib/mysql/mysql.sock
transaction-isolation = READ-COMMITTED
# Disabling symbolic-links is recommended to prevent assorted security risks;
# to do so, uncomment this line:
symbolic-links = 0
# Settings user and group are ignored when systemd is used.
# If you need to run mysqld under a different user or group,
# customize your systemd unit file for mariadb according to the
# instructions in http://fedoraproject.org/wiki/Systemd

key_buffer = 16M
key_buffer_size = 32M
max_allowed_packet = 32M
thread_stack = 256K
thread_cache_size = 64
query_cache_limit = 8M
query_cache_size = 64M
query_cache_type = 1

max_connections = 550
#expire_logs_days = 10
#max_binlog_size = 100M

#log_bin should be on a disk with enough free space.
#Replace '/var/lib/mysql/mysql_binary_log' with an appropriate path for your
#system and chown the specified folder to the mysql user.
log_bin=/var/lib/mysql/mysql_binary_log

#In later versions of MariaDB, if you enable the binary log and do not set
#a server_id, MariaDB will not start. The server_id must be unique within
#the replicating group.
server_id=1

binlog_format = mixed

read_buffer_size = 2M
read_rnd_buffer_size = 16M
sort_buffer_size = 8M
join_buffer_size = 8M

# InnoDB settings
innodb_file_per_table = 1
innodb_flush_log_at_trx_commit  = 2
innodb_log_buffer_size = 64M
innodb_buffer_pool_size = 4G
innodb_thread_concurrency = 8
innodb_flush_method = O_DIRECT
innodb_log_file_size = 512M

[mysqld_safe]
log-error=/var/log/mariadb/mariadb.log
pid-file=/var/run/mariadb/mariadb.pid

#
# include all files from the config directory
#
!includedir /etc/my.cnf.d

###########################配置文件结束################################################


#开机自启动mariaDB
sudo systemctl enable mariadb

#Start the MariaDB server:
sudo systemctl start mariadb

#secure_installation

sudo /usr/bin/mysql_secure_installation

###################按照示例操作###################################
[...]
Enter current password for root (enter for none):
OK, successfully used password, moving on...
[...]
Set root password? [Y/n] Y
New password:
Re-enter new password:
[...]
Remove anonymous users? [Y/n] Y
[...]
Disallow root login remotely? [Y/n] N
[...]
Remove test database and access to it [Y/n] Y
[...]
Reload privilege tables now? [Y/n] Y
[...]
All done!  If you've completed all of the above steps, your MariaDB
installation should now be secure.

Thanks for using MariaDB!
#################################################################


#Installing the MySQL JDBC Driver for MariaDB
wget https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-5.1.46.tar.gz
tar zxvf mysql-connector-java-5.1.46.tar.gz
sudo mkdir -p /usr/share/java/
cd mysql-connector-java-5.1.46
sudo cp mysql-connector-java-5.1.46-bin.jar /usr/share/java/mysql-connector-java.jar

#Creating Databases for Cloudera Software
mysql -u root -p

CREATE DATABASE scm DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE amon DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE rman DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE hue DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE hive DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE sentry DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE nav DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE navms DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE oozie DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;


GRANT ALL ON scm.* TO 'scm'@'%' IDENTIFIED BY 'scm';
GRANT ALL ON amon.* TO 'amon'@'%' IDENTIFIED BY 'amon';
GRANT ALL ON rman.* TO 'rman'@'%' IDENTIFIED BY 'rman';
GRANT ALL ON hue.* TO 'hue'@'%' IDENTIFIED BY 'hue';
GRANT ALL ON hive.* TO 'hive'@'%' IDENTIFIED BY 'hive';
GRANT ALL ON sentry.* TO 'sentry'@'%' IDENTIFIED BY 'sentry';
GRANT ALL ON nav.* TO 'nav'@'%' IDENTIFIED BY 'nav';
GRANT ALL ON navms.* TO 'navms'@'%' IDENTIFIED BY 'navms';
GRANT ALL ON oozie.* TO 'oozie'@'%' IDENTIFIED BY 'oozie';

#检查数据库创建
show databases;

#检查用户授权
SHOW GRANTS FOR 'scm'@'%';
SHOW GRANTS FOR 'amon'@'%';
SHOW GRANTS FOR 'rman'@'%';
SHOW GRANTS FOR 'hue'@'%';
SHOW GRANTS FOR 'hive*'@'%';
SHOW GRANTS FOR 'sentry'@'%';
SHOW GRANTS FOR 'nav'@'%';
SHOW GRANTS FOR 'navms'@'%';
SHOW GRANTS FOR 'oozie'@'%';

#Set up the Cloudera Manager Database in shell ( execute this command one by one)

sudo /opt/cloudera/cm/schema/scm_prepare_database.sh mysql  scm scm

sudo /opt/cloudera/cm/schema/scm_prepare_database.sh mysql  amon amon

sudo /opt/cloudera/cm/schema/scm_prepare_database.sh mysql  rman rman

sudo /opt/cloudera/cm/schema/scm_prepare_database.sh mysql  hue hue

sudo /opt/cloudera/cm/schema/scm_prepare_database.sh mysql  hive hive

sudo /opt/cloudera/cm/schema/scm_prepare_database.sh mysql  sentry sentry

sudo /opt/cloudera/cm/schema/scm_prepare_database.sh mysql  nav nav

sudo /opt/cloudera/cm/schema/scm_prepare_database.sh mysql  navms navms

sudo /opt/cloudera/cm/schema/scm_prepare_database.sh mysql  oozie oozie

#Install CDH and Other Software
#Start Cloudera Manager Server:
sudo systemctl start cloudera-scm-server
#Wait several minutes for the Cloudera Manager Server to start. To observe the startup process, run the following on the Cloudera Manager Server host:
sudo tail -f /var/log/cloudera-scm-server/cloudera-scm-server.log
#When you see this log entry, the Cloudera Manager Admin Console is ready:
INFO WebServerImpl:com.cloudera.server.cmf.WebServerImpl: Started Jetty server.


#Open your Browser then go to the address as below:
#http://<server_host>:7180