cdh本地源安装-自用

yum 安装cm

参考官网

主机名

/etc/hosts

hostnamectl  set-hostname  foo-1.data.com 
hostnamectl  set-hostname  foo-2.data.com
hostnamectl  set-hostname  foo-2.data.com

hostnamectl  set-hostname   cdh-1.data.com
hostnamectl  set-hostname   cdh-2.data.com
cat >> /etc/hosts <<EOF
192.168.2.150  cdh-1.xxxx.com  cdh-1
192.168.2.117  cdh-2.xxxx.com  cdh-2
EOF

192.168.1.142   foo-1.xxxx.com  foo-1
192.168.1.143   foo-2.xxxx.com  foo-2

swappiness

/proc/sys/vm/swappiness 设置为 10
 
echo never > /sys/kernel/mm/transparent_hugepage/defrag

selinux

vi /etc/selinux/config
sed  -i 's/SELINUX=enforcing/SELINUX=disabled/'  /etc/selinux/confi
getenforce 

防火墙

systemctl stop firewalld
systemctl disable firewalld
systemctl status firewalld

ssh免密

ssh-keygen -t rsa
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
chmod 600 ~/.ssh/authorized_keys
在一台机器上自己ssh 自己，成功后将.ssh目录拷贝到其他机器

repo本地源

cat >   /etc/yum.repos.d/local.repo  <<EOF
[localrepo]
name=localrepo
baseurl=file:///cm-5.13.3/
#gpgkey=http://192.168.2.113/cm-5.13.3/RPM-GPG-KEY-cloudera    
enabled = 1
gpgcheck = 0
EOF

远程源

rm -rf  /etc/yum.repos.d/*
cat >   /etc/yum.repos.d/cloudera-manager.repo  <<EOF
[clouderamanager]
name=Cloudera Manager
baseurl=http://192.168.2.113/cm-5.13.3/
gpgkey=http://192.168.2.113/cm-5.13.3/RPM-GPG-KEY-cloudera    
enabled = 1
gpgcheck = 0
EOF


 yum  clean  all  && yum makecache

ntp

主节点
yum -y install  ntp

grep  -v   "^#"   /etc/ntp.conf |   grep  -v "^$"

cat >  /etc/ntp.conf <<EOF
driftfile /var/lib/ntp/drift
restrict default nomodify notrap nopeer noquery
restrict 127.0.0.1 
restrict ::1
restrict  192.168.2.0  mask 255.255.255.0 nomodify notrap
server 127.127.1.0
Fudge 127.127.1.0 stratum 1
includefile /etc/ntp/crypto/pw
keys /etc/ntp/keys
disable monitor
EOF

客户端
grep  -v   "^#"   /etc/ntp.conf |   grep  -v "^$"
cat >  /etc/ntp.conf <<EOF
driftfile /var/lib/ntp/drift
restrict default nomodify notrap nopeer noquery
####restrict 192.168.1.135 nomodify notrap noquery  这个不知道要不要
restrict 127.0.0.1 
restrict ::1
restrict 192.168.2.0  mask 255.255.255.0 nomodify notrap
server 192.168.2.150 profer
Fudge  192.168.2.150 stratum 10
includefile /etc/ntp/crypto/pw
keys /etc/ntp/keys
disable monitor
EOF
开机自动启动
 systemctl enable ntpd.service
 systemctl  start ntpd.service
 手动同步
 ntpdate -u  cdh-1
  ntpq -p 

查看ntp运行状态

    ntpstat

jdk（所有节点）

卸载自带的jdk-open
rpm -qa | grep java
rpm -qa | grep jdk - yum -y remove xxjdk   #删除所有的jdk
yum remove -y *jdk*
yum remove -y *java*

rpm  -ivh jdk-8u202-linux-x64.rpm

yum  install   java -y 

数据库安装

卸载原有的mariadb 
rpm -qa | grep mariadb #查看安装信息
rpm --nodeps -e  `rpm -qa | grep mariadb`
安装数据库顺序  最好解压安装
tar -xf mysql-5.7.21-1.el7.x86_64.rpm-bundle.tar    
首先安装（其他节点也需要安装，同时也安装 mysql-community-libs-compat-5.7.21-1.el7.x86_64.rpm ）

yum install net-tools  -y  #安装net-tools包
yum install perl -y        #安装perl包


安装顺序
rpm -ivh mysql-community-common-5.7.21-1.el7.x86_64.rpm
rpm -ivh mysql-community-libs-5.7.21-1.el7.x86_64.rpm
rpm -ivh mysql-community-client-5.7.21-1.el7.x86_64.rpm
rpm -ivh mysql-community-server-5.7.21-1.el7.x86_64.rpm
rpm -ivh mysql-community-libs-compat-5.7.21-1.el7.x86_64.rpm(compat解决hue数据库连接不上)   比较重要(其他节点都要安装)
rpm -ivh mysql-community-devel-5.7.21-1.el7.x86_64.rpm                                   #安装mysqlclient需要用到

yum  install  mysql-community-common  -y 
yum  install  mysql-community-libs  -y 
yum  install  mysql-community-client   -y  
yum  install  mysql-community-libs-compat  -y 
yum  install  mysql-community-server   -y 
yum  install  mysql-community-devel  -y 



数据库默认配置
[mysqld]
datadir=/var/lib/mysql
socket=/var/lib/mysql/mysql.sock
transaction-isolation = READ-COMMITTED
# Disabling symbolic-links is recommended to prevent assorted security risks;
# to do so, uncomment this line:
symbolic-links = 0

key_buffer_size = 32M
max_allowed_packet = 32M
thread_stack = 256K
thread_cache_size = 64
query_cache_limit = 8M
query_cache_size = 64M
query_cache_type = 1

max_connections = 550
#expire_logs_days = 10
#max_binlog_size = 100M

#log_bin should be on a disk with enough free space.
#Replace '/var/lib/mysql/mysql_binary_log' with an appropriate path for your
#system and chown the specified folder to the mysql user.
log_bin=/var/lib/mysql/mysql_binary_log

#In later versions of MySQL, if you enable the binary log and do not set
#a server_id, MySQL will not start. The server_id must be unique within
#the replicating group.
server_id=1

binlog_format = mixed

read_buffer_size = 2M
read_rnd_buffer_size = 16M
sort_buffer_size = 8M
join_buffer_size = 8M

# InnoDB settings
innodb_file_per_table = 1
innodb_flush_log_at_trx_commit  = 2
innodb_log_buffer_size = 64M
innodb_buffer_pool_size = 4G
innodb_thread_concurrency = 8
innodb_flush_method = O_DIRECT
innodb_log_file_size = 512M

[mysqld_safe]
log-error=/var/log/mysqld.log
pid-file=/var/run/mysqld/mysqld.pid

sql_mode=STRICT_ALL_TABLES




启动mysql 
systemctl start mysqld
systemctl enable mysqld
systemctl daemon-reload

查看root初始密码
cat /var/log/mysqld.log |grep password

#vi /etc/my.cnf 
在[mysqld]的段中加上一句：skip-grant-tables 
例如： 
[mysqld] 
datadir=/var/lib/mysql 
socket=/var/lib/mysql/mysql.sock 
skip-grant-tables 


USE mysql 修改密码; 
ALTER USER 'root'@'localhost' IDENTIFIED BY 'xx.com';
FLUSH PRIVILEGES;


mysql -uroot -pxx.com

登录mysql数据库
mysql -uroot –p密码 # 密码为上一步看到的
首先，修改validate_password_policy参数的值


set global validate_password_policy=0;
set global validate_password_length=2;
set global validate_password_special_char_count=0;
set global validate_password_mixed_case_count=0;
set global validate_password_number_count=0;
set password = password('daxxx.com');

查看密码策略:
SHOW VARIABLES LIKE 'validate_password%';


授权用户远程连接
GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' IDENTIFIED BY 'xxx.com' WITH GRANT OPTION;
GRANT ALL PRIVILEGES ON *.* TO 'cdh'@'%' IDENTIFIED BY 'xxx.com' WITH GRANT OPTION;



set global validate_password_policy=0;
set global validate_password_length=2;
set global validate_password_special_char_count=0;
set global validate_password_mixed_case_count=0;
set global validate_password_number_count=0;

grant all privileges on *.* to 'scm'@'localhost' identified by 'scm' with grant option;
grant all privileges on *.* to 'scm'@'127.0.0.1' identified by 'scm' with grant option;
grant all privileges on *.* to 'scm'@'%' identified by 'scm' with grant option;


grant all privileges on *.* to 'root'@'localhost' identified by 'xxxx.com' with grant option;
grant all privileges on *.* to 'root'@'127.0.0.1' identified by 'dxx.com'  with grant option;
grant all privileges on *.* to 'root'@'%' identified by 'xxa.com'  with grant option;


grant all privileges on *.* to 'cdh'@'localhost' identified by 'dxx.com' with grant option;
grant all privileges on *.* to 'cdh'@'127.0.0.1' identified by 'xx.com'  with grant option;
grant all privileges on *.* to 'cdh'@'%' identified by 'xxxa.com'  with grant option;


flush privileges;





创建需要的数据库：
create database metastore default character set utf8  DEFAULT COLLATE utf8_general_ci;
create database scm default character set utf8  DEFAULT COLLATE utf8_general_ci;
create database amon default character set utf8  DEFAULT COLLATE utf8_general_ci;
create database rmon default character set utf8  DEFAULT COLLATE utf8_general_ci;
create database hue default character set utf8  DEFAULT COLLATE utf8_general_ci;
create database sentry default character set utf8  DEFAULT COLLATE utf8_general_ci;
create database nav default character set utf8  DEFAULT COLLATE utf8_general_ci;
create database navms default character set utf8  DEFAULT COLLATE utf8_general_ci;
create database oozie default character set utf8  DEFAULT COLLATE utf8_general_ci;
create database hive default character set utf8  DEFAULT COLLATE utf8_general_ci;

CM安装

[root@foo-1 yum.repos.d]# cat  cloudera-manager.repo 
[clouderamanager]
name=Cloudera Manager
baseurl=http://192.168.2.113/cm-5.13.3/
gpgkey=http://192.168.2.113/cm-5.13.3/RPM-GPG-KEY-cloudera    
enabled = 1
gpgcheck = 0

install the Cloudera Manager packages. 
在管理节点安装cloudera-manager-daemons cloudera-manager-server
#RHEL, if you have a yum repo configured
sudo yum install cloudera-manager-daemons cloudera-manager-server
sudo yum install cloudera-manager-agent


#On all hosts, run the following command to install the Cloudera Manager agent:
在子节点，安装管理客户端
sudo yum install cloudera-manager-daemons cloudera-manager-agent


*** 在所有节点修改管理代理连接管理服务器的主机名
On all hosts, configure the Cloudera Manager Agent to point to the Cloudera Manager Server by setting the following properties in the /etc/cloudera-scm-agent/config.ini configuration file:
server_host	Name of the host where Cloudera Manager Server is running.
server_port	Port on the host where Cloudera Manager Server is running.


sed -i 's/server_host=localhost/server_host=cdh-1/'  /etc/cloudera-scm-agent/config.ini 

数据库连接程序

初始化  yum安装的scm_prepare_database.sh 位置
 mkdir  /usr/share/java/  &&  cd   /usr/share/java/

curl   -O    http://192.168.2.113/mysql-connector-java/mysql-connector-java-5.1.46.tar.gz


mkdir  /usr/share/java/
cp    mysql-connector-java-5.1.46-bin.jar    mysql-connector-java.jar
chmod  755  mysql-connector-java-5.1.46-bin.jar    mysql-connector-java.jar
 
cp  mysql-connector-java-5.1.46-bin.jar    mysql-connector-java.jar     /usr/share/cmf/lib/
mkdir   /usr/share/java/



初始化数据库：
/usr/share/cmf/schema/scm_prepare_database.sh   mysql cm  -hlocalhost  -uroot   -pxxxx.com   --scm-host cdh-1  scm scm  scm


所有节点运行管理代理
管理节点
      systemctl start cloudera-scm-server
      systemctl start cloudera-scm-agent
 
判断是否启动成功
find  / -name   cloudera-scm-server.log
tail  -f  /var/log/cloudera-scm-server/cloudera-scm-server.log
grep "Started Jetty server"    cloudera-scm-server.log


      
子节点
systemctl start cloudera-scm-agent



安装后节点也需要安装
cd    /usr/share/cmf/lib/
mv   /usr/share/java/mysql-connector-java-5.1.46-bin.jar  .
 
cdh 也需要驱动
scp  mysql-connector-java-5.1.46-bin.jar   foo-2:/opt/cloudera/parcels/CDH-5.13.3-1.cdh5.13.3.p0.2/lib
 
 hive 数据库也需要
scp  mysql-connector-java-5.1.46-bin.jar   foo-2:/opt/cloudera/parcels/CDH-5.13.3-1.cdh5.13.3.p0.2/lib/hive/lib
scp  mysql-connector-java-5.1.46-bin.jar   foo-2:/opt/cloudera/parcels/CDH-5.13.3-1.cdh5.13.3.p0.2/lib
scp  mysql-connector-java-5.1.46-bin.jar   foo-2:/opt/cloudera/parcels/CDH-5.13.3-1.cdh5.13.3.p0.2/lib/hive/lib
scp  mysql-connector-java-5.1.46-bin.jar   foo-2:/opt/cloudera/parcels/CDH-5.13.3-1.cdh5.13.3.p0.2/lib/oozie/lib
scp  mysql-connector-java-5.1.46-bin.jar   foo-2:/opt/cloudera/parcels/CDH-5.13.3-1.cdh5.13.3.p0.2/lib/hue/lib
scp  mysql-connector-java-5.1.46-bin.jar   foo-2:/var/lib/oozie
scp  mysql-connector-java-5.1.46-bin.jar   foo-2:/var/lib/hue

http://192.168.2.150:7180/cmf/clusters/1/express-add-services/index#step=commandDetailsStep