CDH环境搭建

 一、CDH环境搭建目录

--CDH相关包的下载地址：
http://archive.cloudera.com/cm5/cm/5/
http://archive.cloudera.com/cdh5/parcels/latest/

---创建用户
用户：
userdel -r cloudera-scm
groupadd cloudera-scm
useradd -g cloudera-scm cloudera-scm
passwd cloudera-scm

---杀进程
ps -ef | grep cloudera-scm | awk '{ print $2 }' | sudo xargs kill -9
ps -ef | grep mysql | awk '{ print $2 }' | xargs kill -9

ps -ef |grep cloudera-scm |grep -v grep|cut -c 9-15|xargs kill -9
ps -ef |grep mycanal-1.0-SNAPSHOT.jar |grep -v grep| cut -c 9-15 | xargs kill -9

---创建CDH需要目录
mkdir -p /qhapp/cdh/var/data
/qhapp/cdh
.
├── cloudera
│   ├── parcel-cache
│   └── parcels
├── cm
│   └── cm-5.10.0
├── software
│   ├── CDH-5.10.0-1.cdh5.10.0.p0.41-el6.parcel
│   ├── CDH-5.10.0-1.cdh5.10.0.p0.41-el6.parcel.sha
│   ├── cloudera-manager-el6-cm5.10.0_x86_64.tar.gz
│   ├── jdk1.7.0_80.zip
│   ├── manifest.json
│   ├── mysql-advanced-5.7.9-linux-glibc2.5-x86_64.tar.gz
│   └── mysql-connector-java-5.1.40.jar
└── var
├── data
├── he
├── lib
├── log
├── run
└── tmp

----配置hosts文件
127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
10.60.81.157 sz280317

--配置主机名
NETWORKING=yes
HOSTNAME=sz280330

---时间同步

ntpdate -u 10.60.81.168

server 10.60.4.12 prefer
server 10.60.4.13
server 10.50.4.12
server 10.50.4.13
driftfile /var/lib/ntp/drift
keys /etc/ntp/keys
disable monitor
restrict default kod nomodify notrap nopeer noquery
restrict -6 default kod nomodify notrap nopeer noquery
restrict 127.0.0.1
restrict -6 ::1
tinker panic 120

---用户权限配置
root ALL=(ALL) ALL
%cloudera-scm ALL=(ALL) NOPASSWD: CDH

----无密配置
scp ~/.ssh/authorized_keys root@slave01:~/.ssh/
scp ~/.ssh/authorized_keys jboss@sz280317:~/.ssh/
scp ~/.ssh/authorized_keys jboss@sz280318:~/.ssh/
scp ~/.ssh/authorized_keys jboss@sz280319:~/.ssh/
scp ~/.ssh/authorized_keys jboss@sz280320:~/.ssh/
scp ~/.ssh/authorized_keys jboss@sz280321:~/.ssh/
scp ~/.ssh/authorized_keys jboss@sz280322:~/.ssh/
scp ~/.ssh/authorized_keys jboss@sz280323:~/.ssh/
scp ~/.ssh/authorized_keys jboss@sz280324:~/.ssh/
scp ~/.ssh/authorized_keys jboss@sz280325:~/.ssh/
scp ~/.ssh/authorized_keys jboss@sz280326:~/.ssh/
scp ~/.ssh/authorized_keys jboss@sz280327:~/.ssh/
scp ~/.ssh/authorized_keys jboss@sz280328:~/.ssh/
scp ~/.ssh/authorized_keys jboss@sz280329:~/.ssh/
scp ~/.ssh/authorized_keys jboss@sz280330:~/.ssh/
scp ~/.ssh/authorized_keys jboss@sz280331:~/.ssh/

---jdk安装
rpm -ivh jdk-8u151-linux-x64.rpm

---mysql用户配置
alter user 'root'@'localhost' identified by 'Hgx@123456';
flush privileges;
GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' IDENTIFIED BY 'Hgx@123456' WITH GRANT OPTION;

---cdh用户创建
/qhapp/cdh/cm/cm-5.15.0/share/cmf/schema/scm_prepare_database.sh mysql cm -hlocalhost -uroot -pMysql@123456 --scm-host localhost scm scm scm

chown -r cloudera-scm:cloudera-scm cdh

useradd --system --home=/qhapp/cdh/cm/cm-5.15.0/run/cloudera-scm-server/ --no-create-home --shell=/bin/false --comment "Cloudera SCM User" cloudera-scm2

usermod -a -G root cloudera-scm

rpm -e mysql-community-libs-5.7.20-1.el6.x86_64 --nodeps
rpm -e mysql-community-server-5.7.20-1.el6.x86_64 --nodeps
rpm -e mysql-community-common-5.7.20-1.el6.x86_64 --nodeps
rpm -e mysql-community-client-5.7.20-1.el6.x86_64 --nodeps

rpm -e mysql-community-libs-compat-5.7.20-1.el6.x86_64 --nodeps

---启动cdh
sudo /qhapp/cdh/cm/cm-5.15.0/etc/init.d/cloudera-scm-server start
sudo /qhapp/cdh/cm/cm-5.15.0/etc/init.d/cloudera-scm-agent start

---停止cdh
/qhapp/cdh/cm/cm-5.15.0/etc/init.d/cloudera-scm-server stop
/qhapp/cdh/cm/cm-5.15.0/etc/init.d/cloudera-scm-agent stop

---目录权限
chown -R cloudera-scm:cloudera-scm /qhapp/cdh

---创建cdh中hive数据库
create database hive DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
create database oozie DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
create database hue DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
create database hue DEFAULT CHARSET utf8 COLLATE utf8_general_ci;

---java驱动配置
cp mysql-connector-java-8.0.11.jar /qhapp/cdh/cloudera/parcels/CDH-5.15.0-1.cdh5.15.0.p0.21/lib/hive/lib/

kill -9 $(pgrep -f supervisord)

---Hadoop收到启动
hadoop namenode -recover
hadoop namenode -recover
./hadoop-daemon.sh start journalnode

rpm -ivh mysql-community-common-5.7.20-1.el6.x86_64.rpm
rpm -ivh mysql-community-libs-5.7.20-1.el6.x86_64.rpm
rpm -ivh mysql-community-client-5.7.20-1.el6.x86_64.rpm
rpm -ivh mysql-community-server-5.7.20-1.el6.x86_64.rpm

alter user 'root'@'localhost' identified by 'Mysql@123456';

set password for 'root'@'localhost' = password('Mysql@123456');
alter user 'root'@'localhost' identified with mysql_native_password by 'Mysql@123456';
flush privileges;
3. 创建同步数据用户及授权用户
3.1．授权其他机器登陆
GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' IDENTIFIED BY 'Mysql@123456' WITH GRANT OPTION;
flush privileges;

create database hue DEFAULT CHARSET utf8 COLLATE utf8_general_ci;

rm -rf /qhapp/cdh/cloudera/parcel-cache /qhapp/cdh/cloudera/parcels

cp mysql-connector-java-5.1.35.jar /qhapp/cdh/cm/cm-5.15.0/share/cmf/lib/

cp mysql-connector-java-5.1.35.jar /opt/cm-5.3.8/share/cmf/lib/
/qhapp/cdh/cm/cm-5.15.0/share/cmf/schema/scm_prepare_database.sh mysql cm -hlocalhost -uroot -pMysql@123456 --scm-host localhost scm scm scm

 

useradd --system --home=/qhapp/cdh/cm/cm-5.15.0/run/cloudera-scm-server/ --no-create-home --shell=/bin/false --comment "Cloudera SCM User" cloudera-scm

 

mysql主库链接时起始的binlog文件
mysql主库链接时起始的binlog偏移量
mysql主库链接时起始的binlog的时间戳

/qhapp/cdh/cloudera/parcels/CDH/jars
ln -s test.log test2.log
ln -s ../../../jars/jline-2.11.jar jline-2.11.jar
231067965

1533191555000
1483200000

把文件添加到/etc/ld.so.conf,然后执行命令ldconfig 生效
/usr/lib64/mysql/libmysqlclient_r.so.16.0.0
/usr/lib64/mysql/libmysqlclient.so.16.0.0