cdh ubuntu1604 安装cdh5.13.3
基本步骤
ubuntu1604 部署cdh
ntp ssh vim jdk openssh-server openssh-client
离线软件源构建
# 下载离线deb 包, 下载所有的依赖
httpd 支持
/var/www/html/apt/pools 该目录下为deb 包文件
-rw-r--r-- 1 root root 42138 Feb 17 2020 unattended-upgrades_1.1ubuntu1.18.04.7~16.04.6_all.deb
-rw-r--r-- 1 root root 1035720 Mar 23 07:48 vim_2%3a7.4.1689-3ubuntu1.4_amd64.deb
-rw-r--r-- 1 root root 5168982 Mar 23 07:48 vim-runtime_2%3a7.4.1689-3ubuntu1.4_all.deb
-rw-r--r-- 1 root root 78794 Feb 12 2014 xz-utils_5.1.1alpha+20120614-2ubuntu2_amd64.deb
root@cdh-03:/var/www/html/apt/pools# pwd
apt download libaio1 ## 下载包
sudo apt-get install -d package-name
apt-get install dpkg-dev 下载工具
# 扫描软件,构建获取索引
dpkg-scanpackages . /dev/null | gzip -9c > Packages.gz # 构建索引 生成依赖
# 使用服务器搭建成为文件服务器
软件在网页服务器的显示
http://10.30.10.103/apt/pools/ 内有软件
############################################
##########################################
# 客户端配置使用
#############################################
cat > /etc/apt/sources.list<< EOF
deb [trusted=yes] http://192.168.1.104/apt/pools/ /
deb [trusted=yes] http://192.168.1.104/cdh xenial-cdh5 contrib
deb [trusted=yes] http://192.168.1.104/cm xenial-cm5 contrib
EOF
root远程
# 设置root 密码
sudo passwd root
vim /etc/ssh/sshd_config
PermitRootLogin yes
PasswordAuthentication yes
#####################################
sed -i 's/PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config
sed -i 's/#PasswordAuthentication yes/PasswordAuthentication yes/' /etc/ssh/sshd_config
### 重启服务
systemctl restart ssh
主机时间和时区
vim /etc/profile
TZ='Asia/Shanghai'; export TZ
主机名
sudo hostnamectl set-hostname cdh-01
防火墙
sudo ufw disable
hosts
sudo cat > /etc/hosts <<EOF
127.0.0.1 localhost
127.0.1.1 localhost.AirDream localhost
# The following lines are desirable for IPv6 capable hosts
::1 localhost ip6-localhost ip6-loopback
ff02::1 ip6-allnodes
ff02::2 ip6-allrouters
192.168.1.101 cdh-01
192.168.1.103 cdh-02
192.168.1.104 cdh-03
EOF
ssh
ssh-keygen -t rsa
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
chmod 600 ~/.ssh/authorized_keys
scp -r /root/.ssh root@cdh-03:/root/
scp -r /root/.ssh root@cdh-02:/root/
ntp
apt-get install ntp
# 修改/etc/ntp.conf
## 主节点
cat > /etc/ntp.conf <<EOF
driftfile /var/lib/ntp/ntp.drift
statistics loopstats peerstats clockstats
filegen loopstats file loopstats type day enable
filegen peerstats file peerstats type day enable
filegen clockstats file clockstats type day enable
server 127.127.1.0
fudge 127.127.1.0 stratum 5
restrict -4 default kod notrap nomodify nopeer noquery limited
restrict -6 default kod notrap nomodify nopeer noquery limited
restrict 127.0.0.1
restrict ::1
restrict 10.30.10.0 mask 255.255.255.0 nomodify
EOF
#########################
service ntp restart
ntpq -p
######################
### 手动更新
ntpdate -d -u cdh-03
# slave 节点
cat > /etc/ntp.conf << EOF
driftfile /var/lib/ntp/ntp.drift
statistics loopstats peerstats clockstats
filegen loopstats file loopstats type day enable
filegen peerstats file peerstats type day enable
filegen clockstats file clockstats type day enable
server cdh-03 prefer
restrict -4 default kod notrap nomodify nopeer noquery limited
restrict -6 default kod notrap nomodify nopeer noquery limited
restrict 127.0.0.1
restrict ::1
restrict source notrap nomodify noquery
EOF
######操作
apt install ntpdate ntpstat
timedatectl set-timezone Asia/Shanghai
ntpdate -d -u cdh-03
ntpq -p
ntpstat
jdk
curl -LO http://192.168.1.104/jdk-8u141-linux-x64.tar.gz
tar -C /usr/local/ -xf jdk-8u141-linux-x64.tar.gz
ln -sv /usr/local/jdk1.8.0_141 /usr/local/jdk
cat >> ~/.bashrc <<EOF
export JAVA_HOME=/usr/local/jdk
export JRE_HOME=\${JAVA_HOME}/jre
export CLASSPATH=.:\${JAVA_HOME}/lib/tools.jar:\${JRE_HOME}/lib/dt.jar
export PATH=\${JAVA_HOME}/bin:\${JAVA_HOME}/jre/bin:\$PATH
EOF
source ~/.bashrc
## 默认的话无法获取配置的变量,需要设置特殊的链接, 所有的节点, 主节点和子节点/usr/local/jdk 是上一步使用的java 安装根目录
# 问题描述及处理 https://blog.csdn.net/u014310499/article/details/103254272
mkdir /usr/java && ln -sv /usr/local/jdk /usr/java/jdk1.8
数据库
sudo apt install mysql-community-server
systemctl status mysql
#vim /lib/systemd/system/mysql.service
# ExecStart=/usr/sbin/mysqld --daemonize --pid-file=/var/run/mysqld/mysqld.pid --bind-address=0.0.0.0
vim /etc/mysql/mysql.conf.d/mysqld.cnf
bind-address = 0.0.0.0
root@cdh-01:~# systemctl daemon-reload
root@cdh-01:~# systemctl restart mysql
#扩大root 权限
grant all privileges on *.* to 'root'@'localhost' identified by '123123' with grant option;
grant all privileges on *.* to 'root'@'127.0.0.1' identified by '123123' with grant option;
grant all privileges on *.* to 'root'@'%' identified by '123123' with grant option;
flush privileges ;
CREATE DATABASE amon DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE rman DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE hue DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE hive DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE sentry DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE nav DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE navms DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE oozie DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
host反向解析
描述：DNS反向解析错误，不能正确解析Cloudera Manager Server主机名
问题描述:
socket.error: [Errno 111] Connection refused
END (1)
could not contact scm server at localhost:7182, giving up
处理:
mv /usr/bin/host /usr/bin/host.bak
开始的时候并重启机器还是报错,等等自己就好了,可能是有效期过去了,需要在操作之前就操作
参考:https://www.jianshu.com/p/219949aeae8f
主节点初始化数据库
sudo apt-get install cloudera-manager-daemons cloudera-manager-server cloudera-manager-agent
/usr/share/cmf/schema/scm_prepare_database.sh mysql -hcdh-01 -uroot -p123123 --scm-host cdh-01 scm scm #确保数据库没有scm 库,有就删除 还需要输入密码 scm
systemctl start cloudera-scm-server
systemctl start cloudera-scm-agent
slave 节点
sudo apt-get install cloudera-manager-daemons cloudera-manager-agent
mysql-jdbc-driver
curl -LO http://192.168.1.104/mysql-connector-java-5.1.46.tar.gz
cp /home/dataxxa/mysql-connector-java-5.1.48/mysql-connector-java-5.1.46* /usr/share/cmf/lib
scp /home/dataxxa/mysql-connector-java-5.1.46/mysql-connector-java-5.1.46* cdh-02:/usr/share/cmf/lib
## 驱动复制到指定的目录下
### 这里是在网页初始化集群设置里复制,没有的话会报错 具体步骤在验证数据库连接正常后
# hive
scp /home/dataxxa/mysql-connector-java-5.1.46/mysql-connector-java-5.1.4* cdh-02:/opt/cloudera/parcels/CDH-5.13.3-1.cdh5.13.3.p0.2/lib/hive/lib
# oozie
scp /home/dataxxa/mysql-connector-java-5.1.46/mysql-connector-java-5.1.4* cdh-02:/var/lib/oozie
安装后
安装完成后节点操作
su - hdfs
hadoop fs -chmod 777 /
远程调用的主机需要配置cdh hosts 映射
集群完全重启
https://www.cnblogs.com/crazysquirrel/p/7522312.html
参考文档
https://blog.csdn.net/zhangdongan1991/article/details/104031983
https://www.cnblogs.com/getMyCodes/p/12048776.html
https://www.jianshu.com/p/90c209e6c22e
https://blog.csdn.net/a447942008/article/details/101843693
https://blog.csdn.net/q1370992706/article/details/79578444
http://archive.cloudera.com/cdh5/parcels/5.13.3/
https://www.cnblogs.com/chevin/p/9929140.html
https://docs.cloudera.com/documentation/enterprise/release-notes/topics/cm_vd.html#cmvd_topic_1
https://docs.cloudera.com/documentation/enterprise/5-13-x/topics/cm_ig_install_path_b.html
ａｐｔ下载依赖问题
https://www.jianshu.com/p/434a50c6c0ed
自建ｄｅｂ 源问题
https://blog.csdn.net/developerinit/article/details/79249119
主j节点操作步骤
1 ufw diable
2 ufw disable
3 cat > /etc/apt/sources.list<< EOF
4 deb [trusted=yes] http://10.30.10.103/apt/pools/ /
5 deb [trusted=yes] http://10.30.10.103/cdh xenial-cdh5 contrib
6 deb [trusted=yes] http://10.30.10.103/cm xenial-cm5 contrib
7 EOF
8 apt update
9 cat > /etc/apt/sources.list<< EOF
10 deb [trusted=yes] http://10.30.10.54/apt/pools/ /
11 deb [trusted=yes] http://10.30.10.54/cdh xenial-cdh5 contrib
12 deb [trusted=yes] http://10.30.10.54/cm xenial-cm5 contrib
13 EOF
14 apt update
15 ll
16 cat /etc/hosts
17 sed -i 's/PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config
18 sed -i 's/#PasswordAuthentication yes/PasswordAuthentication yes/' /etc/ssh/sshd_config
19 systemctl restart ssh
20 apt search ssh
21 ip a s
22 sudo cat > /etc/hosts <<EOF
23 127.0.0.1 localhost
24 127.0.1.1 localhost.AirDream localhost
25 # The following lines are desirable for IPv6 capable hosts
26 ::1 localhost ip6-localhost ip6-loopback
27 ff02::1 ip6-allnodes
28 ff02::2 ip6-allrouters
29 10.30.10.55 cdh-01
30 10.30.10.56 cdh-02
31 10.30.10.54 cdh-03
32 EOF
33 ssh cdh-02
34 ssh-keygen -t rsa
35 chmod 600 ~/.ssh/authorized_keys
36 ssh cdh-01
37 ll
38 scp -r /root/.ssh root@cdh-03:/root/
39 scp -r /root/.ssh root@cdh-02:/root/
40 ssh cdh-02
41 apt-get install ntp
42 cd /etc/apt/
43 ll
44 rm -rf sources.list~
45 cat ources.bak
46 cat sources.bak
47 ll
48 cat > /etc/ntp.conf << EOF
49 driftfile /var/lib/ntp/ntp.drift
50 statistics loopstats peerstats clockstats
51 filegen loopstats file loopstats type day enable
52 filegen peerstats file peerstats type day enable
53 filegen clockstats file clockstats type day enable
54 server cdh-03 prefer
55 restrict -4 default kod notrap nomodify nopeer noquery limited
56 restrict -6 default kod notrap nomodify nopeer noquery limited
57 restrict 127.0.0.1
58 restrict ::1
59 restrict source notrap nomodify noquery
60 EOF
61 systemctl start ntp
62 systemctl status ntp
63 ntpq -p
64 cat /etc/ntp.conf
65 systemctl restart ntp
66 systemctl status ntp
67 ntpq -p
68 apt install ntpstat
69 apt update
70 sudo gpg --keyserver keyserver.ubuntu.com --recv 327574EE02A818DD
71 sudo gpg --export --armor 327574EE02A818DD | sudo apt-key add -
72 apt update
73 ping www.baidu.com
74 ping 114.114.114.114
75 ip a s
76 apt -h
77 apt-get
78 apt list
79 cd
80 ll
81 cd /home/dataxxa/
82 ll
83 scp jdk-8u141-linux-x64.tar.gz mysql-connector-java-5.1.46.tar.gz cdh-02:~
84 ll
85 tar -C /usr/local/ -xf jdk-8u141-linux-x64.tar.gz
86 ln -sv /usr/local/jdk1.8.0_141 /usr/local/jdk
87 cat >> ~/.bashrc <<EOF
88 export JAVA_HOME=/usr/local/jdk
89 export JRE_HOME=\${JAVA_HOME}/jre
90 export CLASSPATH=.:\${JAVA_HOME}/lib/tools.jar:\${JRE_HOME}/lib/dt.jar
91 export PATH=\${JAVA_HOME}/bin:\${JAVA_HOME}/jre/bin:\$PATH
92 EOF
93 source .bashrc
94 java -version
95 source ~/.bashrc
96 java -version
97 sudo apt search mysql
98 sudo apt install mysql-community-server
99 history
100 cat /etc/mysql/my.cnf
101 cat /etc/mysql/mysql.cnf
102 systemctl status mysql-server
103 systemctl status mysql
104 ss -tanlp
105 systemctl stop mysql
106 vim /lib/systemd/system/mysql.service
107 apt install vim
108 vim /lib/systemd/system/mysql.service
109 systemctl restart mysql
110 systemctl daemon-reload
111 systemctl restart mysql
112 vim /lib/systemd/system/mysql.service
113 ss -tnalp
114 mysql -uroot -p123123
115 date
116 time
117 date
118 sudo apt-get install cloudera-manager-daemons cloudera-manager-server
119 sudo apt-get install cloudera-manager-daemons cloudera-manager-agent
120 cd /home/
121 ll
122 cd dataxxa/
123 ll
124 tar xvf mysql-connector-java-5.1.46.tar.gz
125 cp /home/dataxxa/mysql-connector-java-5.1.48/mysql-connector-java-5.1.48* /usr/share/cmf/lib
126 cp /home/dataxxa/mysql-connector-java-5.1.46/mysql-connector-java-5.1.4* /usr/share/cmf/lib
127 scp /home/dataxxa/mysql-connector-java-5.1.46/mysql-connector-java-5.1.4* cdh-02:/usr/share/cmf/lib
128 /usr/share/cmf/schema/scm_prepare_database.sh mysql -hcdh-01 -uroot -p123123 --scm-host cdh-01 scm scm
129 mysql -uroot -p
130 /usr/share/cmf/schema/scm_prepare_database.sh mysql -hcdh-01 -uroot -p123123 --scm-host cdh-01 scm 123456
131 history
132 systemctl start cloudera-scm-server
133 systemctl status cloudera-scm-server.service
134 cd /var/log/cloudera-
135 cd /var/log/cloudera-scm-server/
136 ll
137 cat cloudera-scm-server.out
138 echo $JAVA_HOME
139 systemctl status cloudera-scm-server.service
140 cat /etc/init.d/cloudera-scm-server
141 vim /etc/init.d/cloudera-scm-server
142 cat /etc/passwd
143 ll
144 cd
145 cat ~/.bashrc
146 vim /etc/profile
147 . /etc/profile
148 ss -tanlp
149 history
150 systemctl start cloudera-scm-server
151 cat /var/log/cloudera-
152 cat /var/log/cloudera-scm-
153 cat /var/log/cloudera-scm-server/
154 cd /var/log/cloudera-scm-server/
155 ll
156 cat cloudera-scm-server.out
157 find / -name cloudera-config.sh
158 cat /usr/lib/cmf/service/common/cloudera-config.sh
159 mkdir /usr/java
160 cd /usr/java
161 ll
162 cd jdk1.8
163 pwd
164 ll
165 cd
166 systemctl start cloudera-scm-server
167 ss -tanlp
168 cd /var/log/cloudera-
169 cd /var/log/cloudera-scm-server/
170 ll
171 tailf cloudera-scm-server.log
172 tailf cloudera-scm-server.out
173 systemctl start cloudera-scm-agent
174 systemctl start cloudera-scm-server
175 systemctl status cloudera-scm-server
176 ss -tanlp
177 systemctl status cloudera-scm-agent
178 ps -ef |grep agent
179 ss -tanlp
180 ip a s
181 history
182 cd /usr/java
183 ll
184 ss -tanlp
185 mv /usr/bin/host /usr/bin/host.bak
186 cd
187 service network restart
188 systemctl restart network
189 history
190 cat .ssh/id_rsa
191 systemctl restart cloudera-scm-server
192 ss -tanlp
193 history
194 /usr/bin/host
195 cat /etc/host
196 cat /etc/hosts
197 vim /etc/hosts
198 ll
199 ufw disable
200 reboot
201 ss -tanlp
202 cat /usr/bin/host
203 which host
204 mysql -uroot -p123123
205 history
206 cd /opt/cloudera/parcels/CDH-5.13.3-1.cdh5.13.3.p0.2/lib/hive/lib
207 history
208 scp /home/dataxxa/mysql-connector-java-5.1.46/mysql-connector-java-5.1.4* cdh-02:/opt/cloudera/parcels/CDH-5.13.3-1.cdh5.13.3.p0.2/lib/hive/lib
209 scp /home/dataxxa/mysql-connector-java-5.1.46/mysql-connector-java-5.1.4* cdh-02:/opt/cloudera/parcels/CDH-5.13.3-1.cdh5.13.3.p0.2/lib/oozie/lib
210 scp /home/dataxxa/mysql-connector-java-5.1.46/mysql-connector-java-5.1.4* cdh-02:/opt/cloudera/parcels/CDH-5.13.3-1.cdh5.13.3.p0.2/lib/hue/lib
211 scp /home/dataxxa/mysql-connector-java-5.1.46/mysql-connector-java-5.1.4* cdh-02:/var/lib/oozie
212 history
213 ip a s
214 ss -tanlp
215 apt search libwrap
216 cd /etc/apt/sources.list.d/
217 ll
218 cd ../
219 ll
220 cat sources.list
221 apt update
222 cd
223 vim /etc/mysql/mysql.conf.d/mysqld.cnf
224 history
离线包
离线包下载地址,网络下载较慢,使用迅雷可以加速
https://archive.cloudera.com/cdh5/repo-as-tarball/5.13.3/
https://archive.cloudera.com/cm5/repo-as-tarball/5.13.3/
https://archive.cloudera.com/cdh5/parcels/5.13.3/
下载后注意需要重新写入校验值
需要为下载的cdh parcel重新校验写入文件内部
sha1sum CDH-5.14.0-1.cdh5.14.0.p0.24-el7.parcel | awk '{ print $1 }' > CDH-5.14.0-1.cdh5.14.0.p0.24-el7.parcel.sha
apt内软件
cdh
cm
parcel
需要重新验证sha 然后写入文件
其他:
ubuntu1604 部署cdh
ubuntu 设置静态ip
ifconfig # 查看网络接口名字
# /etc/network/interfaces 文件 [注意是interfaces，有s]
sudo gedit /etc/network/interfaces // gedit 是用gedit进行编译打开，也可以用其他编译器打开
auto enp7s0
iface enp7s0 inet static
address 10.0.208.222
netmask 255.255.240.0
gateway 10.0.208.1
dns-nameservers 10.0.208.1
# 修改
sudo gedit /etc/NetworkManager/NetworkManager.conf
将“managed=false”修改为“managed=true”。意思是，将网络连接设置为自定义或手动
## 刷新IP
sudo ip addr flush enp7s0
sudo systemctl restart networking.service
# 重启
sudo service network-manager restart
## 解释
auto enp7s0 // 使用的网络接口，之前查询接口是为了这里
iface enp7s0 inet static // enp7s0这个接口，使用静态ip设置
address 10.0.208.222 // 设置ip地址
netmask 255.255.240.0 // 设置子网掩码
gateway 10.0.208.1 // 设置网关
dns-nameservers 10.0.208.1 // 设置dns服务器地址
ntp ssh vim jdk openssh-server openssh-client
离线软件源构建
#下载离线deb 包, 下载所有的依赖
httpd 支持
/var/www/html/apt/pools 该目录下为deb 包文件
-rw-r--r-- 1 root root 42138 Feb 17 2020 unattended-upgrades_1.1ubuntu1.18.04.7~16.04.6_all.deb
-rw-r--r-- 1 root root 1035720 Mar 23 07:48 vim_2%3a7.4.1689-3ubuntu1.4_amd64.deb
-rw-r--r-- 1 root root 5168982 Mar 23 07:48 vim-runtime_2%3a7.4.1689-3ubuntu1.4_all.deb
-rw-r--r-- 1 root root 78794 Feb 12 2014 xz-utils_5.1.1alpha+20120614-2ubuntu2_amd64.deb
root@cdh-03:/var/www/html/apt/pools# pwd
apt download libaio1 ## 下载包
sudo apt-get install -d package-name
apt-get install dpkg-dev 下载工具
# 扫描软件,构建获取索引
dpkg-scanpackages . /dev/null | gzip -9c > Packages.gz # 构建索引 生成依赖
# 使用服务器搭建成为文件服务器
软件在网页服务器的显示
http://10.30.10.103/apt/pools/ 内有软件
root远程
# 设置root 密码
sudo passwd root
vim /etc/ssh/sshd_config
PermitRootLogin yes
PasswordAuthentication yes
#####################################
sed -i 's/PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config && \
sed -i 's/#PasswordAuthentication yes/PasswordAuthentication yes/' /etc/ssh/sshd_config
### 重启服务
systemctl restart ssh
主机时间和时区
vim /etc/profile
TZ='Asia/Shanghai'; export TZ
主机名
sudo hostnamectl set-hostname cdh-01
防火墙
sudo ufw disable
hosts
sudo cat > /etc/hosts <<EOF
127.0.0.1 localhost
127.0.1.1 localhost.AirDream localhost
# The following lines are desirable for IPv6 capable hosts
::1 localhost ip6-localhost ip6-loopback
ff02::1 ip6-allnodes
ff02::2 ip6-allrouters
192.168.4.99 cdh-master
192.168.4.204 cdh-node-01
192.168.4.200 cdh-node-02
192.168.4.201 cdh-node-03
192.168.4.202 cdh-node-04
192.168.4.203 cdh-node-05
EOF
ssh
ssh-keygen -t rsa
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
chmod 600 ~/.ssh/authorized_keys
scp -r /root/.ssh root@cdh-03:/root/
scp -r /root/.ssh root@cdh-02:/root/
配置源
############################################
##########################################
# 客户端配置使用 任何节点都需要配置
#############################################
cat > /etc/apt/sources.list<< EOF
deb [trusted=yes] http://192.168.4.234/apt/pools/ /
deb [trusted=yes] http://192.168.4.234/cdh xenial-cdh5 contrib
deb [trusted=yes] http://192.168.4.234/cm xenial-cm5 contrib
EOF
ntp
apt-get install ntp
#
# 修改/etc/ntp.conf
## 主节点
cat > /etc/ntp.conf <<EOF
driftfile /var/lib/ntp/ntp.drift
statistics loopstats peerstats clockstats
filegen loopstats file loopstats type day enable
filegen peerstats file peerstats type day enable
filegen clockstats file clockstats type day enable
server 127.127.1.0
fudge 127.127.1.0 stratum 5
restrict -4 default kod notrap nomodify nopeer noquery limited
restrict -6 default kod notrap nomodify nopeer noquery limited
restrict 127.0.0.1
restrict ::1
restrict 192.168.4.0 mask 255.255.255.0 nomodify
EOF
#########################
service ntp restart
ntpq -p
######################
### 手动更新
ntpdate -d -u cdh-master
# slave 节点
cat > /etc/ntp.conf << EOF
driftfile /var/lib/ntp/ntp.drift
statistics loopstats peerstats clockstats
filegen loopstats file loopstats type day enable
filegen peerstats file peerstats type day enable
filegen clockstats file clockstats type day enable
server cdh-master prefer
restrict -4 default kod notrap nomodify nopeer noquery limited
restrict -6 default kod notrap nomodify nopeer noquery limited
restrict 127.0.0.1
restrict ::1
restrict source notrap nomodify noquery
EOF
######操作
apt install ntpdate ntpstat
timedatectl set-timezone Asia/Shanghai
ntpdate -d -u cdh-master
ntpq -p
ntpstat # 同步状态查询
jdk
curl -LO http://192.168.4.234/jdk-8u141-linux-x64.tar.gz && \
tar -C /usr/local/ -xf jdk-8u141-linux-x64.tar.gz && \
ln -sv /usr/local/jdk1.8.0_141 /usr/local/jdk
cat >> ~/.bashrc <<EOF
export JAVA_HOME=/usr/local/jdk
export JRE_HOME=\${JAVA_HOME}/jre
export CLASSPATH=.:\${JAVA_HOME}/lib/tools.jar:\${JRE_HOME}/lib/dt.jar
export PATH=\${JAVA_HOME}/bin:\${JAVA_HOME}/jre/bin:\$PATH
EOF
source ~/.bashrc
source .bashrc && java -version
## 默认的话无法获取配置的变量,需要设置特殊的链接, 所有的节点, 主节点和子节点/usr/local/jdk 是上一步使用的java 安装根目录
# 问题描述及处理 https://blog.csdn.net/u014310499/article/details/103254272
mkdir /usr/java && ln -sv /usr/local/jdk /usr/java/jdk1.8
数据库
sudo apt install mysql-community-server
systemctl status mysql
#vim /lib/systemd/system/mysql.service
# ExecStart=/usr/sbin/mysqld --daemonize --pid-file=/var/run/mysqld/mysqld.pid --bind-address=0.0.0.0
vim /etc/mysql/mysql.conf.d/mysqld.cnf
bind-address = 0.0.0.0
root@cdh-01:~# systemctl daemon-reload
root@cdh-01:~# systemctl restart mysql
#扩大root 权限
grant all privileges on *.* to 'root'@'localhost' identified by 'lynn' with grant option;
grant all privileges on *.* to 'root'@'127.0.0.1' identified by 'lynn' with grant option;
grant all privileges on *.* to 'root'@'%' identified by 'lynn' with grant option;
flush privileges ;
CREATE DATABASE amon DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE rman DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE hue DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE hive DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE sentry DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE nav DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE navms DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE oozie DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
host反向解析
描述：DNS反向解析错误，不能正确解析Cloudera Manager Server主机名
问题描述:
socket.error: [Errno 111] Connection refused
END (1)
could not contact scm server at localhost:7182, giving up
处理:
mv /usr/bin/host /usr/bin/host.bak
开始的时候并重启机器还是报错,等等自己就好了,可能是有效期过去了,需要在操作之前就操作
参考:https://www.jianshu.com/p/219949aeae8f
主节点初始化数据库
sudo apt-get install cloudera-manager-daemons cloudera-manager-server cloudera-manager-agent
/usr/share/cmf/schema/scm_prepare_database.sh mysql -hcdh-01 -uroot -plynn --scm-host cdh-01 scm scm #确保数据库没有scm 库,有就删除 还需要输入密码 scm
systemctl start cloudera-scm-server
systemctl start cloudera-scm-agent
slave 节点
sudo apt-get install cloudera-manager-daemons cloudera-manager-agent
mysql-jdbc-driver
curl -LO http://192.168.4.234/mysql-connector-java-5.1.46.tar.gz
cp mysql-connector-java-5.1.46/mysql-connector-java-5.1.46* /usr/share/cmf/lib
scp mysql-connector-java-5.1.46/mysql-connector-java-5.1.46* cdh-02:/usr/share/cmf/lib
## 驱动复制到指定的目录下
### 这里是在网页初始化集群设置里复制,没有的话会报错 具体步骤在验证数据库连接正常后
# hive
scp mysql-connector-java-5.1.46/mysql-connector-java-5.1.4* cdh-02:/opt/cloudera/parcels/CDH-5.13.3-1.cdh5.13.3.p0.2/lib/hive/lib
# oozie
scp /home/dataxxa/mysql-connector-java-5.1.46/mysql-connector-java-5.1.4* cdh-02:/var/lib/oozie
安装后
安装完成后节点操作
su - hdfs
hadoop fs -chmod 777 /
#远程调用的主机需要配置cdh hosts 映射
集群完全重启
https://www.cnblogs.com/crazysquirrel/p/7522312.html
参考文档
https://blog.csdn.net/zhangdongan1991/article/details/104031983
https://www.cnblogs.com/getMyCodes/p/12048776.html
https://www.jianshu.com/p/90c209e6c22e
https://blog.csdn.net/a447942008/article/details/101843693
https://blog.csdn.net/q1370992706/article/details/79578444
http://archive.cloudera.com/cdh5/parcels/5.13.3/
https://www.cnblogs.com/chevin/p/9929140.html
https://docs.cloudera.com/documentation/enterprise/release-notes/topics/cm_vd.html#cmvd_topic_1
https://docs.cloudera.com/documentation/enterprise/5-13-x/topics/cm_ig_install_path_b.html
ａｐｔ下载依赖问题
https://www.jianshu.com/p/434a50c6c0ed
自建ｄｅｂ 源问题
https://blog.csdn.net/developerinit/article/details/79249119
主j节点操作步骤
1 ufw diable
2 ufw disable
3 cat > /etc/apt/sources.list<< EOF
4 deb [trusted=yes] http://10.30.10.103/apt/pools/ /
5 deb [trusted=yes] http://10.30.10.103/cdh xenial-cdh5 contrib
6 deb [trusted=yes] http://10.30.10.103/cm xenial-cm5 contrib
7 EOF
8 apt update
9 cat > /etc/apt/sources.list<< EOF
10 deb [trusted=yes] http://10.30.10.54/apt/pools/ /
11 deb [trusted=yes] http://10.30.10.54/cdh xenial-cdh5 contrib
12 deb [trusted=yes] http://10.30.10.54/cm xenial-cm5 contrib
13 EOF
14 apt update
15 ll
16 cat /etc/hosts
17 sed -i 's/PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config
18 sed -i 's/#PasswordAuthentication yes/PasswordAuthentication yes/' /etc/ssh/sshd_config
19 systemctl restart ssh
20 apt search ssh
21 ip a s
22 sudo cat > /etc/hosts <<EOF
23 127.0.0.1 localhost
24 127.0.1.1 localhost.AirDream localhost
25 # The following lines are desirable for IPv6 capable hosts
26 ::1 localhost ip6-localhost ip6-loopback
27 ff02::1 ip6-allnodes
28 ff02::2 ip6-allrouters
29 10.30.10.55 cdh-01
30 10.30.10.56 cdh-02
31 10.30.10.54 cdh-03
32 EOF
33 ssh cdh-02
34 ssh-keygen -t rsa
35 chmod 600 ~/.ssh/authorized_keys
36 ssh cdh-01
37 ll
38 scp -r /root/.ssh root@cdh-03:/root/
39 scp -r /root/.ssh root@cdh-02:/root/
40 ssh cdh-02
41 apt-get install ntp
42 cd /etc/apt/
43 ll
44 rm -rf sources.list~
45 cat ources.bak
46 cat sources.bak
47 ll
48 cat > /etc/ntp.conf << EOF
49 driftfile /var/lib/ntp/ntp.drift
50 statistics loopstats peerstats clockstats
51 filegen loopstats file loopstats type day enable
52 filegen peerstats file peerstats type day enable
53 filegen clockstats file clockstats type day enable
54 server cdh-03 prefer
55 restrict -4 default kod notrap nomodify nopeer noquery limited
56 restrict -6 default kod notrap nomodify nopeer noquery limited
57 restrict 127.0.0.1
58 restrict ::1
59 restrict source notrap nomodify noquery
60 EOF
61 systemctl start ntp
62 systemctl status ntp
63 ntpq -p
64 cat /etc/ntp.conf
65 systemctl restart ntp
66 systemctl status ntp
67 ntpq -p
68 apt install ntpstat
69 apt update
70 sudo gpg --keyserver keyserver.ubuntu.com --recv 327574EE02A818DD
71 sudo gpg --export --armor 327574EE02A818DD | sudo apt-key add -
72 apt update
73 ping www.baidu.com
74 ping 114.114.114.114
75 ip a s
76 apt -h
77 apt-get
78 apt list
79 cd
80 ll
81 cd /home/dataxxa/
82 ll
83 scp jdk-8u141-linux-x64.tar.gz mysql-connector-java-5.1.46.tar.gz cdh-02:~
84 ll
85 tar -C /usr/local/ -xf jdk-8u141-linux-x64.tar.gz
86 ln -sv /usr/local/jdk1.8.0_141 /usr/local/jdk
87 cat >> ~/.bashrc <<EOF
88 export JAVA_HOME=/usr/local/jdk
89 export JRE_HOME=\${JAVA_HOME}/jre
90 export CLASSPATH=.:\${JAVA_HOME}/lib/tools.jar:\${JRE_HOME}/lib/dt.jar
91 export PATH=\${JAVA_HOME}/bin:\${JAVA_HOME}/jre/bin:\$PATH
92 EOF
93 source .bashrc
94 java -version
95 source ~/.bashrc
96 java -version
97 sudo apt search mysql
98 sudo apt install mysql-community-server
99 history
100 cat /etc/mysql/my.cnf
101 cat /etc/mysql/mysql.cnf
102 systemctl status mysql-server
103 systemctl status mysql
104 ss -tanlp
105 systemctl stop mysql
106 vim /lib/systemd/system/mysql.service
107 apt install vim
108 vim /lib/systemd/system/mysql.service
109 systemctl restart mysql
110 systemctl daemon-reload
111 systemctl restart mysql
112 vim /lib/systemd/system/mysql.service
113 ss -tnalp
114 mysql -uroot -p123123
115 date
116 time
117 date
118 sudo apt-get install cloudera-manager-daemons cloudera-manager-server
119 sudo apt-get install cloudera-manager-daemons cloudera-manager-agent
120 cd /home/
121 ll
122 cd dataxxa/
123 ll
124 tar xvf mysql-connector-java-5.1.46.tar.gz
125 cp /home/dataxxa/mysql-connector-java-5.1.48/mysql-connector-java-5.1.48* /usr/share/cmf/lib
126 cp /home/dataxxa/mysql-connector-java-5.1.46/mysql-connector-java-5.1.4* /usr/share/cmf/lib
127 scp /home/dataxxa/mysql-connector-java-5.1.46/mysql-connector-java-5.1.4* cdh-02:/usr/share/cmf/lib
128 /usr/share/cmf/schema/scm_prepare_database.sh mysql -hcdh-01 -uroot -p123123 --scm-host cdh-01 scm scm
129 mysql -uroot -p
130 /usr/share/cmf/schema/scm_prepare_database.sh mysql -hcdh-01 -uroot -p123123 --scm-host cdh-01 scm 123456
131 history
132 systemctl start cloudera-scm-server
133 systemctl status cloudera-scm-server.service
134 cd /var/log/cloudera-
135 cd /var/log/cloudera-scm-server/
136 ll
137 cat cloudera-scm-server.out
138 echo $JAVA_HOME
139 systemctl status cloudera-scm-server.service
140 cat /etc/init.d/cloudera-scm-server
141 vim /etc/init.d/cloudera-scm-server
142 cat /etc/passwd
143 ll
144 cd
145 cat ~/.bashrc
146 vim /etc/profile
147 . /etc/profile
148 ss -tanlp
149 history
150 systemctl start cloudera-scm-server
151 cat /var/log/cloudera-
152 cat /var/log/cloudera-scm-
153 cat /var/log/cloudera-scm-server/
154 cd /var/log/cloudera-scm-server/
155 ll
156 cat cloudera-scm-server.out
157 find / -name cloudera-config.sh
158 cat /usr/lib/cmf/service/common/cloudera-config.sh
159 mkdir /usr/java
160 cd /usr/java
161 ll
162 cd jdk1.8
163 pwd
164 ll
165 cd
166 systemctl start cloudera-scm-server
167 ss -tanlp
168 cd /var/log/cloudera-
169 cd /var/log/cloudera-scm-server/
170 ll
171 tailf cloudera-scm-server.log
172 tailf cloudera-scm-server.out
173 systemctl start cloudera-scm-agent
174 systemctl start cloudera-scm-server
175 systemctl status cloudera-scm-server
176 ss -tanlp
177 systemctl status cloudera-scm-agent
178 ps -ef |grep agent
179 ss -tanlp
180 ip a s
181 history
182 cd /usr/java
183 ll
184 ss -tanlp
185 mv /usr/bin/host /usr/bin/host.bak
186 cd
187 service network restart
188 systemctl restart network
189 history
190 cat .ssh/id_rsa
191 systemctl restart cloudera-scm-server
192 ss -tanlp
193 history
194 /usr/bin/host
195 cat /etc/host
196 cat /etc/hosts
197 vim /etc/hosts
198 ll
199 ufw disable
200 reboot
201 ss -tanlp
202 cat /usr/bin/host
203 which host
204 mysql -uroot -p123123
205 history
206 cd /opt/cloudera/parcels/CDH-5.13.3-1.cdh5.13.3.p0.2/lib/hive/lib
207 history
208 scp /home/dataxxa/mysql-connector-java-5.1.46/mysql-connector-java-5.1.4* cdh-02:/opt/cloudera/parcels/CDH-5.13.3-1.cdh5.13.3.p0.2/lib/hive/lib
209 scp /home/dataxxa/mysql-connector-java-5.1.46/mysql-connector-java-5.1.4* cdh-02:/opt/cloudera/parcels/CDH-5.13.3-1.cdh5.13.3.p0.2/lib/oozie/lib
210 scp /home/dataxxa/mysql-connector-java-5.1.46/mysql-connector-java-5.1.4* cdh-02:/opt/cloudera/parcels/CDH-5.13.3-1.cdh5.13.3.p0.2/lib/hue/lib
211 scp /home/dataxxa/mysql-connector-java-5.1.46/mysql-connector-java-5.1.4* cdh-02:/var/lib/oozie
212 history
213 ip a s
214 ss -tanlp
215 apt search libwrap
216 cd /etc/apt/sources.list.d/
217 ll
218 cd ../
219 ll
220 cat sources.list
221 apt update
222 cd
223 vim /etc/mysql/mysql.conf.d/mysqld.cnf
224 history
浙公网安备 33010602011771号