ansible 部署hadoop
规划
ansible 节点
ansible controller
镜像rhel 8.2
镜像ansible
hadoop 集群
master
slave1
slave2
镜像centos 1810
0.初始化
hadoop集群配网络修改主机名
10.104.44.25 master
10.104.44.49 slave1
10.104.44.36 slave2
一.配置ansible，配置集群hadoop用户免密
1.配置ansible controller etc/hosts
[root@localhost ansible]# cat /etc/hosts
127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
::1 localhost localhost.localdomain localhost6 localhost6.localdomain6
10.104.44.25 master
10.104.44.49 slave1
10.104.44.36 slave2
2.安装ansible并配置hosts
[root@localhost mnt]# ansible --version
ansible 2.9.11
config file = /etc/ansible/ansible.cfg
configured module search path = ['/root/.ansible/plugins/modules', '/usr/share/ansible/plugins/modules']
ansible python module location = /usr/lib/python3.6/site-packages/ansible
executable location = /usr/bin/ansible
python version = 3.6.8 (default, Dec 5 2019, 15:45:45) [GCC 8.3.1 20191121 (Red Hat 8.3.1-5)]
[root@localhost ansible]# cat hosts
master
slave1
slave2
3.集群添加hadoop用户
[root@localhost ansible]# cat user_hadoopcreate
- hosts: all
vars:
password: '123'
tasks:
- name: create
user:
name: hadoop
uid: 1200
password: "{{ password | password_hash('sha512') }}"
[root@localhost ansible]#
[root@localhost ansible]# ansible-playbook user_hadoopcreate -k
PLAY RECAP *********************************************************************
master : ok=2 changed=1 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0
slave1 : ok=2 changed=1 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0
slave2 : ok=2 changed=1 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0
4.配置hadoop用户提权
ansible all -m copy -a "dest=/etc/sudoers.d/ansible content='hadoop ALL=\(ALL\) NOPASSWD:ALL'" -k
5.使root用户免密登录集群，使用Hadoop用户
ansible controller创建密匙
[root@localhost ansible]# ssh-keygen
ansible免密操控hadoop用户
[root@localhost .ssh]# ansible all -m copy -a 'src=/root/.ssh/id_rsa.pub dest=/home/hadoop/.ssh/authorized_keys owner=hadoop group=hadoop' -k
6.配置ansible 配置文件
host_key_checking = False #不检查公钥
允许提权
become=True
become_method=sudo
become_user=root
become_ask_pass=False
远程用户为hadoop
remote_user = hadoop
sed -i 's/remote_user = root/remote_user = hadoop/' ansible.cfg.bak
sed -i 's/^#become/become/g' ansible.cfg.bak
sed -i 's/^# host_key_checking/host_key_checking/g' ansible.cfg.bak
7.同步hosts
[root@localhost ansible]# ansible all -m copy -a 'src=/etc/hosts dest=/etc/hosts'
8.集群之间的配置hadoop用户免密登录
执行脚本前需要controller下载expect
yum reinstall --downloadonly --downloaddir=/etc/ansible/ -y libtcl*
yum reinstall --downloadonly --downloaddir=/etc/ansible/ -y expect
编写脚本，生产密钥，并发送公钥给其他集群
[root@localhost ansible]# cat ask.sh
expect <<EOF
set timeout 10
spawn ssh-keygen
expect {
"save the key" { send "\n"; exp_continue }
"passphrase" { send "\n"; exp_continue }
"passphrase again" { send "\n"; exp_continue }
}
spawn ssh-copy-id hadoop@slave1
expect {
"yes/no" { send "yes\n";exp_continue }
"password" { send "123\n";exp_continue }
}
spawn ssh-copy-id hadoop@slave2
expect {
"yes/no" { send "yes\n";exp_continue }
"password" { send "123\n";exp_continue }
}
spawn ssh-copy-id hadoop@master
expect {
"yes/no" { send "yes\n";exp_continue }
"password" { send "123\n";exp_continue }
}
EOF
编写playbook，hadoop节点安装expect软件，并执行免密脚本
[root@localhost ansible]# cat key_operation.yml
- hosts: slave2
remote_user: hadoop
become: false
vars:
package: /etc/ansible/tcl-8.6.8-2.el8.x86_64.rpm
package2: /etc/ansible/expect-5.45.4-5.el8.x86_64.rpm
tasks:
- name: deliver
copy:
src: "{{ item }}"
dest: /home/hadoop
loop:
- "{{ package }}"
- "{{ package2 }}"
- name: rpm
yum:
name:
- /home/hadoop/tcl-8.6.8-2.el8.x86_64.rpm
- /home/hadoop/expect-5.45.4-5.el8.x86_64.rpm
state: present
- name: try script
script: /etc/ansible/ask.sh
9.将以上操作整理成first脚本，可以跑脚本部署以上操作
[root@localhost first]# cat first.sh
#!/bin/bash
expect << EOF
set timeout 10
spawn ansible-playbook user_hadoopcreate.yml -k
expect {
"password" { send "123\n";exp_continue }
}
spawn ansible all -m file -a "path=/home/hadoop/.ssh state=directory owner=hadoop group=hadoop" -k
expect {
"password" { send "123\n";exp_continue }
}
spawn ansible all -m copy -a "dest=/etc/sudoers.d/ansible content='hadoop ALL=\(ALL\) NOPASSWD:ALL'" -k
expect {
"password" { send "123\n";exp_continue }
}
spawn ansible all -m copy -a "src=/root/.ssh/id_rsa.pub dest=/home/hadoop/.ssh/authorized_keys owner=hadoop group=hadoop" -k
expect {
"password" { send "123\n";exp_continue }
}
EOF
ansible all -m copy -a 'src=/etc/hosts dest=/etc/hosts'
# change configuration
sed -i 's/remote_user = root/remote_user = hadoop/' ansible.cfg
sed -i 's/^#become/become/g' ansible.cfg
sed -i 's/^# host_key_checking/host_key_checking/g' ansible.cfg
yum reinstall --downloadonly --downloaddir=/etc/ansible/ -y libtcl*
yum reinstall --downloadonly --downloaddir=/etc/ansible/ -y expect
ansible-playbook key_operation.yml
二.配置hadoop基础配置
上传jdk与hadoop到ansible节点
四个配置文件内容
[root@localhost second]# cat hdfs.config
<configuration>
<property>
<name>dfs.namenode.name.dir</name>
<value>file:/opt/hadoop/dfs/name</value>
</property>
<property>
<name>dfs.datanode.data.dir</name>
<value>file:/opt/hadoop/dfs/data</value>
</property>
<property>
<name>dfs.replication</name>
<value>2</value>
</property>
</configuration>
[root@localhost second]# cat yarn.config
<configuration>
<property>
<name>arn.resourcemanager.address</name>
<value>master:8032</value>
</property>
<property>
<name>yarn.resourcemanager.scheduler.address</name>
<value>master:8030</value>
</property>
<property>
<name>yarn.resourcemanager.webapp.address</name>
<value>master:8088</value>
</property>
<property>
<name>yarn.resourcemanager.resource-tracker.address</name>
<value>master:8031</value>
</property>
<property>
<name>yarn.resourcemanager.admin.address</name>
<value>master:8033</value>
</property>
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
<property>
<name>yarn.nodemanager.aux-services.mapreduce_shuffle.class</name>
<value>org.apache.hadoop.mapred.ShuffleHandler</value>
</property>
</configuration>
[root@localhost second]# cat core.config
<configuration>
<property>
<name>fs.defaultFS</name>
<value>hdfs://master:9000</value>
</property>
<property>
<name>io.file.buffer.size</name>
<value>131072</value>
</property>
<property>
<name>hadoop.tmp.dir</name>
<value>file:/opt/hadoop/tmp</value>
</property>
</configuration>
[root@localhost second]# cat mapred.config
<configuration>
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>
<property>
<name>mapreduce.jobhistory.address</name>
<value>master:10020</value>
</property>
<property>
<name>mapreduce.jobhistory.webapp.address</name>
<value>master:19888</value>
</property>
</configuration>
ansible-playbook 配置
cat clucig.yml
#关闭防火墙与selinux
- hosts: all
tasks:
- name: Disable SELinux
selinux:
state: disabled
- name: stop firewalld
systemd:
state: stopped
name: firewalld
enabled: no
ignore_errors: yes
#将包解压到目标主机
- hosts: all
tasks:
- name: tar
unarchive:
src: "/opt/{{ item }}"
dest: /opt/
loop:
- jdk-8u152-linux-x64.tar.gz
- hadoop-2.7.1.tar.gz
- name: mv
shell: mv /opt/jdk1.8.0_152 /opt/jdk
- name: mv2
shell: mv /opt/hadoop-2.7.1 /opt/hadoop
#更改环境变量
- hosts: all
tasks:
- name: copy
copy:
dest: /etc/profile.d/hadoop.sh
content: "export JAVA_HOME=/opt/jdk
export HADOOP_HOME=/opt/hadoop
export PATH=${JAVA_HOME}/bin:${HADOOP_HOME}/bin:${HADOOP_HOME}/sbin:$PATH\n"
- name: echo
shell: echo 'export JAVA_HOME=/opt/jdk' >> /opt/hadoop/etc/hadoop/hadoop-env.sh
#修改配置文件
#1
- name: delete something
shell: sed -i '/<configuration>/,/<\/configuration>/d' /opt/hadoop/etc/hadoop/hdfs-site.xml
- name: copy
copy:
src: hdfs.config
dest: /opt/
- name: add configuration
shell: cat /opt/hdfs.config >> /opt/hadoop/etc/hadoop/hdfs-site.xml
- name: shell operate
shell: mkdir -p /opt/hadoop/dfs/{name,data}
#2
- name: delete something
shell: sed -i '/<configuration>/,/<\/configuration>/d' /opt/hadoop/etc/hadoop/core-site.xml
- name: copy
copy:
src: core.config
dest: /opt/
- name: add configuration
shell: cat /opt/core.config >> /opt/hadoop/etc/hadoop/core-site.xml
- name: shell operate2
shell: mkdir -p /opt/hadoop/tmp
#3
- name: copy template
shell: cp /opt/hadoop/etc/hadoop/mapred-site.xml.template /opt/hadoop/etc/hadoop/mapred-site.xml
- name: delete something
shell: sed -i '/<configuration>/,/<\/configuration>/d' /opt/hadoop/etc/hadoop/mapred-site.xml
- name: copy
copy:
src: mapred.config
dest: /opt/
- name: add configuration
shell: cat /opt/mapred.config >> /opt/hadoop/etc/hadoop/mapred-site.xml
#4
- name: delete something
shell: sed -i '/<configuration>/,/<\/configuration>/d' /opt/hadoop/etc/hadoop/yarn-site.xml
- name: copy
copy:
src: yarn.config
dest: /opt/
- name: add configuration
shell: cat /opt/yarn.config >> /opt/hadoop/etc/hadoop/yarn-site.xml
#configuration finish
- name: master or slave
copy:
dest: /opt/hadoop/etc/hadoop/masters
content: "10.104.44.25\n"
- name: master or slave
copy:
dest: /opt/hadoop/etc/hadoop/slaves
content: "10.104.44.36\n10.104.44.49\n"
- name: chmod
shell: chown -R hadoop.hadoop /opt/
剧本可以完成hadoop第四章配置文件
三.启动hadoop
windwos配置域名解析
c:\windows\system32\drivers\etc\hosts
手动启动hadoop
start-all.sh
hdfs namenode -format
start-dfs.sh
上传文件
hdfs dfs -put rhel-8.2-x86_64-dvd.iso
