python中fab模块的应用实例
import getpass import os from collections import OrderedDict from fabric.api import * def helloworld(): print "helloworld!" def hello(name , age): print "hello ! I am %s , my age=%s" % (name , age) def ls(): local("ls /usr/local") output = local('echo hello' , capture=True) print output def setRoles(): env.roledefs = { 'hadoop1' : ['root@192.168.144.152'] , 'hadoop2' : ['root@192.168.144.165'] , 'hadoop3' : ['root@192.168.144.154'] , 'hadoop4' : ['root@192.168.144.155'] } env.passwords = { 'root@192.168.144.152:22' : 'A8285310' , 'root@192.168.144.165:22' : 'A8285310' , 'root@192.168.144.154:22' : 'A2885310' , 'root@192.168.144.155:22' : 'A8285310' } @roles('hadoop2') def setBaseEnv(): hadoop1 = str(env.roledefs["hadoop1"]).split('@' , 1)[1] hadoop2 = str(env.roledefs["hadoop2"]).split('@' , 1)[1] hadoop3 = str(env.roledefs["hadoop3"]).split('@' , 1)[1] hadoop4 = str(env.roledefs["hadoop4"]).split('@' , 1)[1] #set etc hosts with cd('/etc'): run('echo -e "{0} hadoop1.cbas\n{1} hadoop2.cbas\n{2} hadoop3.cbas\n{3} hadoop4.cbas" >> hosts'.format(hadoop1.split("'" ,1)[0] , hadoop2.split("'" ,1)[0] , hadoop3.split("'" ,1)[0] , hadoop4.split("'" ,1)[0])) run("cat hosts") run("echo -----------------------------------------------------------------------------------------------------------") #set language run("echo LANG=en_US.UTF-8 > /etc/sysconfig/i18n") run("source /etc/sysconfig/i18n") run("echo $LANG") run("echo -----------------------------------------------------------------------------------------------------------") #close the iptables and selinux forever run("chkconfig iptables off") run("chkconfig --list iptables") run("sed -i 's/enforcing/disabled/g' /etc/selinux/config") run("cat /etc/selinux/config") run("echo -----------------------------------------------------------------------------------------------------------") #set localtime run("\cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime") run("cat /etc/localtime") run("echo -----------------------------------------------------------------------------------------------------------") #set file limit run('echo -e "* soft nofile 65536\n* hard nofile 65536" >> /etc/security/limits.conf') run("cat /etc/security/limits.conf") run("sed -i '/1024/s/^/#/' /etc/security/limits.d/90-nproc.conf") run("echo -----------------------------------------------------------------------------------------------------------") #set ntpd and httpd start with the system run("chkconfig ntpd on") run("chkconfig httpd on") run("chkconfig --list ntpd") run("chkconfig --list httpd") run("echo -----------------------------------------------------------------------------------------------------------") #mkdir run("mkdir -p /data/apps") run('mkdir -p /software') run("echo -----------------------------------------------------------------------------------------------------------") #move centos repo run("mv /etc/yum.repos.d/*.repo /etc") run("echo -----------------------------------------------------------------------------------------------------------") #reboot run("reboot") @roles('hadoop1') def installBasePak(): #check selinux run("/usr/sbin/sestatus -v") #close THP run("echo never > /sys/kernel/mm/redhat_transparent_hugepage/defrag") run("echo never > /sys/kernel/mm/redhat_transparent_hugepage/enabled") run("echo never > /sys/kernel/mm/transparent_hugepage/defrag") run("echo never > /sys/kernel/mm/transparent_hugepage/enabled") run("echo -----------------------------------------------------------------------------------------------------------") with cd('/software'): run('unzip basePackage.zip') #install base package run("rpm -Uvh nc-1.84-24.el6.x86_64.rpm") run("rpm -Uvh snappy-1.1.0-1.el6.x86_64.rpm") run("rpm -Uvh snappy-devel-1.1.0-1.el6.x86_64.rpm") run("rpm -Uvh mpfr-2.4.1-6.el6.x86_64.rpm") run("rpm -Uvh ppl-0.10.2-11.el6.x86_64.rpm") run("rpm -Uvh cloog-ppl-0.15.7-1.2.el6.x86_64.rpm") run("rpm -Uvh cpp-4.4.7-18.el6.x86_64.rpm") run("rpm -Uvh gcc-4.4.7-18.el6.x86_64.rpm") run("rpm -Uvh libstdc++-devel-4.4.7-18.el6.x86_64.rpm") run("rpm -Uvh gcc-c++-4.4.7-18.el6.x86_64.rpm") run("rpm -Uvh zlib-devel-1.2.3-29.el6.x86_64.rpm") run("rpm -Uvh python-devel-2.6.6-66.el6_8.x86_64.rpm") run("rpm -Uvh perl-5.10.1-144.el6.x86_64.rpm") run("rpm -Uvh perl-DBI-1.609-4.el6.x86_64.rpm") run("rpm -Uvh perl-DBD-MySQL-4.013-3.el6.x86_64.rpm") run("rpm -Uvh mysql-5.1.73-8.el6_8.x86_64.rpm") run("rpm -Uvh mysql-server-5.1.73-8.el6_8.x86_64.rpm") run("echo -----------------------------------------------------------------------------------------------------------") @roles('hadoop1') def sendPak(): with cd('/software'): run('scp basePackage.zip mysql.zip redis.zip root@hadoop2.cbas:/software') run('scp basePackage.zip redis.zip nginx.zip root@hadoop3.cbas:/software') run('scp basePackage.zip redis.zip nginx.zip ruby.zip ftp.zip hue.tar root@hadoop4.cbas:/software') @roles('hadoop2') def test(): with cd('/software/ruby-2.4.1/ext/openssl'): run('ruby extconf.rb --with-openssl-include=/software/openssl1f/include --with-openssl-lib=/software/openssl1f/lib') run('rm -rf Makefile') run('cp /software/Makefile_openssl /software/ruby-2.4.1/ext/openssl/Makefile') run('make') run('make install') with cd('/software'): run('gem install redis-4.0.1.gem') run('gem list') @roles('hadoop1') def buildHdpMirrorCenter(): with cd('/software'): run('tar -xf AMBARI-2.4.1.0.tar') run('tar -xf HDP-2.5.0.0.tar') run('tar -xf HDP-UTILS-1.1.0.21.tar') run('\cp AMBARI-2.4.1.0/centos6/2.4.1.0-22/ambari.repo /etc/yum.repos.d') run('\cp HDP/centos6/hdp.repo /etc/yum.repos.d') run('ln -s /software/AMBARI-2.4.1.0 /var/www/html/AMBARI-2.4.1.0') run('ln -s /software/HDP /var/www/html/HDP') run('ln -s /software/HDP-UTILS-1.1.0.21 /var/www/html/HDP-UTILS-1.1.0.21') run('service httpd restart') run('yum clean all') run('yum makecache') run('yum repolist') run('curl http://hadoop1.cbas/HDP') run('curl http://hadoop1.cbas/HDP-UTILS-1.1.0.21') run('curl http://hadoop1.cbas/AMBARI-2.4.1.0') @roles('hadoop1') def deployAmbariServer(): run('yum install -y ambari-server') @roles('hadoop4') def deployHue(): with cd('/software'): run('mkdir /hue') run('mv hue.tar /hue') run('groupadd hue') run('useradd -g hue hue') with cd('/hue'): run('tar -xf hue.tar') run('chown hue:hue /hue -R') @roles('hadoop1' , 'hadoop2') def deployJDK(): with cd('/software'): run('tar -xf jdk-8u144-linux-x64.tar.gz') run('mv jdk1.8.0_144 jdk1.8') with cd('/etc'): run('echo -e "JAVA_HOME=/software/jdk1.8\nCLASSPATH=.:/software/jdk1.8/lib:/software/jdk1.8/jre/lib\nPATH=/software/jdk1.8/bin:$PATH:/software/jdk1.8/jre/bin\nexport JAVA_HOME CLASSPATH PATH" >> profile') run('source profile') run('java -version') @roles('hadoop1' , 'hadoop2') def deployNginx(): with cd('/software'): run('service httpd stop') run('chkconfig httpd off') run('unzip nginx.zip') run('tar -xf nginx-1.8.1.tar.gz') run('tar -xf openssl-1.0.2j.tar.gz') run('tar -xf pcre-8.21.tar.gz') run('tar -xf zlib-1.2.7.tar.gz') run('unzip nginx-http-concat-master.zip') run('unzip nginx_upstream_check_module-master.zip') with cd('/software/nginx-1.8.1'): run('./configure --prefix=/software/nginx --with-pcre=/software/pcre-8.21 --with-zlib=/software/zlib-1.2.7 --add-module=/software/nginx-http-concat-master --add-module=/software/nginx_upstream_check_module-master --with-http_ssl_module --with-openssl=/software/openssl-1.0.2j --with-http_realip_module') run('make') run('make install') run('/software/nginx/sbin/nginx -t') run('/software/nginx/sbin/nginx') @roles('hadoop1' , 'hadoop2') def deployRedis(): with cd('/software'): run('unzip redis.zip') run('tar -xf redis-3.2.8.tar.gz') with cd('/software/redis-3.2.8'): run('make') run('make install PREFIX=/software/redis') run('mkdir /software/redis-cluster') run('mkdir /software/redis-cluster/redis6371') run('mkdir /software/redis-cluster/redis6372') run('cp /software/redis/bin/* /software/redis-cluster/redis6371') run('cp /software/redis/bin/* /software/redis-cluster/redis6372') run('cp /software/redis6371.conf /software/redis-cluster/redis6371') run('cp /software/redis6372.conf /software/redis-cluster/redis6372') run('cp /software/redis-3.2.8/src/redis-trib.rb /software/redis-cluster') run('/software/redis-cluster/redis6371/redis-server /software/redis-cluster/redis6371/redis6371.conf') run('/software/redis-cluster/redis6372/redis-server /software/redis-cluster/redis6372/redis6372.conf') run('netstat -ntlp | grep redis') @roles('hadoop2') def deployRuby(): with cd('/software'): run('mkdir ruby') run('unzip ruby.zip') run('tar -xf ruby-2.4.1.tar.gz') run('tar -xf openssl-1.0.1f.tar.gz') with cd('/software/ruby-2.4.1'): run('./configure --prefix=/software/ruby') run('make') run('make install') run('ln -s /software/ruby/bin/ruby /usr/bin/ruby') run('ln -s /software/ruby/bin/gem /usr/bin/gem') with cd('/software/openssl-1.0.1f'): run('./config -fPIC --prefix=/software/openssl1f enable-shared') run('./config -t') run('make depend') run('make install') with cd('/software/ruby-2.4.1/ext/zlib'): run('ruby extconf.rb') run("sed -i '16a top_srcdir = /software/ruby-2.4.1' Makefile") run('make') run('make install') with cd('/software/ruby-2.4.1/ext/openssl'): run('ruby extconf.rb --with-openssl-include=/software/openssl1f/include --with-openssl-lib=/software/openssl1f/lib') run("sed -i '16a top_srcdir = /software/ruby-2.4.1' Makefile") run('make') run('make install') with cd('/software'): run('gem install redis-4.0.1.gem') run('gem list') with cd('/software/redis-cluster'): run('./redis-trib.rb create --replicas 1 hadoop2.cbas:6371 hadoop2.cbas:6372 hadoop3.cbas:6371 hadoop3.cbas:6372 hadoop4.cbas:6371 hadoop4.cbas:6372') @roles('hadoop4') def deployFtp(): with cd('/software'): run('unzip ftp.zip') run('rpm -ivh vsftpd-2.2.2-24.el6.x86_64.rpm') run('rpm -ivh easy-rsa-3.0.3-1.el6.noarch.rpm') run('mkdir -p /ftp') run('useradd -M -D /ftp -s /sbin/nologin ftptest') with cd('/etc/vsftpd'): run('rm -rf vsftpd.conf') run('cp /software/vsftpd.conf ./') @roles('hadoop1') def deployPgsql(): with cd('/software'): run('unzip pgsql.zip') run('rpm -ivh postgresql-libs-8.4.20-7.el6.x86_64.rpm') run('rpm -ivh postgresql-8.4.20-7.el6.x86_64.rpm') run('rpm -ivh postgresql-server-8.4.20-7.el6.x86_64.rpm') def deployMysql(masterOrslave , password): global MYSQLCMD MYSQLCMD = "grant replication slave on *.* to 'root'@'hadoop2.cbas' identified by '%s'" % (password) if (masterOrslave == 'master' or masterOrslave == 'slave') and not password is None: with cd('/software'): run('rpm -ivh mysql-connector-java-5.1.37-1.noarch.rpm') run('unzip mysql.zip') run('tar -xf mysql56.tar') run('useradd mysql') run('mkdir -p /data/mysql') run('rm -rf /etc/my.cnf') run('rm -rf /etc/init.d/mysqld') run('rm -rf /usr/bin/mysql') run('/software/mysql/scripts/mysql_install_db --basedir=/software/mysql --datadir=/data/mysql') run('cp /software/mysql/support-files/mysql.server /etc/rc.d/init.d/mysqld') run('chkconfig --add mysqld') run('chkconfig mysqld on') with cd('/data'): run('touch mysqld.log') run('chown mysql:mysql /data -R') if masterOrslave == 'master': with cd('/software/mysql'): run('echo -e "basedir = /software/mysql\ndatadir = /data/mysql\nport = 3306\nsocket = /tmp/mysql.sock\npid-file=/data/mysql.pid\nbind-address=0.0.0.0\nmax_connections=3000\ndefault-storage-engine=innoDB\nbinlog_format=mixed\nserver-id=1\nlog-bin=master-bin\nlog-bin-index=master-bin.index\n[mysqld_safe]\nlog-error = /data/mysqld.log" >> my.cnf') with cd('/software/mysql/bin'): run('ln -s /software/mysql/bin/mysql /usr/bin/mysql') run('service mysqld start') run("./mysqladmin -u root password '%s'" % (password)) run(" ./mysql -uroot -p{0} -e {1}{2};{3} ".format(password , '"' , MYSQLCMD , '"')) run('service mysqld restart') elif masterOrslave == 'slave': with cd('/software/mysql'): run('echo -e "basedir = /software/mysql\ndatadir = /data/mysql\nport = 3306\nsocket = /tmp/mysql.sock\npid-file=/data/mysql.pid\nbind-address=0.0.0.0\nmax_connections=3000\ndefault-storage-engine=innoDB\nbinlog_format=mixed\nserver-id=2\nrelay-log=slave-relay-bin\nrelay-log-index=slave-relay-bin.index\n[mysqld_safe]\nlog-error = /data/mysqld.log" >> my.cnf') with cd('/software/mysql/bin'): run('ln -s /software/mysql/bin/mysql /usr/bin/mysql') run('service mysqld start') run("./mysqladmin -u root password '%s'" % (password)) else: run('echo usage:fab -H root@IP(the host which you want to deploy mysql master or slave) setRoles deployMysql:masterOrslave=master|slave , password=Mysqlpassword