安装环境

最近使用云服务器的时候，总是需要安装环境，在此记录一些常用的命令

常用命令

查看系统版本：
getconf LONG_BIT

断开连接：
vim /etc/ssh/sshd_config

ClientAliveInterval 60
ClientAliveCountMax 600

systemctl restart sshd.service

创建软件目录

mkdir /opt/hadoop
mkdir /opt/hive
mkdir /opt/pkg
mkdir /opt/mysql

JAVA

mkdir /opt/java
cd /opt/java
wget https://mirrors.tuna.tsinghua.edu.cn/Adoptium/8/jdk/x64/linux/OpenJDK8U-jdk_x64_linux_hotspot_8u372b07.tar.gz
mv OpenJDK8U-jdk_x64_linux_hotspot_8u372b07.tar.gz jdk.tar.gz
tar -xvf jdk.tar.gz
mv jdk8u372-b07 jdk
mv jdk.tar.gz /opt/pkg/
环境变量：
vim /root/.bashrc

alias l='ls -l'

export JAVA_HOME=/opt/java/jdk
export JRE_HOME=/opt/java/jdk/jre
export CLASSPATH=.:$JAVA_HOME/lib:$JRE_HOME/lib:$CLASSPATH
export PATH=$JAVA_HOME/bin:$JRE_HOME/bin:$JAVA_HOME:$PATH

source /root/.bashrc

java

HADOOP

cd /opt/hadoop/
wget https://mirrors.tuna.tsinghua.edu.cn/apache/hadoop/common/hadoop-3.3.2/hadoop-3.3.2.tar.gz
tar -xvf hadoop-3.3.2.tar.gz
mv hadoop-3.3.2 hadoop
mv hadoop-3.3.2.tar.gz hadoop.tar.gz
mv hadoop.tar.gz /opt/pkg/
环境变量：
vim /root/.bashrc

export HADOOP_HOME=/opt/hadoop/hadoop/
export PATH=${PATH}:${HADOOP_HOME}/bin

修改配置：
vim /opt/hadoop/hadoop//etc/hadoop/hadoop-env.sh
export JAVA_HOME=/opt/java/jdk

source /root/.bashrc

hadoop version

MYSQL

mkdir /opt/pkg
mkdir /opt/mysql
cd /opt/mysql
wget https://mirrors.tuna.tsinghua.edu.cn/mysql/downloads/MySQL-8.0/mysql-8.0.27-el7-x86_64.tar.gz
tar -xvf mysql-8.0.27-el7-x86_64.tar.gz
mv mysql-8.0.27-el7-x86_64.tar.gz mysql.tar.gz
mv mysql.tar.gz /opt/pkg/
mv mysql-8.0.27-el7-x86_64 mysql

groupadd mysql && useradd -r -g mysql mysql
mkdir -p /opt/mysql/mysql/data/mysql
chown mysql:mysql -R /opt/mysql

vim /etc/my.cnf


[mysqld]
bind-address=0.0.0.0
port=3306
user=mysql
basedir=/opt/mysql/mysql
datadir=/opt/mysql/mysql/data/mysql
socket=/tmp/mysql.sock
log-error=/opt/mysql/mysql/data/mysql/mysql.err
pid-file=/opt/mysql/mysql/data/mysql/mysql.pid
character_set_server=utf8mb4
symbolic-links=0
explicit_defaults_for_timestamp=true


yum install -y libaio
cd /opt/mysql/mysql/bin
./mysqld --defaults-file=/etc/my.cnf --basedir=/opt/mysql/mysql --datadir=/opt/mysql/mysql/data/mysql --user=mysql --initialize

密码：
cat /opt/mysql/mysql/data/mysql/mysql.err

root zEJ<S3qZflE,

cp /opt/mysql/mysql/support-files/mysql.server /etc/init.d/mysql
service mysql start

cd /opt/mysql/mysql/bin
./mysql -u root -p'123456'
use mysql;
alter user 'root'@'localhost' identified by '123456';
update user set host = '%' where user = 'root';
--显示端口
show global variables like 'port';



create database test;
use test;
CREATE TABLE IF NOT EXISTS student (
    id INT(20) NOT NULL AUTO_INCREMENT COMMENT '学号',
    name VARCHAR(30) NOT NULL COMMENT '姓名',
    score INT(2) NOT NULL  COMMENT '性别',
    PRIMARY KEY(id)
)ENGINE=INNODB DEFAULT CHARSET=utf8;
insert into student values(1,'zhangsan',1);
insert into student values(2,'lisi',2);
insert into student values(3,'wangwu',3);
insert into student values(4,'sunliu',4);
insert into student values(5,'liuqi',5);


vim /root/.bashrc

export MYSQL_HOME=/opt/mysql/mysql
export PATH=$PATH:$MYSQL_HOME/bin


mysql常用命令：
ALTER TABLE table_name change column diff dif float(10,4) ;  --修改列名


set global local_infile=1;
show global variables like 'local_infile';
load data local infile '/root/data/test.sql' ---mysql导数命令
into table test             # 表名
character set utf8                 # 编码
fields terminated by ','           # 分隔符
lines terminated by '\r\n'         # 换行符，windows下是\r\n
ignore 1 lines;                    # 忽略第一行，因为表头已建好


select trx_mysql_thread_id from information_schema.innodb_trx ; --mysql事务

HIVE

cd /opt/hive
wget https://mirrors.tuna.tsinghua.edu.cn/apache/hive/hive-3.1.3/apache-hive-3.1.3-bin.tar.gz
tar -xvf apache-hive-3.1.3-bin.tar.gz
mv apache-hive-3.1.3-bin hive
mv apache-hive-3.1.3-bin.tar.gz hive.tar.gz 
mv hive.tar.gz /opt/pkg/

环境变量：
vim /root/.bashrc

export HIVE_HOME=/opt/hive/hive
export PATH=$PATH:$HIVE_HOME/bin

source /root/.bashrc
hive --version

cd /opt/hive/hive/conf
cp hive-env.sh.template hive-env.sh

vim hive-env.sh

export HADOOP_HOME=/opt/hadoop/hadoop/
export HIVE_CONF_DIR=/opt/hive/hive/conf

vim hive-site.xml

<configuration>
  <property>
     <!-- 查询数据时 显示出列的名字 -->
     <name>hive.cli.print.header</name>
     <value>true</value>
  </property>
  <property>
     <!-- 在命令行中显示当前所使用的数据库 -->
     <name>hive.cli.print.current.db</name>
     <value>true</value>
  </property>
  <property>
     <!-- 默认数据仓库存储的位置，该位置为HDFS上的路径 -->
     <name>hive.metastore.warehouse.dir</name>
     <value>/opt/hive/hive/data/</value>
  </property>
  <!-- 8.x -->
  <property>
     <name>javax.jdo.option.ConnectionURL</name>
     <value>jdbc:mysql://localhost:3306/hive?createDatabaseIfNotExist=true&amp;useSSL=false&amp;serverTimezone=GMT&amp;allowPublicKeyRetrieval=true</value>
  </property>
  <!-- 8.x -->
  <property>
     <name>javax.jdo.option.ConnectionDriverName</name>
     <value>com.mysql.jdbc.Driver</value>
  </property>
  <property>
     <name>javax.jdo.option.ConnectionUserName</name>
     <value>root</value>
  </property>
  <property>
     <name>javax.jdo.option.ConnectionPassword</name>
     <value>123456</value>
  </property>
  <!-- hiveserver2服务的端口号以及绑定的主机名 -->
  <property>
     <name>hive.server2.thrift.port</name>
     <value>10000</value>
  </property>
  <property>
     <name>hive.server2.thrift.bind.host</name>
     <value>localhost</value>
  </property>
</configuration>



hadoop fs -mkdir -p /opt/hive/hive/data/
hadoop fs -chmod g+w /tmp
hadoop fs -chmod g+w /opt/hive/hive/data/
schematool -dbType mysql -initSchema


drop database hive;
create database hive;
alter database hive character set latin1;

hive --debug -hiveconf hive.root.logger=DEBUG,console

debug

create table  data (id bigint,user_id string,desc string) PARTITIONED BY (dt string) row format delimited fields terminated by '|';

insert into table data partition (dt='20220801')
select 1 as id,'1' as user_id,'1' as desc
union all
select 2 as id,'2' as user_id,'2' as desc
union all
select 3 as id,'3' as user_id,'3' as desc
union all
select 4 as id,'4' as user_id,'4' as desc
union all
select 5 as id,'5' as user_id,'5' as desc;

Spark

        mkdir /opt/spark/
        cd /opt/spark
        wget https://mirrors.tuna.tsinghua.edu.cn/apache/spark/spark-3.1.3/spark-3.1.3-bin-hadoop3.2.tgz
        tar -xvf spark-3.1.3-bin-hadoop3.2.tgz
        mv spark-3.1.3-bin-hadoop3.2 spark
        mv spark-3.1.3-bin-hadoop3.2.tgz spark.tgz
        mv spark.tgz /opt/pkg/

        cd /opt/spark/spark/conf
        cp spark-env.sh.template spark-env.sh
        vim spark-env.sh
        ---------------------------------------------
        export JAVA_HOME=/opt/java/jdk
        export JRE_HOME=/opt/java/jdk/jre
        export CLASSPATH=.:$JAVA_HOME/lib:$JRE_HOME/lib:$CLASSPATH
        export PATH=$JAVA_HOME/bin:$JRE_HOME/bin:$JAVA_HOME:$PATH
        export HADOOP_HOME=/opt/hadoop/hadoop
        export PATH=${PATH}:${HADOOP_HOME}/bin

        cp spark-defaults.conf.template spark-defaults.conf
        mkdir /opt/spark/spark/dbdata
        mkdir /opt/spark/spark/dbdata/warehouse
        vim spark-defaults.conf
        --------------------------------------------
        spark.driver.extraJavaOptions -Dderby.system.home=/opt/spark/spark/dbdata
        spark.sql.warehouse.dir=/opt/spark/spark/dbdata/warehouse

        vim /root/.bashrc
        --------------------------------------------
        export SPARK_HOME=/opt/spark/spark
        export PATH=$PATH:$SPARK_HOME/bin


        源码编译：mvn clean package -Dmaven.test.skip=true -Dmaven.clean.failOnError=false
        修改 pom.xml 里面 118行  maven.version   查看自己mvn版本 mvn -v
        服务器4G的内存不够的，需要更大的内存
        <mirror>
            <id>huaweicloud</id>
            <mirrorOf>central</mirrorOf>
            <url>https://repo.huaweicloud.com/repository/maven/</url>
       </mirror>
       <mirror>
           <id>aliyun</id>
           <mirrorOf>central</mirrorOf>
           <name>maven.aliyun.com</name>
           <url>http://maven.aliyun.com/nexus/content/groups/public/</url>
       </mirror>
       <mirror>
           <id>crespo1</id>
           <mirrorOf>central</mirrorOf>
           <name>maven.aliyun.com</name>
           <url>http://repository.cloudera.com/artifactory/cloudera-repos/</url>
       </mirror>
       <mirror>
           <id>default</id>
           <mirrorOf>central</mirrorOf>
           <name>maven.cloudera.com</name>
           <url>https://repo.maven.apache.org/maven2/</url>
       </mirror>

ZK

    mkdir /opt/zookeeper
    cd /opt/zookeeper
    wget https://mirrors.tuna.tsinghua.edu.cn/apache/zookeeper/zookeeper-3.6.3/apache-zookeeper-3.6.3-bin.tar.gz
    tar -xvf apache-zookeeper-3.6.3-bin.tar.gz
    mv apache-zookeeper-3.6.3-bin zookeeper
    mv apache-zookeeper-3.6.3-bin.tar.gz zookeeper.tar.gz
    mv zookeeper.tar.gz /opt/pkg/
    cd /opt/zookeeper/zookeeper/conf/
    mv zoo_sample.cfg zoo.cfg
    mkdir /opt/zookeeper/zookeeper/data
    vim zoo.cfg
        dataDir=/opt/zookeeper/zookeeper/data/

    cd /opt/zookeeper/zookeeper/bin

    server进程启动：
    ./zkServer.sh start
    ./zkServer.sh stop

    客户端进程启动：
    ./zkCli.sh

hbase

    不需要单独安装zk
    mkdir /opt/hbase
    cd /opt/hbase
    wget https://mirrors.tuna.tsinghua.edu.cn/apache/hbase/2.5.0/hbase-2.5.0-bin.tar.gz
    tar -xvf hbase-2.5.0-bin.tar.gz
    mv hbase-2.5.0 hbase
    mv hbase-2.5.0-bin.tar.gz hbase.tar.gz
    mv hbase.tar.gz /opt/pkg/
    mkdir /opt/hbase/hbase/data

    cd /opt/hbase/hbase/conf/
    vim hbase-env.sh
        export JAVA_HOME=/opt/java/jdk/
    vim hbase-site.xml

           <property>
             <name>hbase.cluster.distributed</name>
             <value>false</value>
           </property>
           <property>
             <name>hbase.tmp.dir</name>
             <value>/tmp/hbase/</value>
           </property>
           <property>
             <name>hbase.unsafe.stream.capability.enforce</name>
             <value>false</value>
           </property>
           <property>
                <name>hbase.rootdir</name>
                <value>file:///opt/hbase/hbase/data</value>
           </property>
           <property>
                <name>hbase.zookeeper.property.dataDir</name>
                <value>/root/data/hbase/zookeeper</value>
           </property>
           <property>
           <name>hbase.unsafe.stream.capability.enforce</name>
                 <value>false</value>
           </property>
           <property>
                 <name>hbase.master.info.port</name>
                 <value>60010</value>
           </property>

    vim /root/.bashrc
    export HBASE_HOME=/opt/hbase/hbase
    export PATH=$PATH:$HBASE_HOME/bin

    启动命令：
    cd /opt/hbase/hbase/bin
    ./start-hbase.sh
    ./stop-hbase.sh

flink

    mkdir /opt/flink
    cd /opt/flink/
    wget https://mirrors.tuna.tsinghua.edu.cn/apache/flink/flink-1.15.2/flink-1.15.2-bin-scala_2.12.tgz
    tar -xvf flink-1.15.2-bin-scala_2.12.tgz
    mv flink-1.15.2-bin-scala_2.12.tgz flink_scala_2.12.tgz
    mv flink_scala_2.12.tgz /opt/pkg/
    mv flink-1.15.2 flink

    vim .bashrc
    export FLNK_HOME=/opt/flink/flink
    export PATH=$FLINK_HOME/bin:$PATH

    cd /opt/flink/flink/bin/
    ./start-cluster.sh

    flink run ../examples/batch/WordCount.jar --input /opt/flink/flink/README.txt --output /opt/flink/flink/result

redis

mkdir /opt/redis/
cd /opt/redis/
wget https://download.redis.io/releases/redis-6.2.6.tar.gz
mv redis-6.2.6.tar.gz redis.tar.gz
tar -xvf redis.tar.gz 
mv redis.tar.gz /opt/pkg/
mv redis-6.2.6 redis


cd /opt/redis/redis
yum -y install gcc automake autoconf libtool make
make install PREFIX=/opt/redis/redis

vim redis.conf
daemonize yes
bind 0.0.0.0

./bin/redis-server redis.conf


./bin/redis-cli
set test hello
get test

Kafka

    mkdir /opt/kafka/
    cd /opt/kafka/
    wget https://mirrors.tuna.tsinghua.edu.cn/apache/kafka/3.1.2/kafka_2.13-3.1.2.tgz .
    mv kafka_2.13-3.1.2.tgz kafka.tgz
    tar -xvf kafka.tgz
    mv kafka.tgz /opt/pkg/
    mv kafka_2.13-3.1.2 kafka

    cd /opt/kafka/kafka/config

    vim server.properties

    broker.id=0 
    advertised.listeners=PLAINTEXT://公网IP:9092
    log.dirs=/opt/kafka/kafka/log/ #日志存放路径可修改可不修改
    zookeeper.connect=局域网IP:2181  #给公网IP 局域网IP 127.0.0.1添加白名单

    cd /opt/kafka/kafka/bin/
    ##启动ZK
    ./zookeeper-server-start.sh -daemon /opt/kafka/kafka/config/zookeeper.properties  
    ##启动kafka
    ./kafka-server-start.sh -daemon /opt/kafka/kafka/config/server.properties
    
    ##删除topics
    ./kafka-topics.sh --delete --bootstrap-server localhost:9092 --topic events
    ##创建topics
    ./kafka-topics.sh --create --topic events --bootstrap-server localhost:9092
    ##desc topics
    ./kafka-topics.sh --describe --topic events --bootstrap-server localhost:9092
    
    
    ##创建命令行生产者
    ./kafka-console-producer.sh --topic events --bootstrap-server localhost:9092
    ##打印topic
    ./kafka-console-consumer.sh --topic events --from-beginning --bootstrap-server localhost:9092

datax

    mkdir  /opt/datax
    cd   /opt/datax
    wget https://datax-opensource.oss-cn-hangzhou.aliyuncs.com/202209/datax.tar.gz
    tar -xvf datax.tar.gz 
    mv datax.tar.gz /opt/pkg/
    cd /opt/datax/datax/bin/
    chmod 700 ./datax.py

maven

mkdir -p /opt/maven
cd /opt/maven
wget https://repo.huaweicloud.com/apache/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz
tar -xvf apache-maven-3.6.3-bin.tar.gz
mv apache-maven-3.6.3 maven
mv apache-maven-3.6.3-bin.tar.gz maven.tar.gz
mv  maven.tar.gz /opt/pkg/

vim /root/.bashrc
export MAVEN_HOME=/opt/maven/maven/
export PATH=$PATH:$MAVEN_HOME/bin
source /root/.bashrc

vim /opt/maven/maven/conf/settings.xml
<mirror>
     <id>huaweicloud</id>
     <mirrorOf>central</mirrorOf>
     <url>https://repo.huaweicloud.com/repository/maven/</url>
</mirror>
<mirror>
    <id>aliyun</id>
    <mirrorOf>central</mirrorOf>
    <name>maven.aliyun.com</name>
    <url>http://maven.aliyun.com/nexus/content/groups/public/</url>
</mirror>
<mirror>
    <id>crespo1</id>
    <mirrorOf>central</mirrorOf>
    <name>maven.aliyun.com</name>
    <url>http://repository.cloudera.com/artifactory/cloudera-repos/</url>
</mirror>
<mirror>
    <id>default</id>
    <mirrorOf>central</mirrorOf>
    <name>maven.cloudera.com</name>
    <url>https://repo.maven.apache.org/maven2/</url>
</mirror>

apt install git
apt install python