大数据平台运行监控
查看Linux系统的信息
[root@master ~]# uname -a
Linux master 3.10.0-862.el7.x86_64 #1 SMP Fri Apr 20 16:44:24 UTC 2018 x86_64 x86_64 x86_64 GNU/Linux
查看硬盘信息
查看所有分区
[root@master ~]# fdisk -l
Disk /dev/sda: 21.5 GB, 21474836480 bytes, 41943040 sectors
Units = sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk label type: dos
Disk identifier: 0x0009a8cd
Device Boot Start End Blocks Id System
/dev/sda1 * 2048 2099199 1048576 83 Linux
/dev/sda2 2099200 41943039 19921920 8e Linux LVM
Disk /dev/mapper/centos-root: 18.2 GB, 18249416704 bytes, 35643392 sectors
Units = sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk /dev/mapper/centos-swap: 2147 MB, 2147483648 bytes, 4194304 sectors
Units = sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
查看所有交换分区
[root@master ~]# swapon -s
Filename Type Size Used Priority
/dev/dm-1 partition 2097148 0 -1
查看系统文件占比
[root@master ~]# df -h
Filesystem Size Used Avail Use% Mounted on
/dev/mapper/centos-root 17G 4.7G 13G 28% /
devtmpfs 980M 0 980M 0% /dev
tmpfs 992M 0 992M 0% /dev/shm
tmpfs 992M 9.5M 982M 1% /run
tmpfs 992M 0 992M 0% /sys/fs/cgroup
/dev/sda1 1014M 130M 885M 13% /boot
tmpfs 199M 0 199M 0% /run/user/0
查看网络IP地址
[root@master ~]# ifconfig
ens33: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet 192.168.88.10 netmask 255.255.255.0 broadcast 192.168.88.255
inet6 fe80::2f46:dfe2:4827:2352 prefixlen 64 scopeid 0x20<link>
ether 00:0c:29:e5:4e:63 txqueuelen 1000 (Ethernet)
RX packets 179 bytes 18743 (18.3 KiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 136 bytes 19092 (18.6 KiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
lo: flags=73<UP,LOOPBACK,RUNNING> mtu 65536
inet 127.0.0.1 netmask 255.0.0.0
inet6 ::1 prefixlen 128 scopeid 0x10<host>
loop txqueuelen 1000 (Local Loopback)
RX packets 0 bytes 0 (0.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 0 bytes 0 (0.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
查看所有监听端口
[root@master ~]# netstat -lntp
Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name
tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN 881/sshd
tcp6 0 0 :::3306 :::* LISTEN 915/mysqld
tcp6 0 0 :::22 :::* LISTEN 881/sshd
查看所有已经建立的连接
[root@master ~]# netstat -antp
Active Internet connections (servers and established)
Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name
tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN 881/sshd
tcp 0 52 192.168.88.10:22 192.168.88.1:2124 ESTABLISHED 1122/sshd: root@pts
tcp6 0 0 :::3306 :::* LISTEN 915/mysqld
tcp6 0 0 :::22 :::* LISTEN 881/sshd
实时显示进程状态（top），该命令可以查看进程对 CPU、内存的占比
[root@master ~]# top
top - 10:17:53 up 24 min, 1 user, load average: 0.00, 0.01, 0.05
Tasks: 99 total, 1 running, 98 sleeping, 0 stopped, 0 zombie
%Cpu(s): 0.0 us, 0.0 sy, 0.0 ni,100.0 id, 0.0 wa, 0.0 hi, 0.0 si, 0.0 st
KiB Mem : 2030172 total, 1582316 free, 281340 used, 166516 buff/cache
KiB Swap: 2097148 total, 2097148 free, 0 used. 1574280 avail Mem
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
1 root 20 0 125352 3760 2564 S 0.0 0.2 0:00.90 systemd
2 root 20 0 0 0 0 S 0.0 0.0 0:00.00 kthreadd
3 root 20 0 0 0 0 S 0.0 0.0 0:00.03 ksoftirqd/0
5 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 kworker/0:0H
7 root rt 0 0 0 0 S 0.0 0.0 0:00.02 migration/0
8 root 20 0 0 0 0 S 0.0 0.0 0:00.00 rcu_bh
9 root 20 0 0 0 0 S 0.0 0.0 0:00.22 rcu_sched
10 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 lru-add-drain
11 root rt 0 0 0 0 S 0.0 0.0 0:00.00 watchdog/0
12 root rt 0 0 0 0 S 0.0 0.0 0:00.00 watchdog/1
13 root rt 0 0 0 0 S 0.0 0.0 0:00.00 migration/1
14 root 20 0 0 0 0 S 0.0 0.0 0:00.02 ksoftirqd/1
16 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 kworker/1:0H
18 root 20 0 0 0 0 S 0.0 0.0 0:00.00 kdevtmpfs
19 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 netns
20 root 20 0 0 0 0 S 0.0 0.0 0:00.00 khungtaskd
21 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 writeback
22 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 kintegrityd
23 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 bioset
24 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 kblockd
25 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 md
26 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 edac-poller
27 root 20 0 0 0 0 S 0.0 0.0 0:01.10 kworker/0:1
32 root 20 0 0 0 0 S 0.0 0.0 0:00.00 kswapd0
33 root 25 5 0 0 0 S 0.0 0.0 0:00.00 ksmd
34 root 39 19 0 0 0 S 0.0 0.0 0:00.26 khugepaged
35 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 crypto
43 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 kthrotld
44 root 20 0 0 0 0 S 0.0 0.0 0:00.00 kworker/u256:1
45 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 kmpath_rdacd
46 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 kaluad
48 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 kpsmoused
50 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 ipv6_addrconf
63 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 deferwq
94 root 20 0 0 0 0 S 0.0 0.0 0:00.00 kauditd
查看 CPU 信息
[root@master ~]# cat /proc/cpuinfo
processor : 0
vendor_id : GenuineIntel
cpu family : 6
model : 158
model name : Intel(R) Core(TM) i5-9500 CPU @ 3.00GHz
stepping : 10
microcode : 0xca
cpu MHz : 3000.000
cache size : 9216 KB
physical id : 0
siblings : 1
core id : 0
cpu cores : 1
apicid : 0
initial apicid : 0
fpu : yes
fpu_exception : yes
cpuid level : 22
wp : yes
flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon nopl xtopology tsc_reliable nonstop_tsc eagerfpu pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch fsgsbase tsc_adjust bmi1 avx2 smep bmi2 invpcid mpx rdseed adx smap clflushopt xsaveopt xsavec ibpb ibrs stibp arat spec_ctrl intel_stibp arch_capabilities
bogomips : 6000.00
clflush size : 64
cache_alignment : 64
address sizes : 43 bits physical, 48 bits virtual
power management:
processor : 1
vendor_id : GenuineIntel
cpu family : 6
model : 158
model name : Intel(R) Core(TM) i5-9500 CPU @ 3.00GHz
stepping : 10
microcode : 0xca
cpu MHz : 3000.000
cache size : 9216 KB
physical id : 2
siblings : 1
core id : 0
cpu cores : 1
apicid : 2
initial apicid : 2
fpu : yes
fpu_exception : yes
cpuid level : 22
wp : yes
flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon nopl xtopology tsc_reliable nonstop_tsc eagerfpu pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch fsgsbase tsc_adjust bmi1 avx2 smep bmi2 invpcid mpx rdseed adx smap clflushopt xsaveopt xsavec ibpb ibrs stibp arat spec_ctrl intel_stibp arch_capabilities
bogomips : 6000.00
clflush size : 64
cache_alignment : 64
address sizes : 43 bits physical, 48 bits virtual
power management:
查看内存信息，可以查看总内存、空闲内存等信息
[root@master ~]# cat /proc/meminfo
MemTotal: 2030172 kB
MemFree: 1582904 kB
MemAvailable: 1574868 kB
Buffers: 2112 kB
Cached: 122488 kB
SwapCached: 0 kB
Active: 243220 kB
Inactive: 105124 kB
Active(anon): 224160 kB
Inactive(anon): 9312 kB
Active(file): 19060 kB
Inactive(file): 95812 kB
Unevictable: 0 kB
Mlocked: 0 kB
SwapTotal: 2097148 kB
SwapFree: 2097148 kB
Dirty: 0 kB
Writeback: 0 kB
AnonPages: 223808 kB
Mapped: 28564 kB
Shmem: 9728 kB
Slab: 41916 kB
SReclaimable: 17664 kB
SUnreclaim: 24252 kB
KernelStack: 4256 kB
PageTables: 3748 kB
NFS_Unstable: 0 kB
Bounce: 0 kB
WritebackTmp: 0 kB
CommitLimit: 3112232 kB
Committed_AS: 771820 kB
VmallocTotal: 34359738367 kB
VmallocUsed: 179204 kB
VmallocChunk: 34359310332 kB
HardwareCorrupted: 0 kB
AnonHugePages: 182272 kB
CmaTotal: 0 kB
CmaFree: 0 kB
HugePages_Total: 0
HugePages_Free: 0
HugePages_Rsvd: 0
HugePages_Surp: 0
Hugepagesize: 2048 kB
DirectMap4k: 65408 kB
DirectMap2M: 2031616 kB
DirectMap1G: 0 kB
命令查看Hadoop状态
切换到hadoop用户
[root@master ~]# su - hadoop
Last login: Wed Apr 19 10:30:12 CST 2023 on pts/0
切换到Hadoop的安装目录
[hadoop@master ~]$ cd /usr/local/src/hadoop/
[hadoop@master hadoop]$
启动Hadoop
#master
[hadoop@master hadoop]$ start-all.sh
This script is Deprecated. Instead use start-dfs.sh and start-yarn.sh
Starting namenodes on [master]
master: starting namenode, logging to /usr/local/src/hadoop/logs/hadoop-hadoop-namenode-master.out
192.168.88.30: starting datanode, logging to /usr/local/src/hadoop/logs/hadoop-hadoop-datanode-slave2.out
192.168.88.20: starting datanode, logging to /usr/local/src/hadoop/logs/hadoop-hadoop-datanode-slave1.out
Starting secondary namenodes [0.0.0.0]
0.0.0.0: starting secondarynamenode, logging to /usr/local/src/hadoop/logs/hadoop-hadoop-secondarynamenode-master.out
starting yarn daemons
starting resourcemanager, logging to /usr/local/src/hadoop/logs/yarn-hadoop-resourcemanager-master.out
192.168.88.20: nodemanager running as process 1263. Stop it first.
192.168.88.30: nodemanager running as process 1260. Stop it first.
[hadoop@master hadoop]$ jps
1522 SecondaryNameNode
1335 NameNode
1674 ResourceManager
1935 Jps
#slave1
[hadoop@slave1 ~]$ jps
1236 DataNode
1343 Jps
#slave2
[hadoop@slave2 ~]$ jps
1337 Jps
1231 DataNode
关闭Hadoop
#master
[hadoop@master hadoop]$ stop-all.sh
This script is Deprecated. Instead use stop-dfs.sh and stop-yarn.sh
Stopping namenodes on [master]
master: stopping namenode
192.168.88.20: stopping datanode
192.168.88.30: stopping datanode
Stopping secondary namenodes [0.0.0.0]
0.0.0.0: stopping secondarynamenode
stopping yarn daemons
stopping resourcemanager
192.168.88.20: no nodemanager to stop
192.168.88.30: no nodemanager to stop
no proxyserver to stop
[hadoop@master hadoop]$ jps
2404 Jps
#slave1
[hadoop@slave1 ~]$ jps
1416 Jps
#slave2
[hadoop@slave2 ~]$ jps
1412 Jps
大数据平台资源状态监控
查看YARN状态
切换目录
[hadoop@master ~]$ cd /usr/local/src/hadoop/
[hadoop@master hadoop]$
执行start-all.sh,启动hadoop,执行start-yarn.sh启动yarn
#master
[hadoop@master hadoop]$ start-all.sh
This script is Deprecated. Instead use start-dfs.sh and start-yarn.sh
Starting namenodes on [master]
master: starting namenode, logging to /usr/local/src/hadoop/logs/hadoop-hadoop-namenode-master.out
192.168.88.20: starting datanode, logging to /usr/local/src/hadoop/logs/hadoop-hadoop-datanode-slave1.out
192.168.88.30: starting datanode, logging to /usr/local/src/hadoop/logs/hadoop-hadoop-datanode-slave2.out
Starting secondary namenodes [0.0.0.0]
0.0.0.0: starting secondarynamenode, logging to /usr/local/src/hadoop/logs/hadoop-hadoop-secondarynamenode-master.out
starting yarn daemons
starting resourcemanager, logging to /usr/local/src/hadoop/logs/yarn-hadoop-resourcemanager-master.out
192.168.88.20: starting nodemanager, logging to /usr/local/src/hadoop/logs/yarn-hadoop-nodemanager-slave1.out
192.168.88.30: starting nodemanager, logging to /usr/local/src/hadoop/logs/yarn-hadoop-nodemanager-slave2.out
[hadoop@master hadoop]$ start-yarn.sh
starting yarn daemons
resourcemanager running as process 2885. Stop it first.
192.168.88.30: nodemanager running as process 1564. Stop it first.
192.168.88.20: nodemanager running as process 1568. Stop it first.
[hadoop@master hadoop]$ jps
3219 Jps
2885 ResourceManager
2536 NameNode
2728 SecondaryNameNode
#slave1
[hadoop@slave1 ~]$ jps
1568 NodeManager
1458 DataNode
1722 Jps
#slave2
[hadoop@slave2 ~]$ jps
1718 Jps
1564 NodeManager
1454 DataNode
查看HDFS状态
切换到hadoop目录，并查看HDFS目录
[hadoop@master hadoop]$ cd /usr/local/src/hadoop/
[hadoop@master hadoop]$
#查看
[hadoop@master hadoop]$ ./bin/hdfs dfs -ls /
Found 5 items
drwxr-xr-x - hadoop supergroup 0 2023-04-06 22:55 /hbase
drwxr-xr-x - hadoop supergroup 0 2023-04-19 10:49 /input
drwxr-xr-x - hadoop supergroup 0 2023-04-19 10:59 /output
drwx-wx-wx - hadoop supergroup 0 2023-04-19 10:37 /tmp
drwxr-xr-x - hadoop supergroup 0 2023-04-12 14:28 /user
查看HDFS的报告
[hadoop@master hadoop]$ bin/hdfs dfsadmin -report
Configured Capacity: 36477861888 (33.97 GB)
Present Capacity: 30873477120 (28.75 GB)
DFS Remaining: 30868611072 (28.75 GB)
DFS Used: 4866048 (4.64 MB)
DFS Used%: 0.02%
Under replicated blocks: 161
Blocks with corrupt replicas: 0
Missing blocks: 0
Missing blocks (with replication factor 1): 0
-------------------------------------------------
Live datanodes (2):
Name: 192.168.88.30:50010 (slave2)
Hostname: slave2
Decommission Status : Normal
Configured Capacity: 18238930944 (16.99 GB)
DFS Used: 2433024 (2.32 MB)
Non DFS Used: 2802823168 (2.61 GB)
DFS Remaining: 15433674752 (14.37 GB)
DFS Used%: 0.01%
DFS Remaining%: 84.62%
Configured Cache Capacity: 0 (0 B)
Cache Used: 0 (0 B)
Cache Remaining: 0 (0 B)
Cache Used%: 100.00%
Cache Remaining%: 0.00%
Xceivers: 1
Last contact: Wed Apr 26 10:39:55 CST 2023
Name: 192.168.88.20:50010 (slave1)
Hostname: slave1
Decommission Status : Normal
Configured Capacity: 18238930944 (16.99 GB)
DFS Used: 2433024 (2.32 MB)
Non DFS Used: 2801561600 (2.61 GB)
DFS Remaining: 15434936320 (14.37 GB)
DFS Used%: 0.01%
DFS Remaining%: 84.63%
Configured Cache Capacity: 0 (0 B)
Cache Used: 0 (0 B)
Cache Remaining: 0 (0 B)
Cache Used%: 100.00%
Cache Remaining%: 0.00%
Xceivers: 1
Last contact: Wed Apr 26 10:39:55 CST 2023
查看HDFS空间情况
[hadoop@master hadoop]$ hdfs dfs -df
Filesystem Size Used Available Use%
hdfs://master:9000 36477861888 4866048 30868611072 0%
查看HBase状态
启动运行HBase
#切换到HBae的安装目录
[hadoop@master hadoop]$ cd /usr/local/src/hbase/
[hadoop@master hbase]$
#显示版本号
[hadoop@master hbase]$ hbase version
HBase 1.2.1
Source code repository git://asf-dev/home/busbey/projects/hbase revision=8d8a7107dc4ccbf36a92f64675dc60392f85c015
Compiled by busbey on Wed Mar 30 11:19:21 CDT 2016
From source with checksum f4bb4a14bb4e0b72b46f729dae98a772
#启动HBse
[hadoop@master hbase]$ hbase version
HBase 1.2.1
Source code repository git://asf-dev/home/busbey/projects/hbase revision=8d8a7107dc4ccbf36a92f64675dc60392f85c015
Compiled by busbey on Wed Mar 30 11:19:21 CDT 2016
From source with checksum f4bb4a14bb4e0b72b46f729dae98a772
[hadoop@master hbase]$ start-hbase.sh
starting master, logging to /usr/local/src/hbase/logs/hbase-hadoop-master-master.out
SLF4J: Class path contains multiple SLF4J bindings.
SLF4J: Found binding in [jar:file:/usr/local/src/hbase/lib/slf4j-log4j12-1.7.5.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: Found binding in [jar:file:/usr/local/src/hadoop/share/hadoop/common/lib/slf4j-log4j12-1.7.10.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation.
SLF4J: Actual binding is of type [org.slf4j.impl.Log4jLoggerFactory]
slave2: starting regionserver, logging to /usr/local/src/hbase/logs/hbase-hadoop-regionserver-slave2.out
slave1: starting regionserver, logging to /usr/local/src/hbase/logs/hbase-hadoop-regionserver-slave1.out
查看HBase版本信息与状态
#进入hbase shell
[hadoop@master hbase]$ hbase shell
SLF4J: Class path contains multiple SLF4J bindings.
SLF4J: Found binding in [jar:file:/usr/local/src/hbase/lib/slf4j-log4j12-1.7.5.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: Found binding in [jar:file:/usr/local/src/hadoop/share/hadoop/common/lib/slf4j-log4j12-1.7.10.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation.
SLF4J: Actual binding is of type [org.slf4j.impl.Log4jLoggerFactory]
HBase Shell; enter 'help<RETURN>' for list of supported commands.
Type "exit<RETURN>" to leave the HBase Shell
Version 1.2.1, r8d8a7107dc4ccbf36a92f64675dc60392f85c015, Wed Mar 30 11:19:21 CDT 2016
hbase(main):001:0> version
1.2.1, r8d8a7107dc4ccbf36a92f64675dc60392f85c015, Wed Mar 30 11:19:21 CDT 2016
#查看状态，查看前需要开启zookeeper，否则会报错
hbase(main):002:0> status
1 active master, 0 backup masters, 2 servers, 0 dead, 1.0000 average load
#简单查看状态
hbase(main):003:0> status 'simple'
active master: master:16000 1682585254586
0 backup masters
2 live servers
slave1:16020 1682585256209
requestsPerSecond=0.0, numberOfOnlineRegions=0, usedHeapMB=9, maxHeapMB=235, numberOfStores=0, numberOfStorefiles=0, storefileUncompressedSizeMB=0, storefileSizeMB=0, memstoreSizeMB=0, storefileIndexSizeMB=0, readRequestsCount=0, writeRequestsCount=0, rootIndexSizeKB=0, totalStaticIndexSizeKB=0, totalStaticBloomSizeKB=0, totalCompactingKVs=0, currentCompactedKVs=0, compactionProgressPct=NaN, coprocessors=[]
slave2:16020 1682585255941
requestsPerSecond=0.0, numberOfOnlineRegions=2, usedHeapMB=20, maxHeapMB=235, numberOfStores=2, numberOfStorefiles=3, storefileUncompressedSizeMB=0, storefileSizeMB=0, memstoreSizeMB=0, storefileIndexSizeMB=0, readRequestsCount=10, writeRequestsCount=1, rootIndexSizeKB=0, totalStaticIndexSizeKB=0, totalStaticBloomSizeKB=0, totalCompactingKVs=0, currentCompactedKVs=0, compactionProgressPct=NaN, coprocessors=[MultiRowMutationEndpoint]
0 dead servers
Aggregate load: 0, regions: 2
#显示所有关于状态的命令
hbase(main):004:0> help 'status'
Show cluster status. Can be 'summary', 'simple', 'detailed', or 'replication'. The
default is 'summary'. Examples:
hbase> status
hbase> status 'simple'
hbase> status 'summary'
hbase> status 'detailed'
hbase> status 'replication'
hbase> status 'replication', 'source'
hbase> status 'replication', 'sink'
停止HBase服务
[root@master ~]# stop-hbase.sh
stopping hbase...............
[root@master ~]# jps
2480 NameNode
2825 ResourceManager
2668 SecondaryNameNode
3486 QuorumPeerMain
4447 Jps
查看Hive状态
启动运行Hive
#切换目录
[root@master ~]# cd /usr/local/src/hive/
#启动hive
[root@master hive]# hive
SLF4J: Class path contains multiple SLF4J bindings.
SLF4J: Found binding in [jar:file:/usr/local/src/hive/lib/hive-jdbc-2.0.0-standalone.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: Found binding in [jar:file:/usr/local/src/hive/lib/log4j-slf4j-impl-2.4.1.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: Found binding in [jar:file:/usr/local/src/hadoop/share/hadoop/common/lib/slf4j-log4j12-1.7.10.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation.
SLF4J: Actual binding is of type [org.apache.logging.slf4j.Log4jLoggerFactory]
Logging initialized using configuration in jar:file:/usr/local/src/hive/lib/hive-common-2.0.0.jar!/hive-log4j2.properties
Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases.
hive>
Hive基本操作命令
查看数据库
hive> show databases;
OK
default
sample
Time taken: 0.689 seconds, Fetched: 2 row(s)
查看default数据库所有表
hive> use default;
OK
Time taken: 0.011 seconds
hive> show tables;
OK
student
test
Time taken: 0.11 seconds, Fetched: 2 row(s)
创建表stu，表的id为整数型，name为字符型
hive> create table stu(id int,name string);
OK
Time taken: 0.281 seconds
为表stu插入一条信息，id号为001，name为张三
hive> insert into stu values (1001,"zhangsan");
WARNING: Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases.
Query ID = root_20230427170324_c3499e91-7cf9-4967-b7e3-9a12afa2912b
Total jobs = 3
Launching Job 1 out of 3
Number of reduce tasks is set to 0 since there's no reduce operator
Starting Job = job_1682584839688_0001, Tracking URL = http://master:8088/proxy/application_1682584839688_0001/
Kill Command = /usr/local/src/hadoop/bin/hadoop job -kill job_1682584839688_0001
Hadoop job information for Stage-1: number of mappers: 1; number of reducers: 0
2023-04-27 17:08:18,607 Stage-1 map = 0%, reduce = 0%
2023-04-27 17:08:23,842 Stage-1 map = 100%, reduce = 0%, Cumulative CPU 0.85 sec
MapReduce Total cumulative CPU time: 850 msec
Ended Job = job_1682584839688_0001
Stage-4 is selected by condition resolver.
Stage-3 is filtered out by condition resolver.
Stage-5 is filtered out by condition resolver.
Moving data to: hdfs://master:9000/user/hive/warehouse/stu/.hive-staging_hive_2023-04-27_17-08-10_251_7508438910956791798-1/-ext-10000
Loading data to table default.stu
MapReduce Jobs Launched:
Stage-Stage-1: Map: 1 Cumulative CPU: 0.85 sec HDFS Read: 4132 HDFS Write: 81 SUCCESS
Total MapReduce CPU Time Spent: 850 msec
OK
Time taken: 14.815 seconds
#按照以上操作，继续插入两条信息：id 和 name 分别为 1002、1003 和 lisi、wangwu
hive> insert into stu values (1002,"lisi");
WARNING: Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases.
Query ID = root_20230427170324_c3499e91-7cf9-4967-b7e3-9a12afa2912b
Total jobs = 3
Launching Job 1 out of 3
Number of reduce tasks is set to 0 since there's no reduce operator
Starting Job = job_1682584839688_0002, Tracking URL = http://master:8088/proxy/application_1682584839688_0002/
Kill Command = /usr/local/src/hadoop/bin/hadoop job -kill job_1682584839688_0002
Hadoop job information for Stage-1: number of mappers: 1; number of reducers: 0
2023-04-27 17:10:30,381 Stage-1 map = 0%, reduce = 0%
2023-04-27 17:10:35,577 Stage-1 map = 100%, reduce = 0%, Cumulative CPU 1.15 sec
MapReduce Total cumulative CPU time: 1 seconds 150 msec
Ended Job = job_1682584839688_0002
Stage-4 is selected by condition resolver.
Stage-3 is filtered out by condition resolver.
Stage-5 is filtered out by condition resolver.
Moving data to: hdfs://master:9000/user/hive/warehouse/stu/.hive-staging_hive_2023-04-27_17-10-25_571_3098162169129405922-1/-ext-10000
Loading data to table default.stu
MapReduce Jobs Launched:
Stage-Stage-1: Map: 1 Cumulative CPU: 1.15 sec HDFS Read: 4234 HDFS Write: 77 SUCCESS
Total MapReduce CPU Time Spent: 1 seconds 150 msec
OK
Time taken: 12.172 seconds
hive> insert into stu values (1003,"wangwu");
WARNING: Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases.
Query ID = root_20230427170324_c3499e91-7cf9-4967-b7e3-9a12afa2912b
Total jobs = 3
Launching Job 1 out of 3
Number of reduce tasks is set to 0 since there's no reduce operator
Starting Job = job_1682584839688_0003, Tracking URL = http://master:8088/proxy/application_1682584839688_0003/
Kill Command = /usr/local/src/hadoop/bin/hadoop job -kill job_1682584839688_0003
Hadoop job information for Stage-1: number of mappers: 1; number of reducers: 0
2023-04-27 17:11:15,711 Stage-1 map = 0%, reduce = 0%
2023-04-27 17:11:20,935 Stage-1 map = 100%, reduce = 0%, Cumulative CPU 1.03 sec
MapReduce Total cumulative CPU time: 1 seconds 30 msec
Ended Job = job_1682584839688_0003
Stage-4 is selected by condition resolver.
Stage-3 is filtered out by condition resolver.
Stage-5 is filtered out by condition resolver.
Moving data to: hdfs://master:9000/user/hive/warehouse/stu/.hive-staging_hive_2023-04-27_17-11-10_851_3623561733284738226-1/-ext-10000
Loading data to table default.stu
MapReduce Jobs Launched:
Stage-Stage-1: Map: 1 Cumulative CPU: 1.03 sec HDFS Read: 4236 HDFS Write: 79 SUCCESS
Total MapReduce CPU Time Spent: 1 seconds 30 msec
OK
Time taken: 11.254 seconds
插入数据后查看表的信息
hive> show tables;
OK
stu
student
test
values__tmp__table__1
values__tmp__table__2
values__tmp__table__3
Time taken: 0.008 seconds, Fetched: 6 row(s)
查看表结构
hive> desc stu;
OK
id int
name string
Time taken: 0.019 seconds, Fetched: 2 row(s)
查看表stu内容
hive> select * from stu;
OK
1001 zhangsan
1002 lisi
1003 wangwu
Time taken: 0.042 seconds, Fetched: 3 row(s)
查看文件系统和历史命令
查看本地文件系统
#在命令前加!表示可以不退出hive命令行界面去执行主机的命令，这里就是在不退出hive命令行的情况下查看/usr/local/src文件夹下面有什么
hive> ! ls /usr/local/src;
flume
hadoop
hbase
hive
jdk
sqoop
zookeeper
查看HDFS文件系统
hive> dfs -ls /;
Found 5 items
drwxr-xr-x - hadoop supergroup 0 2023-04-27 16:47 /hbase
drwxr-xr-x - hadoop supergroup 0 2023-04-19 10:49 /input
drwxr-xr-x - hadoop supergroup 0 2023-04-19 10:59 /output
drwx-wx-wx - hadoop supergroup 0 2023-04-19 10:37 /tmp
drwxr-xr-x - hadoop supergroup 0 2023-04-12 14:28 /user
查看在 Hive 中输入的所有历史命令
[root@master hive]# cd /home/hadoop/
[root@master hadoop]# cat .hivehistory
exit
quit
create database sample;
show databases;
use sample;
create table student(number STRING,name STRING)
create table student(number STRING,name STRING);
exit
show databases;
use sample;
show tables;
select * form student;
exit
quit
show databases;
use sample;
show tables;
select * from student;
大数据平台服务状态
查看Zookeeper状态
查看 ZooKeeper 状态，执行命令 zkServer.sh status
[root@master ~]# zkServer.sh status
ZooKeeper JMX enabled by default
Using config: /usr/local/src/zookeeper/bin/../conf/zoo.cfg
Mode: follower
[root@slave1 ~]# zkServer.sh status
ZooKeeper JMX enabled by default
Using config: /usr/local/src/zookeeper/bin/../conf/zoo.cfg
Mode: leader
[root@slave2 tmp]# zkServer.sh status
ZooKeeper JMX enabled by default
Using config: /usr/local/src/zookeeper/bin/../conf/zoo.cfg
Mode: follower
查看运行进程
[root@master ~]# jps
2480 NameNode
5027 Jps
2825 ResourceManager
2668 SecondaryNameNode
3486 QuorumPeerMain
[root@slave1 ~]# jps
1379 DataNode
1765 QuorumPeerMain
1480 NodeManager
2200 Jps
[root@slave2 ~]# jps
1377 DataNode
2900 Jps
1765 QuorumPeerMain
1478 NodeManager
输入命令 zkCli.sh，连接到ZooKeeper 服务
[root@master ~]# zkCli.sh
Connecting to localhost:2181
2023-04-27 17:31:19,471 [myid:] - INFO [main:Environment@100] - Client environment:zookeeper.version=3.4.8--1, built on 02/06/2016 03:18 GMT
2023-04-27 17:31:19,473 [myid:] - INFO [main:Environment@100] - Client environment:host.name=master
2023-04-27 17:31:19,474 [myid:] - INFO [main:Environment@100] - Client environment:java.version=1.8.0_152
2023-04-27 17:31:19,475 [myid:] - INFO [main:Environment@100] - Client environment:java.vendor=Oracle Corporation
2023-04-27 17:31:19,475 [myid:] - INFO [main:Environment@100] - Client environment:java.home=/usr/local/src/jdk/jre
2023-04-27 17:31:19,475 [myid:] - INFO [main:Environment@100] - Client environment:java.class.path=/usr/local/src/zookeeper/bin/../build/classes:/usr/local/src/zookeeper/bin/../build/lib/*.jar:/usr/local/src/zookeeper/bin/../lib/slf4j-log4j12-1.6.1.jar:/usr/local/src/zookeeper/bin/../lib/slf4j-api-1.6.1.jar:/usr/local/src/zookeeper/bin/../lib/netty-3.7.0.Final.jar:/usr/local/src/zookeeper/bin/../lib/log4j-1.2.16.jar:/usr/local/src/zookeeper/bin/../lib/jline-0.9.94.jar:/usr/local/src/zookeeper/bin/../zookeeper-3.4.8.jar:/usr/local/src/zookeeper/bin/../src/java/lib/*.jar:/usr/local/src/zookeeper/bin/../conf::/usr/local/src/sqoop/lib
2023-04-27 17:31:19,475 [myid:] - INFO [main:Environment@100] - Client environment:java.library.path=/usr/java/packages/lib/amd64:/usr/lib64:/lib64:/lib:/usr/lib
2023-04-27 17:31:19,475 [myid:] - INFO [main:Environment@100] - Client environment:java.io.tmpdir=/tmp
2023-04-27 17:31:19,475 [myid:] - INFO [main:Environment@100] - Client environment:java.compiler=<NA>
2023-04-27 17:31:19,475 [myid:] - INFO [main:Environment@100] - Client environment:os.name=Linux
2023-04-27 17:31:19,475 [myid:] - INFO [main:Environment@100] - Client environment:os.arch=amd64
2023-04-27 17:31:19,475 [myid:] - INFO [main:Environment@100] - Client environment:os.version=3.10.0-862.el7.x86_64
2023-04-27 17:31:19,475 [myid:] - INFO [main:Environment@100] - Client environment:user.name=root
2023-04-27 17:31:19,475 [myid:] - INFO [main:Environment@100] - Client environment:user.home=/root
2023-04-27 17:31:19,475 [myid:] - INFO [main:Environment@100] - Client environment:user.dir=/root
2023-04-27 17:31:19,476 [myid:] - INFO [main:ZooKeeper@438] - Initiating client connection, connectString=localhost:2181 sessionTimeout=30000 watcher=org.apache.zookeeper.ZooKeeperMain$MyWatcher@69d0a921
Welcome to ZooKeeper!
2023-04-27 17:31:19,488 [myid:] - INFO [main-SendThread(localhost:2181):ClientCnxn$SendThread@1032] - Opening socket connection to server localhost/127.0.0.1:2181. Will not attempt to authenticate using SASL (unknown error)
JLine support is enabled
2023-04-27 17:31:19,531 [myid:] - INFO [main-SendThread(localhost:2181):ClientCnxn$SendThread@876] - Socket connection established to localhost/127.0.0.1:2181, initiating session
[zk: localhost:2181(CONNECTING) 0] 2023-04-27 17:31:19,544 [myid:] - INFO [main-SendThread(localhost:2181):ClientCnxn$SendThread@1299] - Session establishment complete on server localhost/127.0.0.1:2181, sessionid = 0x187c1e45c740006, negotiated timeout = 30000
WATCHER::
WatchedEvent state:SyncConnected type:None path:null
使用 Watch 监听/hbase 目录，一旦/hbase 内容有变化，将会有提示，打开监视，执行命令 get /hbase 1
[zk: localhost:2181(CONNECTED) 0] get /hbase 1
cZxid = 0x200000002
ctime = Thu Apr 06 22:47:39 CST 2023
mZxid = 0x200000002
mtime = Thu Apr 06 22:47:39 CST 2023
pZxid = 0x500000057
cversion = 28
dataVersion = 0
aclVersion = 0
ephemeralOwner = 0x0
dataLength = 0
numChildren = 14
[zk: localhost:2181(CONNECTED) 2] set /hbase value-update
cZxid = 0x200000002
ctime = Thu Apr 06 22:47:39 CST 2023
mZxid = 0x50000005f
mtime = Thu Apr 27 17:36:42 CST 2023
pZxid = 0x500000057
cversion = 28
dataVersion = 2
aclVersion = 0
ephemeralOwner = 0x0
dataLength = 12
numChildren = 14
结果显示，当执行命令 set /hbase value-update 后，数据版本由 0 变成 1，说明 /hbase 处于监控中。
命令查看 Sqoop 状态
查询 Sqoop 版本号，验证 Sqoop 是否启动成功
[hadoop@master ~]$ cd /usr/local/src/sqoop/
[hadoop@master sqoop]$ ./bin/sqoop-version
Warning: /usr/local/src/sqoop/../hcatalog does not exist! HCatalog jobs will fail.
Please set $HCAT_HOME to the root of your HCatalog installation.
Warning: /usr/local/src/sqoop/../accumulo does not exist! Accumulo imports will fail.
Please set $ACCUMULO_HOME to the root of your Accumulo installation.
23/05/02 09:34:17 INFO sqoop.Sqoop: Running Sqoop version: 1.4.7
Sqoop 1.4.7
git commit id 2328971411f57f0cb683dfb79d19d4d19d185dd8
Compiled by maugli on Thu Dec 21 15:59:58 STD 2017
结果显示：Sqoop 1.4.7，说明 Sqoop 版本号为 1.4.7，并启动成功
测试 Sqoop 是否能够成功连接数据库
[hadoop@master sqoop]$ bin/sqoop list-databases --connect jdbc:mysql://master:3306/ --username root --password Password@123_
Warning: /usr/local/src/sqoop/../hcatalog does not exist! HCatalog jobs will fail.
Please set $HCAT_HOME to the root of your HCatalog installation.
Warning: /usr/local/src/sqoop/../accumulo does not exist! Accumulo imports will fail.
Please set $ACCUMULO_HOME to the root of your Accumulo installation.
23/05/02 09:52:43 INFO sqoop.Sqoop: Running Sqoop version: 1.4.7
23/05/02 09:52:43 WARN tool.BaseSqoopTool: Setting your password on the command-line is insecure. Consider using -P instead.
23/05/02 09:52:43 INFO manager.MySQLManager: Preparing to use a MySQL streaming resultset.
Tue May 02 09:52:43 CST 2023 WARN: Establishing SSL connection without server's identity verification is not recommended. According to MySQL 5.5.45+, 5.6.26+ and 5.7.6+ requirements SSL connection must be established by default if explicit option isn't set. For compliance with existing applications not using SSL the verifyServerCertificate property is set to 'false'. You need either to explicitly disable SSL by setting useSSL=false, or set useSSL=true and provide truststore for server certificate verification.
information_schema
hive
mysql
performance_schema
sample
sys
结果显示，可以连接到 MySQL，并查看到 Master 主机中 MySQL 的所有库实例，如 information_schema、hive、mysql、performance_schema 和 sys 等数据库
执行命令 sqoop help，可以看到如下内容，代表 Sqoop 启动成功
[hadoop@master sqoop]$ sqoop help
Warning: /usr/local/src/sqoop/../hcatalog does not exist! HCatalog jobs will fail.
Please set $HCAT_HOME to the root of your HCatalog installation.
Warning: /usr/local/src/sqoop/../accumulo does not exist! Accumulo imports will fail.
Please set $ACCUMULO_HOME to the root of your Accumulo installation.
23/05/02 09:54:19 INFO sqoop.Sqoop: Running Sqoop version: 1.4.7
usage: sqoop COMMAND [ARGS]
Available commands:
codegen Generate code to interact with database records
create-hive-table Import a table definition into Hive
eval Evaluate a SQL statement and display the results
export Export an HDFS directory to a database table
help List available commands
import Import a table from a database to HDFS
import-all-tables Import tables from a database to HDFS
import-mainframe Import datasets from a mainframe server to HDFS
job Work with saved jobs
list-databases List available databases on a server
list-tables List available tables in a database
merge Merge results of incremental imports
metastore Run a standalone Sqoop metastore
version Display version information
See 'sqoop help COMMAND' for information on a specific command.
命令查看 Flume 状态
检查 Flume 安装是否成功，执行 flume-ng version 命令，查看 Flume 的版本
[hadoop@master flume]$ flume-ng version
Flume 1.6.0
Source code repository: https://git-wip-us.apache.org/repos/asf/flume.git
Revision: 2561a23240a71ba20bf288c7c2cda88f443c2080
Compiled by hshreedharan on Mon May 11 11:15:44 PDT 2015
From source ith checksum b29e416802ce9ece3269d34233baf43f
查看.conf文件，文件名不唯一，若没有就创建并写入如下cat命令所显示的内容
[hadoop@master flume]$ cd /usr/local/src/flume/
[hadoop@master flume]$ ls
bin CHANGELOG conf DEVNOTES docs lib LICENSE NOTICE rainmom-flume.conf README RELEASE-NOTES tools
[hadoop@master flume]$ cat rainmom-flume.conf
# rainmom是代理名，r1,k1,c1 是 a1 的三个组件，代理名不唯一，可自行定义，但在之后执行启动命令时要输入自己定义的正确的名称，这里使用的是雨妈作为代理名
rainmom.sources = r1
rainmom.sinks = k1
rainmom.channels = c1
# 设置 r1 源文件的类型、路径和文件头属性
rainmom.sources.r1.type = spooldir
rainmom.sources.r1.spoolDir = /usr/local/src/hadoop/logs/
rainmom.sources.r1.fileHeader = true
# 设置 k1 目标存储器属性
rainmom.sinks.k1.type = hdfs # 目标存储器类型 hdfs
rainmom.sinks.k1.hdfs.path = hdfs://master:9000/tmp/flume # 目标存储位置
rainmom.sinks.k1.hdfs.rollsize = 1048760 #临时文件达 1048760 bytes 时，滚动形
成目标文件
rainmom.sinks.k1.hdfs.rollCount = 0 #0 表示不根据 events 数量来滚动形成目标文件
rainmom.sinks.k1.hdfs.rollInterval = 900 # 间隔 900 秒将临时文件滚动形成目标文
件
rainmom.sinks.k1.hdfs.useLocalTimeStamp = true # 使用本地时间戳
# 设置 c1 暂存容器属性
rainmom.channels.c1.type = file # 使用文件作为暂存容器
rainmom.channels.c1.capacity = 1000
rainmom.channels.c1.transactionCapacity = 100
# 使用 c1 作为源和目标数据的传输通道
rainmom.sources.r1.channels= c1
rainmom.sinks.k1.channel = c1
启动 Flume Agent a1 日志控制台
[hadoop@master ~]$ flume-ng agent --conf-file /usr/local/src/flume/rainmom-flume.conf --name rainmom
Warning: No configuration directory set! Use --conf <dir> to override.
Info: Including Hadoop libraries found via (/usr/local/src/hadoop/bin/hadoop) for HDFS access
Info: Excluding /usr/local/src/hadoop/share/hadoop/common/lib/slf4j-api-1.7.10.jar from classpath
Info: Excluding /usr/local/src/hadoop/share/hadoop/common/lib/slf4j-log4j12-1.7.10.jar from classpath
Info: Including HBASE libraries found via (/usr/local/src/hbase/bin/hbase) for HBASE access
Info: Excluding /usr/local/src/hbase/lib/slf4j-api-1.7.7.jar from classpath
Info: Excluding /usr/local/src/hbase/lib/slf4j-log4j12-1.7.5.jar from classpath
Info: Excluding /usr/local/src/hadoop/share/hadoop/common/lib/slf4j-api-1.7.10.jar from classpath
Info: Excluding /usr/local/src/hadoop/share/hadoop/common/lib/slf4j-log4j12-1.7.10.jar from classpath
Info: Including Hive libraries found via (/usr/local/src/hive) for Hive access
..................
查看结果
[hadoop@master ~]$ hdfs dfs -ls /tmp
Found 3 items
drwx-wx-wx - hadoop supergroup 0 2023-04-19 10:37 /tmp/flume
drwx------ - hadoop supergroup 0 2023-04-12 13:47 /tmp/hadoop-yarn
drwx-wx-wx - hadoop supergroup 0 2023-03-25 18:51 /tmp/hive
