大数据平台运行监控

0|1查看Linux系统的信息

[root@master ~]# uname -a Linux master 3.10.0-862.el7.x86_64 #1 SMP Fri Apr 20 16:44:24 UTC 2018 x86_64 x86_64 x86_64 GNU/Linux

1|0查看硬盘信息

1|0查看所有分区

[root@master ~]# fdisk -l Disk /dev/sda: 21.5 GB, 21474836480 bytes, 41943040 sectors Units = sectors of 1 * 512 = 512 bytes Sector size (logical/physical): 512 bytes / 512 bytes I/O size (minimum/optimal): 512 bytes / 512 bytes Disk label type: dos Disk identifier: 0x0009a8cd Device Boot Start End Blocks Id System /dev/sda1 * 2048 2099199 1048576 83 Linux /dev/sda2 2099200 41943039 19921920 8e Linux LVM Disk /dev/mapper/centos-root: 18.2 GB, 18249416704 bytes, 35643392 sectors Units = sectors of 1 * 512 = 512 bytes Sector size (logical/physical): 512 bytes / 512 bytes I/O size (minimum/optimal): 512 bytes / 512 bytes Disk /dev/mapper/centos-swap: 2147 MB, 2147483648 bytes, 4194304 sectors Units = sectors of 1 * 512 = 512 bytes Sector size (logical/physical): 512 bytes / 512 bytes I/O size (minimum/optimal): 512 bytes / 512 bytes

1|0查看所有交换分区

[root@master ~]# swapon -s Filename Type Size Used Priority /dev/dm-1 partition 2097148 0 -1

1|0查看系统文件占比

[root@master ~]# df -h Filesystem Size Used Avail Use% Mounted on /dev/mapper/centos-root 17G 4.7G 13G 28% / devtmpfs 980M 0 980M 0% /dev tmpfs 992M 0 992M 0% /dev/shm tmpfs 992M 9.5M 982M 1% /run tmpfs 992M 0 992M 0% /sys/fs/cgroup /dev/sda1 1014M 130M 885M 13% /boot tmpfs 199M 0 199M 0% /run/user/0

1|1查看网络IP地址

[root@master ~]# ifconfig ens33: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500 inet 192.168.88.10 netmask 255.255.255.0 broadcast 192.168.88.255 inet6 fe80::2f46:dfe2:4827:2352 prefixlen 64 scopeid 0x20<link> ether 00:0c:29:e5:4e:63 txqueuelen 1000 (Ethernet) RX packets 179 bytes 18743 (18.3 KiB) RX errors 0 dropped 0 overruns 0 frame 0 TX packets 136 bytes 19092 (18.6 KiB) TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0 lo: flags=73<UP,LOOPBACK,RUNNING> mtu 65536 inet 127.0.0.1 netmask 255.0.0.0 inet6 ::1 prefixlen 128 scopeid 0x10<host> loop txqueuelen 1000 (Local Loopback) RX packets 0 bytes 0 (0.0 B) RX errors 0 dropped 0 overruns 0 frame 0 TX packets 0 bytes 0 (0.0 B) TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0

1|2查看所有监听端口

[root@master ~]# netstat -lntp Active Internet connections (only servers) Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN 881/sshd tcp6 0 0 :::3306 :::* LISTEN 915/mysqld tcp6 0 0 :::22 :::* LISTEN 881/sshd

1|3查看所有已经建立的连接

[root@master ~]# netstat -antp Active Internet connections (servers and established) Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN 881/sshd tcp 0 52 192.168.88.10:22 192.168.88.1:2124 ESTABLISHED 1122/sshd: root@pts tcp6 0 0 :::3306 :::* LISTEN 915/mysqld tcp6 0 0 :::22 :::* LISTEN 881/sshd

1|4实时显示进程状态(top),该命令可以查看进程对 CPU、内存的占比

[root@master ~]# top top - 10:17:53 up 24 min, 1 user, load average: 0.00, 0.01, 0.05 Tasks: 99 total, 1 running, 98 sleeping, 0 stopped, 0 zombie %Cpu(s): 0.0 us, 0.0 sy, 0.0 ni,100.0 id, 0.0 wa, 0.0 hi, 0.0 si, 0.0 st KiB Mem : 2030172 total, 1582316 free, 281340 used, 166516 buff/cache KiB Swap: 2097148 total, 2097148 free, 0 used. 1574280 avail Mem PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND 1 root 20 0 125352 3760 2564 S 0.0 0.2 0:00.90 systemd 2 root 20 0 0 0 0 S 0.0 0.0 0:00.00 kthreadd 3 root 20 0 0 0 0 S 0.0 0.0 0:00.03 ksoftirqd/0 5 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 kworker/0:0H 7 root rt 0 0 0 0 S 0.0 0.0 0:00.02 migration/0 8 root 20 0 0 0 0 S 0.0 0.0 0:00.00 rcu_bh 9 root 20 0 0 0 0 S 0.0 0.0 0:00.22 rcu_sched 10 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 lru-add-drain 11 root rt 0 0 0 0 S 0.0 0.0 0:00.00 watchdog/0 12 root rt 0 0 0 0 S 0.0 0.0 0:00.00 watchdog/1 13 root rt 0 0 0 0 S 0.0 0.0 0:00.00 migration/1 14 root 20 0 0 0 0 S 0.0 0.0 0:00.02 ksoftirqd/1 16 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 kworker/1:0H 18 root 20 0 0 0 0 S 0.0 0.0 0:00.00 kdevtmpfs 19 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 netns 20 root 20 0 0 0 0 S 0.0 0.0 0:00.00 khungtaskd 21 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 writeback 22 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 kintegrityd 23 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 bioset 24 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 kblockd 25 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 md 26 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 edac-poller 27 root 20 0 0 0 0 S 0.0 0.0 0:01.10 kworker/0:1 32 root 20 0 0 0 0 S 0.0 0.0 0:00.00 kswapd0 33 root 25 5 0 0 0 S 0.0 0.0 0:00.00 ksmd 34 root 39 19 0 0 0 S 0.0 0.0 0:00.26 khugepaged 35 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 crypto 43 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 kthrotld 44 root 20 0 0 0 0 S 0.0 0.0 0:00.00 kworker/u256:1 45 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 kmpath_rdacd 46 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 kaluad 48 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 kpsmoused 50 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 ipv6_addrconf 63 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 deferwq 94 root 20 0 0 0 0 S 0.0 0.0 0:00.00 kauditd

1|5查看 CPU 信息

[root@master ~]# cat /proc/cpuinfo processor : 0 vendor_id : GenuineIntel cpu family : 6 model : 158 model name : Intel(R) Core(TM) i5-9500 CPU @ 3.00GHz stepping : 10 microcode : 0xca cpu MHz : 3000.000 cache size : 9216 KB physical id : 0 siblings : 1 core id : 0 cpu cores : 1 apicid : 0 initial apicid : 0 fpu : yes fpu_exception : yes cpuid level : 22 wp : yes flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon nopl xtopology tsc_reliable nonstop_tsc eagerfpu pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch fsgsbase tsc_adjust bmi1 avx2 smep bmi2 invpcid mpx rdseed adx smap clflushopt xsaveopt xsavec ibpb ibrs stibp arat spec_ctrl intel_stibp arch_capabilities bogomips : 6000.00 clflush size : 64 cache_alignment : 64 address sizes : 43 bits physical, 48 bits virtual power management: processor : 1 vendor_id : GenuineIntel cpu family : 6 model : 158 model name : Intel(R) Core(TM) i5-9500 CPU @ 3.00GHz stepping : 10 microcode : 0xca cpu MHz : 3000.000 cache size : 9216 KB physical id : 2 siblings : 1 core id : 0 cpu cores : 1 apicid : 2 initial apicid : 2 fpu : yes fpu_exception : yes cpuid level : 22 wp : yes flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon nopl xtopology tsc_reliable nonstop_tsc eagerfpu pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch fsgsbase tsc_adjust bmi1 avx2 smep bmi2 invpcid mpx rdseed adx smap clflushopt xsaveopt xsavec ibpb ibrs stibp arat spec_ctrl intel_stibp arch_capabilities bogomips : 6000.00 clflush size : 64 cache_alignment : 64 address sizes : 43 bits physical, 48 bits virtual power management:

1|6查看内存信息,可以查看总内存、空闲内存等信息

[root@master ~]# cat /proc/meminfo MemTotal: 2030172 kB MemFree: 1582904 kB MemAvailable: 1574868 kB Buffers: 2112 kB Cached: 122488 kB SwapCached: 0 kB Active: 243220 kB Inactive: 105124 kB Active(anon): 224160 kB Inactive(anon): 9312 kB Active(file): 19060 kB Inactive(file): 95812 kB Unevictable: 0 kB Mlocked: 0 kB SwapTotal: 2097148 kB SwapFree: 2097148 kB Dirty: 0 kB Writeback: 0 kB AnonPages: 223808 kB Mapped: 28564 kB Shmem: 9728 kB Slab: 41916 kB SReclaimable: 17664 kB SUnreclaim: 24252 kB KernelStack: 4256 kB PageTables: 3748 kB NFS_Unstable: 0 kB Bounce: 0 kB WritebackTmp: 0 kB CommitLimit: 3112232 kB Committed_AS: 771820 kB VmallocTotal: 34359738367 kB VmallocUsed: 179204 kB VmallocChunk: 34359310332 kB HardwareCorrupted: 0 kB AnonHugePages: 182272 kB CmaTotal: 0 kB CmaFree: 0 kB HugePages_Total: 0 HugePages_Free: 0 HugePages_Rsvd: 0 HugePages_Surp: 0 Hugepagesize: 2048 kB DirectMap4k: 65408 kB DirectMap2M: 2031616 kB DirectMap1G: 0 kB

1|7命令查看Hadoop状态

1|0切换到hadoop用户

[root@master ~]# su - hadoop Last login: Wed Apr 19 10:30:12 CST 2023 on pts/0

1|0切换到Hadoop的安装目录

[hadoop@master ~]$ cd /usr/local/src/hadoop/ [hadoop@master hadoop]$

1|0启动Hadoop

#master [hadoop@master hadoop]$ start-all.sh This script is Deprecated. Instead use start-dfs.sh and start-yarn.sh Starting namenodes on [master] master: starting namenode, logging to /usr/local/src/hadoop/logs/hadoop-hadoop-namenode-master.out 192.168.88.30: starting datanode, logging to /usr/local/src/hadoop/logs/hadoop-hadoop-datanode-slave2.out 192.168.88.20: starting datanode, logging to /usr/local/src/hadoop/logs/hadoop-hadoop-datanode-slave1.out Starting secondary namenodes [0.0.0.0] 0.0.0.0: starting secondarynamenode, logging to /usr/local/src/hadoop/logs/hadoop-hadoop-secondarynamenode-master.out starting yarn daemons starting resourcemanager, logging to /usr/local/src/hadoop/logs/yarn-hadoop-resourcemanager-master.out 192.168.88.20: nodemanager running as process 1263. Stop it first. 192.168.88.30: nodemanager running as process 1260. Stop it first. [hadoop@master hadoop]$ jps 1522 SecondaryNameNode 1335 NameNode 1674 ResourceManager 1935 Jps #slave1 [hadoop@slave1 ~]$ jps 1236 DataNode 1343 Jps #slave2 [hadoop@slave2 ~]$ jps 1337 Jps 1231 DataNode

1|0关闭Hadoop

#master [hadoop@master hadoop]$ stop-all.sh This script is Deprecated. Instead use stop-dfs.sh and stop-yarn.sh Stopping namenodes on [master] master: stopping namenode 192.168.88.20: stopping datanode 192.168.88.30: stopping datanode Stopping secondary namenodes [0.0.0.0] 0.0.0.0: stopping secondarynamenode stopping yarn daemons stopping resourcemanager 192.168.88.20: no nodemanager to stop 192.168.88.30: no nodemanager to stop no proxyserver to stop [hadoop@master hadoop]$ jps 2404 Jps #slave1 [hadoop@slave1 ~]$ jps 1416 Jps #slave2 [hadoop@slave2 ~]$ jps 1412 Jps

2|0大数据平台资源状态监控

2|1查看YARN状态

1|0切换目录

[hadoop@master ~]$ cd /usr/local/src/hadoop/ [hadoop@master hadoop]$

1|0执行start-all.sh,启动hadoop,执行start-yarn.sh启动yarn

#master [hadoop@master hadoop]$ start-all.sh This script is Deprecated. Instead use start-dfs.sh and start-yarn.sh Starting namenodes on [master] master: starting namenode, logging to /usr/local/src/hadoop/logs/hadoop-hadoop-namenode-master.out 192.168.88.20: starting datanode, logging to /usr/local/src/hadoop/logs/hadoop-hadoop-datanode-slave1.out 192.168.88.30: starting datanode, logging to /usr/local/src/hadoop/logs/hadoop-hadoop-datanode-slave2.out Starting secondary namenodes [0.0.0.0] 0.0.0.0: starting secondarynamenode, logging to /usr/local/src/hadoop/logs/hadoop-hadoop-secondarynamenode-master.out starting yarn daemons starting resourcemanager, logging to /usr/local/src/hadoop/logs/yarn-hadoop-resourcemanager-master.out 192.168.88.20: starting nodemanager, logging to /usr/local/src/hadoop/logs/yarn-hadoop-nodemanager-slave1.out 192.168.88.30: starting nodemanager, logging to /usr/local/src/hadoop/logs/yarn-hadoop-nodemanager-slave2.out [hadoop@master hadoop]$ start-yarn.sh starting yarn daemons resourcemanager running as process 2885. Stop it first. 192.168.88.30: nodemanager running as process 1564. Stop it first. 192.168.88.20: nodemanager running as process 1568. Stop it first. [hadoop@master hadoop]$ jps 3219 Jps 2885 ResourceManager 2536 NameNode 2728 SecondaryNameNode #slave1 [hadoop@slave1 ~]$ jps 1568 NodeManager 1458 DataNode 1722 Jps #slave2 [hadoop@slave2 ~]$ jps 1718 Jps 1564 NodeManager 1454 DataNode

2|2查看HDFS状态

1|0切换到hadoop目录,并查看HDFS目录

[hadoop@master hadoop]$ cd /usr/local/src/hadoop/ [hadoop@master hadoop]$ #查看 [hadoop@master hadoop]$ ./bin/hdfs dfs -ls / Found 5 items drwxr-xr-x - hadoop supergroup 0 2023-04-06 22:55 /hbase drwxr-xr-x - hadoop supergroup 0 2023-04-19 10:49 /input drwxr-xr-x - hadoop supergroup 0 2023-04-19 10:59 /output drwx-wx-wx - hadoop supergroup 0 2023-04-19 10:37 /tmp drwxr-xr-x - hadoop supergroup 0 2023-04-12 14:28 /user

1|0查看HDFS的报告

[hadoop@master hadoop]$ bin/hdfs dfsadmin -report Configured Capacity: 36477861888 (33.97 GB) Present Capacity: 30873477120 (28.75 GB) DFS Remaining: 30868611072 (28.75 GB) DFS Used: 4866048 (4.64 MB) DFS Used%: 0.02% Under replicated blocks: 161 Blocks with corrupt replicas: 0 Missing blocks: 0 Missing blocks (with replication factor 1): 0 ------------------------------------------------- Live datanodes (2): Name: 192.168.88.30:50010 (slave2) Hostname: slave2 Decommission Status : Normal Configured Capacity: 18238930944 (16.99 GB) DFS Used: 2433024 (2.32 MB) Non DFS Used: 2802823168 (2.61 GB) DFS Remaining: 15433674752 (14.37 GB) DFS Used%: 0.01% DFS Remaining%: 84.62% Configured Cache Capacity: 0 (0 B) Cache Used: 0 (0 B) Cache Remaining: 0 (0 B) Cache Used%: 100.00% Cache Remaining%: 0.00% Xceivers: 1 Last contact: Wed Apr 26 10:39:55 CST 2023 Name: 192.168.88.20:50010 (slave1) Hostname: slave1 Decommission Status : Normal Configured Capacity: 18238930944 (16.99 GB) DFS Used: 2433024 (2.32 MB) Non DFS Used: 2801561600 (2.61 GB) DFS Remaining: 15434936320 (14.37 GB) DFS Used%: 0.01% DFS Remaining%: 84.63% Configured Cache Capacity: 0 (0 B) Cache Used: 0 (0 B) Cache Remaining: 0 (0 B) Cache Used%: 100.00% Cache Remaining%: 0.00% Xceivers: 1 Last contact: Wed Apr 26 10:39:55 CST 2023

1|0查看HDFS空间情况

[hadoop@master hadoop]$ hdfs dfs -df Filesystem Size Used Available Use% hdfs://master:9000 36477861888 4866048 30868611072 0%

2|3查看HBase状态

1|0启动运行HBase

#切换到HBae的安装目录 [hadoop@master hadoop]$ cd /usr/local/src/hbase/ [hadoop@master hbase]$ #显示版本号 [hadoop@master hbase]$ hbase version HBase 1.2.1 Source code repository git://asf-dev/home/busbey/projects/hbase revision=8d8a7107dc4ccbf36a92f64675dc60392f85c015 Compiled by busbey on Wed Mar 30 11:19:21 CDT 2016 From source with checksum f4bb4a14bb4e0b72b46f729dae98a772 #启动HBse [hadoop@master hbase]$ hbase version HBase 1.2.1 Source code repository git://asf-dev/home/busbey/projects/hbase revision=8d8a7107dc4ccbf36a92f64675dc60392f85c015 Compiled by busbey on Wed Mar 30 11:19:21 CDT 2016 From source with checksum f4bb4a14bb4e0b72b46f729dae98a772 [hadoop@master hbase]$ start-hbase.sh starting master, logging to /usr/local/src/hbase/logs/hbase-hadoop-master-master.out SLF4J: Class path contains multiple SLF4J bindings. SLF4J: Found binding in [jar:file:/usr/local/src/hbase/lib/slf4j-log4j12-1.7.5.jar!/org/slf4j/impl/StaticLoggerBinder.class] SLF4J: Found binding in [jar:file:/usr/local/src/hadoop/share/hadoop/common/lib/slf4j-log4j12-1.7.10.jar!/org/slf4j/impl/StaticLoggerBinder.class] SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation. SLF4J: Actual binding is of type [org.slf4j.impl.Log4jLoggerFactory] slave2: starting regionserver, logging to /usr/local/src/hbase/logs/hbase-hadoop-regionserver-slave2.out slave1: starting regionserver, logging to /usr/local/src/hbase/logs/hbase-hadoop-regionserver-slave1.out

1|0查看HBase版本信息与状态

#进入hbase shell [hadoop@master hbase]$ hbase shell SLF4J: Class path contains multiple SLF4J bindings. SLF4J: Found binding in [jar:file:/usr/local/src/hbase/lib/slf4j-log4j12-1.7.5.jar!/org/slf4j/impl/StaticLoggerBinder.class] SLF4J: Found binding in [jar:file:/usr/local/src/hadoop/share/hadoop/common/lib/slf4j-log4j12-1.7.10.jar!/org/slf4j/impl/StaticLoggerBinder.class] SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation. SLF4J: Actual binding is of type [org.slf4j.impl.Log4jLoggerFactory] HBase Shell; enter 'help<RETURN>' for list of supported commands. Type "exit<RETURN>" to leave the HBase Shell Version 1.2.1, r8d8a7107dc4ccbf36a92f64675dc60392f85c015, Wed Mar 30 11:19:21 CDT 2016 hbase(main):001:0> version 1.2.1, r8d8a7107dc4ccbf36a92f64675dc60392f85c015, Wed Mar 30 11:19:21 CDT 2016 #查看状态,查看前需要开启zookeeper,否则会报错 hbase(main):002:0> status 1 active master, 0 backup masters, 2 servers, 0 dead, 1.0000 average load #简单查看状态 hbase(main):003:0> status 'simple' active master: master:16000 1682585254586 0 backup masters 2 live servers slave1:16020 1682585256209 requestsPerSecond=0.0, numberOfOnlineRegions=0, usedHeapMB=9, maxHeapMB=235, numberOfStores=0, numberOfStorefiles=0, storefileUncompressedSizeMB=0, storefileSizeMB=0, memstoreSizeMB=0, storefileIndexSizeMB=0, readRequestsCount=0, writeRequestsCount=0, rootIndexSizeKB=0, totalStaticIndexSizeKB=0, totalStaticBloomSizeKB=0, totalCompactingKVs=0, currentCompactedKVs=0, compactionProgressPct=NaN, coprocessors=[] slave2:16020 1682585255941 requestsPerSecond=0.0, numberOfOnlineRegions=2, usedHeapMB=20, maxHeapMB=235, numberOfStores=2, numberOfStorefiles=3, storefileUncompressedSizeMB=0, storefileSizeMB=0, memstoreSizeMB=0, storefileIndexSizeMB=0, readRequestsCount=10, writeRequestsCount=1, rootIndexSizeKB=0, totalStaticIndexSizeKB=0, totalStaticBloomSizeKB=0, totalCompactingKVs=0, currentCompactedKVs=0, compactionProgressPct=NaN, coprocessors=[MultiRowMutationEndpoint] 0 dead servers Aggregate load: 0, regions: 2 #显示所有关于状态的命令 hbase(main):004:0> help 'status' Show cluster status. Can be 'summary', 'simple', 'detailed', or 'replication'. The default is 'summary'. Examples: hbase> status hbase> status 'simple' hbase> status 'summary' hbase> status 'detailed' hbase> status 'replication' hbase> status 'replication', 'source' hbase> status 'replication', 'sink'

1|0停止HBase服务

[root@master ~]# stop-hbase.sh stopping hbase............... [root@master ~]# jps 2480 NameNode 2825 ResourceManager 2668 SecondaryNameNode 3486 QuorumPeerMain 4447 Jps

2|4查看Hive状态

1|0启动运行Hive

#切换目录 [root@master ~]# cd /usr/local/src/hive/ #启动hive [root@master hive]# hive SLF4J: Class path contains multiple SLF4J bindings. SLF4J: Found binding in [jar:file:/usr/local/src/hive/lib/hive-jdbc-2.0.0-standalone.jar!/org/slf4j/impl/StaticLoggerBinder.class] SLF4J: Found binding in [jar:file:/usr/local/src/hive/lib/log4j-slf4j-impl-2.4.1.jar!/org/slf4j/impl/StaticLoggerBinder.class] SLF4J: Found binding in [jar:file:/usr/local/src/hadoop/share/hadoop/common/lib/slf4j-log4j12-1.7.10.jar!/org/slf4j/impl/StaticLoggerBinder.class] SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation. SLF4J: Actual binding is of type [org.apache.logging.slf4j.Log4jLoggerFactory] Logging initialized using configuration in jar:file:/usr/local/src/hive/lib/hive-common-2.0.0.jar!/hive-log4j2.properties Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases. hive>

1|0Hive基本操作命令

1|0查看数据库
hive> show databases; OK default sample Time taken: 0.689 seconds, Fetched: 2 row(s)
1|0查看default数据库所有表
hive> use default; OK Time taken: 0.011 seconds hive> show tables; OK student test Time taken: 0.11 seconds, Fetched: 2 row(s)
1|0创建表stu,表的id为整数型,name为字符型
hive> create table stu(id int,name string); OK Time taken: 0.281 seconds
1|0为表stu插入一条信息,id号为001,name为张三
hive> insert into stu values (1001,"zhangsan"); WARNING: Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases. Query ID = root_20230427170324_c3499e91-7cf9-4967-b7e3-9a12afa2912b Total jobs = 3 Launching Job 1 out of 3 Number of reduce tasks is set to 0 since there's no reduce operator Starting Job = job_1682584839688_0001, Tracking URL = http://master:8088/proxy/application_1682584839688_0001/ Kill Command = /usr/local/src/hadoop/bin/hadoop job -kill job_1682584839688_0001 Hadoop job information for Stage-1: number of mappers: 1; number of reducers: 0 2023-04-27 17:08:18,607 Stage-1 map = 0%, reduce = 0% 2023-04-27 17:08:23,842 Stage-1 map = 100%, reduce = 0%, Cumulative CPU 0.85 sec MapReduce Total cumulative CPU time: 850 msec Ended Job = job_1682584839688_0001 Stage-4 is selected by condition resolver. Stage-3 is filtered out by condition resolver. Stage-5 is filtered out by condition resolver. Moving data to: hdfs://master:9000/user/hive/warehouse/stu/.hive-staging_hive_2023-04-27_17-08-10_251_7508438910956791798-1/-ext-10000 Loading data to table default.stu MapReduce Jobs Launched: Stage-Stage-1: Map: 1 Cumulative CPU: 0.85 sec HDFS Read: 4132 HDFS Write: 81 SUCCESS Total MapReduce CPU Time Spent: 850 msec OK Time taken: 14.815 seconds #按照以上操作,继续插入两条信息:id 和 name 分别为 1002、1003 和 lisi、wangwu hive> insert into stu values (1002,"lisi"); WARNING: Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases. Query ID = root_20230427170324_c3499e91-7cf9-4967-b7e3-9a12afa2912b Total jobs = 3 Launching Job 1 out of 3 Number of reduce tasks is set to 0 since there's no reduce operator Starting Job = job_1682584839688_0002, Tracking URL = http://master:8088/proxy/application_1682584839688_0002/ Kill Command = /usr/local/src/hadoop/bin/hadoop job -kill job_1682584839688_0002 Hadoop job information for Stage-1: number of mappers: 1; number of reducers: 0 2023-04-27 17:10:30,381 Stage-1 map = 0%, reduce = 0% 2023-04-27 17:10:35,577 Stage-1 map = 100%, reduce = 0%, Cumulative CPU 1.15 sec MapReduce Total cumulative CPU time: 1 seconds 150 msec Ended Job = job_1682584839688_0002 Stage-4 is selected by condition resolver. Stage-3 is filtered out by condition resolver. Stage-5 is filtered out by condition resolver. Moving data to: hdfs://master:9000/user/hive/warehouse/stu/.hive-staging_hive_2023-04-27_17-10-25_571_3098162169129405922-1/-ext-10000 Loading data to table default.stu MapReduce Jobs Launched: Stage-Stage-1: Map: 1 Cumulative CPU: 1.15 sec HDFS Read: 4234 HDFS Write: 77 SUCCESS Total MapReduce CPU Time Spent: 1 seconds 150 msec OK Time taken: 12.172 seconds hive> insert into stu values (1003,"wangwu"); WARNING: Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases. Query ID = root_20230427170324_c3499e91-7cf9-4967-b7e3-9a12afa2912b Total jobs = 3 Launching Job 1 out of 3 Number of reduce tasks is set to 0 since there's no reduce operator Starting Job = job_1682584839688_0003, Tracking URL = http://master:8088/proxy/application_1682584839688_0003/ Kill Command = /usr/local/src/hadoop/bin/hadoop job -kill job_1682584839688_0003 Hadoop job information for Stage-1: number of mappers: 1; number of reducers: 0 2023-04-27 17:11:15,711 Stage-1 map = 0%, reduce = 0% 2023-04-27 17:11:20,935 Stage-1 map = 100%, reduce = 0%, Cumulative CPU 1.03 sec MapReduce Total cumulative CPU time: 1 seconds 30 msec Ended Job = job_1682584839688_0003 Stage-4 is selected by condition resolver. Stage-3 is filtered out by condition resolver. Stage-5 is filtered out by condition resolver. Moving data to: hdfs://master:9000/user/hive/warehouse/stu/.hive-staging_hive_2023-04-27_17-11-10_851_3623561733284738226-1/-ext-10000 Loading data to table default.stu MapReduce Jobs Launched: Stage-Stage-1: Map: 1 Cumulative CPU: 1.03 sec HDFS Read: 4236 HDFS Write: 79 SUCCESS Total MapReduce CPU Time Spent: 1 seconds 30 msec OK Time taken: 11.254 seconds
1|0插入数据后查看表的信息
hive> show tables; OK stu student test values__tmp__table__1 values__tmp__table__2 values__tmp__table__3 Time taken: 0.008 seconds, Fetched: 6 row(s)
1|0查看表结构
hive> desc stu; OK id int name string Time taken: 0.019 seconds, Fetched: 2 row(s)
1|0查看表stu内容
hive> select * from stu; OK 1001 zhangsan 1002 lisi 1003 wangwu Time taken: 0.042 seconds, Fetched: 3 row(s)

1|0查看文件系统和历史命令

1|0查看本地文件系统
#在命令前加!表示可以不退出hive命令行界面去执行主机的命令,这里就是在不退出hive命令行的情况下查看/usr/local/src文件夹下面有什么 hive> ! ls /usr/local/src; flume hadoop hbase hive jdk sqoop zookeeper
1|0查看HDFS文件系统
hive> dfs -ls /; Found 5 items drwxr-xr-x - hadoop supergroup 0 2023-04-27 16:47 /hbase drwxr-xr-x - hadoop supergroup 0 2023-04-19 10:49 /input drwxr-xr-x - hadoop supergroup 0 2023-04-19 10:59 /output drwx-wx-wx - hadoop supergroup 0 2023-04-19 10:37 /tmp drwxr-xr-x - hadoop supergroup 0 2023-04-12 14:28 /user
1|0查看在 Hive 中输入的所有历史命令
[root@master hive]# cd /home/hadoop/ [root@master hadoop]# cat .hivehistory exit quit create database sample; show databases; use sample; create table student(number STRING,name STRING) create table student(number STRING,name STRING); exit show databases; use sample; show tables; select * form student; exit quit show databases; use sample; show tables; select * from student;

3|0大数据平台服务状态

3|1查看Zookeeper状态

1|0查看 ZooKeeper 状态,执行命令 zkServer.sh status

[root@master ~]# zkServer.sh status ZooKeeper JMX enabled by default Using config: /usr/local/src/zookeeper/bin/../conf/zoo.cfg Mode: follower [root@slave1 ~]# zkServer.sh status ZooKeeper JMX enabled by default Using config: /usr/local/src/zookeeper/bin/../conf/zoo.cfg Mode: leader [root@slave2 tmp]# zkServer.sh status ZooKeeper JMX enabled by default Using config: /usr/local/src/zookeeper/bin/../conf/zoo.cfg Mode: follower

1|0查看运行进程

[root@master ~]# jps 2480 NameNode 5027 Jps 2825 ResourceManager 2668 SecondaryNameNode 3486 QuorumPeerMain [root@slave1 ~]# jps 1379 DataNode 1765 QuorumPeerMain 1480 NodeManager 2200 Jps [root@slave2 ~]# jps 1377 DataNode 2900 Jps 1765 QuorumPeerMain 1478 NodeManager

1|0输入命令 zkCli.sh,连接到ZooKeeper 服务

[root@master ~]# zkCli.sh Connecting to localhost:2181 2023-04-27 17:31:19,471 [myid:] - INFO [main:Environment@100] - Client environment:zookeeper.version=3.4.8--1, built on 02/06/2016 03:18 GMT 2023-04-27 17:31:19,473 [myid:] - INFO [main:Environment@100] - Client environment:host.name=master 2023-04-27 17:31:19,474 [myid:] - INFO [main:Environment@100] - Client environment:java.version=1.8.0_152 2023-04-27 17:31:19,475 [myid:] - INFO [main:Environment@100] - Client environment:java.vendor=Oracle Corporation 2023-04-27 17:31:19,475 [myid:] - INFO [main:Environment@100] - Client environment:java.home=/usr/local/src/jdk/jre 2023-04-27 17:31:19,475 [myid:] - INFO [main:Environment@100] - Client environment:java.class.path=/usr/local/src/zookeeper/bin/../build/classes:/usr/local/src/zookeeper/bin/../build/lib/*.jar:/usr/local/src/zookeeper/bin/../lib/slf4j-log4j12-1.6.1.jar:/usr/local/src/zookeeper/bin/../lib/slf4j-api-1.6.1.jar:/usr/local/src/zookeeper/bin/../lib/netty-3.7.0.Final.jar:/usr/local/src/zookeeper/bin/../lib/log4j-1.2.16.jar:/usr/local/src/zookeeper/bin/../lib/jline-0.9.94.jar:/usr/local/src/zookeeper/bin/../zookeeper-3.4.8.jar:/usr/local/src/zookeeper/bin/../src/java/lib/*.jar:/usr/local/src/zookeeper/bin/../conf::/usr/local/src/sqoop/lib 2023-04-27 17:31:19,475 [myid:] - INFO [main:Environment@100] - Client environment:java.library.path=/usr/java/packages/lib/amd64:/usr/lib64:/lib64:/lib:/usr/lib 2023-04-27 17:31:19,475 [myid:] - INFO [main:Environment@100] - Client environment:java.io.tmpdir=/tmp 2023-04-27 17:31:19,475 [myid:] - INFO [main:Environment@100] - Client environment:java.compiler=<NA> 2023-04-27 17:31:19,475 [myid:] - INFO [main:Environment@100] - Client environment:os.name=Linux 2023-04-27 17:31:19,475 [myid:] - INFO [main:Environment@100] - Client environment:os.arch=amd64 2023-04-27 17:31:19,475 [myid:] - INFO [main:Environment@100] - Client environment:os.version=3.10.0-862.el7.x86_64 2023-04-27 17:31:19,475 [myid:] - INFO [main:Environment@100] - Client environment:user.name=root 2023-04-27 17:31:19,475 [myid:] - INFO [main:Environment@100] - Client environment:user.home=/root 2023-04-27 17:31:19,475 [myid:] - INFO [main:Environment@100] - Client environment:user.dir=/root 2023-04-27 17:31:19,476 [myid:] - INFO [main:ZooKeeper@438] - Initiating client connection, connectString=localhost:2181 sessionTimeout=30000 watcher=org.apache.zookeeper.ZooKeeperMain$MyWatcher@69d0a921 Welcome to ZooKeeper! 2023-04-27 17:31:19,488 [myid:] - INFO [main-SendThread(localhost:2181):ClientCnxn$SendThread@1032] - Opening socket connection to server localhost/127.0.0.1:2181. Will not attempt to authenticate using SASL (unknown error) JLine support is enabled 2023-04-27 17:31:19,531 [myid:] - INFO [main-SendThread(localhost:2181):ClientCnxn$SendThread@876] - Socket connection established to localhost/127.0.0.1:2181, initiating session [zk: localhost:2181(CONNECTING) 0] 2023-04-27 17:31:19,544 [myid:] - INFO [main-SendThread(localhost:2181):ClientCnxn$SendThread@1299] - Session establishment complete on server localhost/127.0.0.1:2181, sessionid = 0x187c1e45c740006, negotiated timeout = 30000 WATCHER:: WatchedEvent state:SyncConnected type:None path:null

1|0使用 Watch 监听/hbase 目录,一旦/hbase 内容有变化,将会有提示,打开监视,执行命令 get /hbase 1

[zk: localhost:2181(CONNECTED) 0] get /hbase 1 cZxid = 0x200000002 ctime = Thu Apr 06 22:47:39 CST 2023 mZxid = 0x200000002 mtime = Thu Apr 06 22:47:39 CST 2023 pZxid = 0x500000057 cversion = 28 dataVersion = 0 aclVersion = 0 ephemeralOwner = 0x0 dataLength = 0 numChildren = 14 [zk: localhost:2181(CONNECTED) 2] set /hbase value-update cZxid = 0x200000002 ctime = Thu Apr 06 22:47:39 CST 2023 mZxid = 0x50000005f mtime = Thu Apr 27 17:36:42 CST 2023 pZxid = 0x500000057 cversion = 28 dataVersion = 2 aclVersion = 0 ephemeralOwner = 0x0 dataLength = 12 numChildren = 14

结果显示,当执行命令 set /hbase value-update 后,数据版本由 0 变成 1,说明 /hbase 处于监控中。

3|2命令查看 Sqoop 状态

1|0查询 Sqoop 版本号,验证 Sqoop 是否启动成功

[hadoop@master ~]$ cd /usr/local/src/sqoop/ [hadoop@master sqoop]$ ./bin/sqoop-version Warning: /usr/local/src/sqoop/../hcatalog does not exist! HCatalog jobs will fail. Please set $HCAT_HOME to the root of your HCatalog installation. Warning: /usr/local/src/sqoop/../accumulo does not exist! Accumulo imports will fail. Please set $ACCUMULO_HOME to the root of your Accumulo installation. 23/05/02 09:34:17 INFO sqoop.Sqoop: Running Sqoop version: 1.4.7 Sqoop 1.4.7 git commit id 2328971411f57f0cb683dfb79d19d4d19d185dd8 Compiled by maugli on Thu Dec 21 15:59:58 STD 2017

结果显示:Sqoop 1.4.7,说明 Sqoop 版本号为 1.4.7,并启动成功

1|0测试 Sqoop 是否能够成功连接数据库

[hadoop@master sqoop]$ bin/sqoop list-databases --connect jdbc:mysql://master:3306/ --username root --password Password@123_ Warning: /usr/local/src/sqoop/../hcatalog does not exist! HCatalog jobs will fail. Please set $HCAT_HOME to the root of your HCatalog installation. Warning: /usr/local/src/sqoop/../accumulo does not exist! Accumulo imports will fail. Please set $ACCUMULO_HOME to the root of your Accumulo installation. 23/05/02 09:52:43 INFO sqoop.Sqoop: Running Sqoop version: 1.4.7 23/05/02 09:52:43 WARN tool.BaseSqoopTool: Setting your password on the command-line is insecure. Consider using -P instead. 23/05/02 09:52:43 INFO manager.MySQLManager: Preparing to use a MySQL streaming resultset. Tue May 02 09:52:43 CST 2023 WARN: Establishing SSL connection without server's identity verification is not recommended. According to MySQL 5.5.45+, 5.6.26+ and 5.7.6+ requirements SSL connection must be established by default if explicit option isn't set. For compliance with existing applications not using SSL the verifyServerCertificate property is set to 'false'. You need either to explicitly disable SSL by setting useSSL=false, or set useSSL=true and provide truststore for server certificate verification. information_schema hive mysql performance_schema sample sys

结果显示,可以连接到 MySQL,并查看到 Master 主机中 MySQL 的所有库实例,如 information_schema、hive、mysql、performance_schema 和 sys 等数据库

1|0执行命令 sqoop help,可以看到如下内容,代表 Sqoop 启动成功

[hadoop@master sqoop]$ sqoop help Warning: /usr/local/src/sqoop/../hcatalog does not exist! HCatalog jobs will fail. Please set $HCAT_HOME to the root of your HCatalog installation. Warning: /usr/local/src/sqoop/../accumulo does not exist! Accumulo imports will fail. Please set $ACCUMULO_HOME to the root of your Accumulo installation. 23/05/02 09:54:19 INFO sqoop.Sqoop: Running Sqoop version: 1.4.7 usage: sqoop COMMAND [ARGS] Available commands: codegen Generate code to interact with database records create-hive-table Import a table definition into Hive eval Evaluate a SQL statement and display the results export Export an HDFS directory to a database table help List available commands import Import a table from a database to HDFS import-all-tables Import tables from a database to HDFS import-mainframe Import datasets from a mainframe server to HDFS job Work with saved jobs list-databases List available databases on a server list-tables List available tables in a database merge Merge results of incremental imports metastore Run a standalone Sqoop metastore version Display version information See 'sqoop help COMMAND' for information on a specific command.

3|3命令查看 Flume 状态

1|0检查 Flume 安装是否成功,执行 flume-ng version 命令,查看 Flume 的版本

[hadoop@master flume]$ flume-ng version Flume 1.6.0 Source code repository: https://git-wip-us.apache.org/repos/asf/flume.git Revision: 2561a23240a71ba20bf288c7c2cda88f443c2080 Compiled by hshreedharan on Mon May 11 11:15:44 PDT 2015 From source ith checksum b29e416802ce9ece3269d34233baf43f

1|0查看.conf文件,文件名不唯一,若没有就创建并写入如下cat命令所显示的内容

[hadoop@master flume]$ cd /usr/local/src/flume/ [hadoop@master flume]$ ls bin CHANGELOG conf DEVNOTES docs lib LICENSE NOTICE rainmom-flume.conf README RELEASE-NOTES tools [hadoop@master flume]$ cat rainmom-flume.conf # rainmom是代理名,r1,k1,c1 是 a1 的三个组件,代理名不唯一,可自行定义,但在之后执行启动命令时要输入自己定义的正确的名称,这里使用的是雨妈作为代理名 rainmom.sources = r1 rainmom.sinks = k1 rainmom.channels = c1 # 设置 r1 源文件的类型、路径和文件头属性 rainmom.sources.r1.type = spooldir rainmom.sources.r1.spoolDir = /usr/local/src/hadoop/logs/ rainmom.sources.r1.fileHeader = true # 设置 k1 目标存储器属性 rainmom.sinks.k1.type = hdfs # 目标存储器类型 hdfs rainmom.sinks.k1.hdfs.path = hdfs://master:9000/tmp/flume # 目标存储位置 rainmom.sinks.k1.hdfs.rollsize = 1048760 #临时文件达 1048760 bytes 时,滚动形 成目标文件 rainmom.sinks.k1.hdfs.rollCount = 0 #0 表示不根据 events 数量来滚动形成目标文件 rainmom.sinks.k1.hdfs.rollInterval = 900 # 间隔 900 秒将临时文件滚动形成目标文 rainmom.sinks.k1.hdfs.useLocalTimeStamp = true # 使用本地时间戳 # 设置 c1 暂存容器属性 rainmom.channels.c1.type = file # 使用文件作为暂存容器 rainmom.channels.c1.capacity = 1000 rainmom.channels.c1.transactionCapacity = 100 # 使用 c1 作为源和目标数据的传输通道 rainmom.sources.r1.channels= c1 rainmom.sinks.k1.channel = c1

1|0启动 Flume Agent a1 日志控制台

[hadoop@master ~]$ flume-ng agent --conf-file /usr/local/src/flume/rainmom-flume.conf --name rainmom Warning: No configuration directory set! Use --conf <dir> to override. Info: Including Hadoop libraries found via (/usr/local/src/hadoop/bin/hadoop) for HDFS access Info: Excluding /usr/local/src/hadoop/share/hadoop/common/lib/slf4j-api-1.7.10.jar from classpath Info: Excluding /usr/local/src/hadoop/share/hadoop/common/lib/slf4j-log4j12-1.7.10.jar from classpath Info: Including HBASE libraries found via (/usr/local/src/hbase/bin/hbase) for HBASE access Info: Excluding /usr/local/src/hbase/lib/slf4j-api-1.7.7.jar from classpath Info: Excluding /usr/local/src/hbase/lib/slf4j-log4j12-1.7.5.jar from classpath Info: Excluding /usr/local/src/hadoop/share/hadoop/common/lib/slf4j-api-1.7.10.jar from classpath Info: Excluding /usr/local/src/hadoop/share/hadoop/common/lib/slf4j-log4j12-1.7.10.jar from classpath Info: Including Hive libraries found via (/usr/local/src/hive) for Hive access ..................

1|0查看结果

[hadoop@master ~]$ hdfs dfs -ls /tmp Found 3 items drwx-wx-wx - hadoop supergroup 0 2023-04-19 10:37 /tmp/flume drwx------ - hadoop supergroup 0 2023-04-12 13:47 /tmp/hadoop-yarn drwx-wx-wx - hadoop supergroup 0 2023-03-25 18:51 /tmp/hive

__EOF__

本文作者SkyRainmom
本文链接https://www.cnblogs.com/skyrainmom/p/17489343.html
关于博主:评论和私信会在第一时间回复。或者直接私信我。
版权声明:本博客所有文章除特别声明外,均采用 BY-NC-SA 许可协议。转载请注明出处!
声援博主:如果您觉得文章对您有帮助,可以点击文章右下角推荐一下。您的鼓励是博主的最大动力!
posted @   SkyRainmom  阅读(148)  评论(0编辑  收藏  举报
相关博文:
阅读排行:
· 被坑几百块钱后,我竟然真的恢复了删除的微信聊天记录!
· 没有Manus邀请码?试试免邀请码的MGX或者开源的OpenManus吧
· 【自荐】一款简洁、开源的在线白板工具 Drawnix
· 园子的第一款AI主题卫衣上架——"HELLO! HOW CAN I ASSIST YOU TODAY
· Docker 太简单,K8s 太复杂?w7panel 让容器管理更轻松!
点击右上角即可分享
微信分享提示