【ceph】ceph 网络问题和工具
yum install -y net-tools
netstat -anp|grep ganesha.nfsd
ceph daemon /var/run/ceph/ceph-client.admin.1114156.139819448467456.asok config get xxx
查看socket
netstat -lnxp
unix STREAM LISTENING 84402953 2946130/rados /var/run/ceph/ceph-admin.2946130.94651089703712.asok
unix STREAM LISTENING 336026461 2807156/rados /var/run/ceph/ceph-admin.2807156.94357368345376.asok
unix STREAM LISTENING 186741342 3320797/rados /var/run/ceph/ceph-admin.3320797.94106476679968.asok
unix STREAM LISTENING 97390990 690481/rados /var/run/ceph/ceph-admin.690481.94219819733792.asok
unix STREAM LISTENING 111978 1/systemd /run/systemd/private
unix STREAM LISTENING 343380102 1831050/rados /var/run/ceph/ceph-admin.1831050.94326674244384.asok
unix STREAM LISTENING 224768569 3726894/rados /var/run/ceph/ceph-admin.3726894.94886983721760.asok
unix STREAM LISTENING 150163438 3744464/rados /var/run/ceph/ceph-admin.3744464.93915040211744.asok
unix STREAM LISTENING 111987 1/systemd /run/lvm/lvmetad.socket
unix STREAM LISTENING 74050952 2166906/rados /var/run/ceph/ceph-admin.2166906.93862492377888.asok
unix STREAM LISTENING 69560405 21215/rados /var/run/ceph/ceph-admin.21215.94443882525472.asok
unix STREAM LISTENING 380506858 1926693/rados /var/run/ceph/ceph-admin.1926693.94827481985824.asok
unix STREAM LISTENING 241625897 2794366/rados /var/run/ceph/ceph-admin.2794366.94245238936352.asokceph daemon /var/run/ceph/ceph-admin.2794366.94245238936352.asok status
ceph --admin-daemon ceph-osd.1.asok config show
ceph --admin-daemon ceph-mon.node1.asok config show
[root@rdma55 system]# ceph daemon /var/run/ceph/ceph-admin.2794366.94245238936352.asok help
{
"config diff": "dump diff of current config and default config",
"config diff get": "dump diff get <field>: dump diff of current and default config setting <field>",
"config get": "config get <field>: get the config value",
"config help": "get config setting schema and descriptions",
"config set": "config set <field> <val> [<val> ...]: set a config variable",
"config show": "dump current config settings",
"get_command_descriptions": "list available commands",
"git_version": "get git sha1",
"help": "list available commands",
"log dump": "dump recent log entries to log file",
"log flush": "flush log entries to log file",
"log reopen": "reopen log file",
"perf dump": "dump perfcounters value",
"perf histogram dump": "dump perf histogram values",
"perf histogram schema": "dump perf histogram schema",
"perf reset": "perf reset <name>: perf reset all or one perfcounter name",
"perf schema": "dump perfcounters schema",
"version": "get ceph version"
}
[root@rdma55 system]# ceph daemon /var/run/ceph/ceph-admin.2794366.94245238936352.asok version
{"version":"12.2.1-UniStorOS_V100R001B62","release":"luminous","release_type":"stable"}[root@rdma55 system]#
ceph链接查看
public_addr = 172.16.156.123
manage_addr = 172.16.31.123
cluster_addr = 192.168.156.123[root@node123 ~]# ceph osd tree
ID CLASS WEIGHT TYPE NAME STATUS REWEIGHT PRI-AFF
-11 0 root maintain
-10 1.17224 root dcache-d1
-17 1.17224 rack r1.dcache-d1
-21 0.39075 host node122.dcache-d1
5 0.09769 osd.5 up 1.00000 1.00000
11 0.09769 osd.11 up 1.00000 1.00000
17 0.09769 osd.17 up 1.00000 1.00000
23 0.09769 osd.23 up 1.00000 1.00000
-23 0.39075 host node123.dcache-d1
4 0.09769 osd.4 up 1.00000 1.00000
10 0.09769 osd.10 up 1.00000 1.00000
16 0.09769 osd.16 up 1.00000 1.00000
22 0.09769 osd.22 up 1.00000 1.00000
-25 0.39075 host node124.dcache-d1
3 0.09769 osd.3 up 1.00000 1.00000
9 0.09769 osd.9 up 1.00000 1.00000
15 0.09769 osd.15 up 1.00000 1.00000
21 0.09769 osd.21 up 1.00000 1.00000
-9 9.73792 root d1
-12 9.73792 rack r1.d1
-19 3.24597 host node122.d1
2 nvme 0.81149 osd.2 up 1.00000 1.00000
8 nvme 0.81149 osd.8 up 1.00000 1.00000
14 nvme 0.81149 osd.14 up 1.00000 1.00000
20 nvme 0.81149 osd.20 up 1.00000 1.00000
-7 3.24597 host node123.d1
1 nvme 0.81149 osd.1 up 1.00000 1.00000
7 nvme 0.81149 osd.7 up 1.00000 1.00000
13 nvme 0.81149 osd.13 up 1.00000 1.00000
19 nvme 0.81149 osd.19 up 1.00000 1.00000
-4 3.24597 host node124.d1
0 nvme 0.81149 osd.0 up 1.00000 1.00000
6 nvme 0.81149 osd.6 up 1.00000 1.00000
12 nvme 0.81149 osd.12 up 1.00000 1.00000
18 nvme 0.81149 osd.18 up 1.00000 1.00000
-1 0 root default
[root@node123 ~]# ceph daemon osd.1 messenger dump
{
"cluster": {
"name": "osd.1",
"addr": "192.168.156.123:6808/1060851",
"network stack type": "posix",
"number of established connection": 12,
"number of accepting connection": 0,
"number of deleted connection": 1
},
"client": {
"name": "osd.1",
"addr": "172.16.156.123:6819/1060851",
"network stack type": "posix",
"number of established connection": 7,
"number of accepting connection": 0,
"number of deleted connection": 2
},
"ms_objecter": {
"name": "osd.1",
"addr": "-",
"network stack type": "posix",
"number of established connection": 0,
"number of accepting connection": 0,
"number of deleted connection": 0
},
"hb_front_client": {
"name": "osd.1",
"addr": "172.16.156.123:0/1060851",
"network stack type": "posix",
"number of established connection": 14,
"number of accepting connection": 0,
"number of deleted connection": 4
},
"hb_back_client": {
"name": "osd.1",
"addr": "192.168.156.123:0/1060851",
"network stack type": "posix",
"number of established connection": 14,
"number of accepting connection": 0,
"number of deleted connection": 4
},
"hb_front_server": {
"name": "osd.1",
"addr": "172.16.156.123:6820/1060851",
"network stack type": "posix",
"number of established connection": 11,
"number of accepting connection": 0,
"number of deleted connection": 3
},
"hb_back_server": {
"name": "osd.1",
"addr": "192.168.156.123:6809/1060851",
"network stack type": "posix",
"number of established connection": 11,
"number of accepting connection": 0,
"number of deleted connection": 3
}
}
[root@node123 ~]# ps -ef | grep 1060851
ceph 1060851 1 1 13:51 ? 00:00:25 /opt/h3c/bin/ceph-osd --cluster=ceph -i 1 -f --setuser ceph --setgroup ceph
root 1217911 3511408 0 14:13 pts/0 00:00:00 grep --color=auto 1060851
[root@node123 ~]#[root@node123 ~]# ceph daemon osd.1 connection dump cluster
{
"established connections": [
{
"conn": "0x7fa98dd8e000",
"peer_type": "osd",
"peer_name": "osd.18",
"peer_addr": "192.168.156.124:6814/2011059",
"state": "STATE_OPEN",
"worker_name": "io-worker",
"worker_id": 2
},
{
"conn": "0x7fa9c344f000",
"peer_type": "osd",
"peer_name": "osd.7",
"peer_addr": "192.168.156.123:6810/1060879",
"state": "STATE_OPEN",
"worker_name": "io-worker",
"worker_id": 1
},
{
"conn": "0x7fa98c8d1000",
"peer_type": "osd",
"peer_name": "osd.0",
"peer_addr": "192.168.156.124:6808/2010936",
"state": "STATE_OPEN",
"worker_name": "io-worker",
"worker_id": 1
},
{
"conn": "0x7fa9c3454000",
"peer_type": "osd",
"peer_name": "osd.13",
"peer_addr": "192.168.156.123:6812/1060936",
"state": "STATE_OPEN",
"worker_name": "io-worker",
"worker_id": 0
},
{
"conn": "0x7fa9a1474000",
"peer_type": "osd",
"peer_name": "osd.18",
"peer_addr": "192.168.156.124:6806/1530644",
"state": "STATE_CLOSED",
"worker_name": "io-worker",
"worker_id": 2
},
{
"conn": "0x7fa999f3b000",
"peer_type": "osd",
"peer_name": "osd.8",
"peer_addr": "192.168.156.122:6802/3705373",
"state": "STATE_OPEN",
"worker_name": "io-worker",
"worker_id": 0
},
{
"conn": "0x7fa98dc8e000",
"peer_type": "osd",
"peer_name": "osd.20",
"peer_addr": "192.168.156.122:6806/3705519",
"state": "STATE_OPEN",
"worker_name": "io-worker",
"worker_id": 2
},
{
"conn": "0x7fa98c8dc000",
"peer_type": "osd",
"peer_name": "osd.2",
"peer_addr": "192.168.156.122:6800/3705344",
"state": "STATE_OPEN",
"worker_name": "io-worker",
"worker_id": 2
},
{
"conn": "0x7fa9c3458000",
"peer_type": "osd",
"peer_name": "osd.19",
"peer_addr": "192.168.156.123:6814/1060973",
"state": "STATE_OPEN",
"worker_name": "io-worker",
"worker_id": 1
},
{
"conn": "0x7fa999f88000",
"peer_type": "osd",
"peer_name": "osd.14",
"peer_addr": "192.168.156.122:6804/3705438",
"state": "STATE_OPEN",
"worker_name": "io-worker",
"worker_id": 1
},
{
"conn": "0x7fa99ac4c000",
"peer_type": "osd",
"peer_name": "osd.6",
"peer_addr": "192.168.156.124:6810/2010961",
"state": "STATE_OPEN",
"worker_name": "io-worker",
"worker_id": 2
},
{
"conn": "0x7fa98dc9a000",
"peer_type": "osd",
"peer_name": "osd.12",
"peer_addr": "192.168.156.124:6812/2011020",
"state": "STATE_OPEN",
"worker_name": "io-worker",
"worker_id": 0
}
],
"accepting connections": [],
"deleted connections": [
{
"conn": "0x7fa9a1474000",
"peer_type": "osd",
"peer_name": "osd.18",
"peer_addr": "192.168.156.124:6806/1530644",
"state": "STATE_CLOSED",
"worker_name": "io-worker",
"worker_id": 2
}
]
}
[root@node123 ~]#[root@node123 ~]# ceph daemon osd.1 connection dump client
{
"established connections": [
{
"conn": "0x7fa98c891000",
"peer_type": "dse",
"peer_name": "dse.70684427",
"peer_addr": "172.16.156.122:6819/3705334",
"state": "STATE_OPEN",
"worker_name": "io-worker",
"worker_id": 0
},
{
"conn": "0x7fa9c5840000",
"peer_type": "dse",
"peer_name": "dse.70684199",
"peer_addr": "172.16.156.124:6810/2008338",
"state": "STATE_OPEN",
"worker_name": "io-worker",
"worker_id": 2
},
{
"conn": "0x7fa97647e000",
"peer_type": "mgr",
"peer_name": "mgr.70684394",
"peer_addr": "172.16.156.122:6809/3705389",
"state": "STATE_OPEN",
"worker_name": "io-worker",
"worker_id": 1
},
{
"conn": "0x7fa98dd68000",
"peer_type": "dse",
"peer_name": "dse.70684099",
"peer_addr": "172.16.156.123:6810/1058464",
"state": "STATE_OPEN",
"worker_name": "io-worker",
"worker_id": 0
},
{
"conn": "0x7fa9a740a000",
"peer_type": "mon",
"peer_name": "mon.1",
"peer_addr": "172.16.156.123:6789/0",
"state": "STATE_CLOSED",
"worker_name": "io-worker",
"worker_id": 2
},
{
"conn": "0x7fa9a7243000",
"peer_type": "mon",
"peer_name": "mon.0",
"peer_addr": "172.16.156.122:6789/0",
"state": "STATE_CLOSED",
"worker_name": "io-worker",
"worker_id": 1
},
{
"conn": "0x7fa9a71d4000",
"peer_type": "mon",
"peer_name": "mon.2",
"peer_addr": "172.16.156.124:6789/0",
"state": "STATE_OPEN",
"worker_name": "io-worker",
"worker_id": 0
}
],
"accepting connections": [],
"deleted connections": [
{
"conn": "0x7fa9a7243000",
"peer_type": "mon",
"peer_name": "mon.0",
"peer_addr": "172.16.156.122:6789/0",
"state": "STATE_CLOSED",
"worker_name": "io-worker",
"worker_id": 1
},
{
"conn": "0x7fa9a740a000",
"peer_type": "mon",
"peer_name": "mon.1",
"peer_addr": "172.16.156.123:6789/0",
"state": "STATE_CLOSED",
"worker_name": "io-worker",
"worker_id": 2
}
]
}
[root@node123 ~]#[root@node123 ~]# ceph daemon osd.1 connection dump ms_objecter
{
"established connections": [],
"accepting connections": [],
"deleted connections": []
}
[root@node123 ~]#
[root@node123 ~]# ceph daemon osd.1 connection dump hb_front_client
{
"established connections": [
{
"conn": "0x7fa9760f8000",
"peer_type": "osd",
"peer_name": "osd.6",
"peer_addr": "172.16.156.124:6822/2010961",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7fa9760e4000",
"peer_type": "osd",
"peer_name": "osd.18",
"peer_addr": "172.16.156.124:6826/2011059",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7fa9760d9000",
"peer_type": "osd",
"peer_name": "osd.0",
"peer_addr": "172.16.156.124:6820/2010936",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7fa99ad2d000",
"peer_type": "osd",
"peer_name": "osd.20",
"peer_addr": "172.16.156.122:6807/3705519",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7fa999f6f000",
"peer_type": "osd",
"peer_name": "osd.8",
"peer_addr": "172.16.156.122:6803/3705373",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7fa98dd66000",
"peer_type": "osd",
"peer_name": "osd.14",
"peer_addr": "172.16.156.122:6805/3705438",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7fa999f9f000",
"peer_type": "osd",
"peer_name": "osd.2",
"peer_addr": "172.16.156.122:6801/3705344",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7fa976068000",
"peer_type": "osd",
"peer_name": "osd.18",
"peer_addr": "172.16.156.124:6808/1530644",
"state": "STATE_CLOSED",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7fa976072000",
"peer_type": "osd",
"peer_name": "osd.6",
"peer_addr": "172.16.156.124:6804/1530563",
"state": "STATE_CLOSED",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7fa99ace1000",
"peer_type": "osd",
"peer_name": "osd.19",
"peer_addr": "172.16.156.123:6826/1060973",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7fa9760ee000",
"peer_type": "osd",
"peer_name": "osd.12",
"peer_addr": "172.16.156.124:6824/2011020",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7fa97609a000",
"peer_type": "osd",
"peer_name": "osd.7",
"peer_addr": "172.16.156.123:6822/1060879",
"state": "STATE_CLOSED",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7fa9760a4000",
"peer_type": "osd",
"peer_name": "osd.13",
"peer_addr": "172.16.156.123:6824/1060936",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7fa976086000",
"peer_type": "osd",
"peer_name": "osd.12",
"peer_addr": "172.16.156.124:6806/1530602",
"state": "STATE_CLOSED",
"worker_name": "hb-worker",
"worker_id": 0
}
],
"accepting connections": [],
"deleted connections": [
{
"conn": "0x7fa976068000",
"peer_type": "osd",
"peer_name": "osd.18",
"peer_addr": "172.16.156.124:6808/1530644",
"state": "STATE_CLOSED",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7fa976072000",
"peer_type": "osd",
"peer_name": "osd.6",
"peer_addr": "172.16.156.124:6804/1530563",
"state": "STATE_CLOSED",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7fa976086000",
"peer_type": "osd",
"peer_name": "osd.12",
"peer_addr": "172.16.156.124:6806/1530602",
"state": "STATE_CLOSED",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7fa97609a000",
"peer_type": "osd",
"peer_name": "osd.7",
"peer_addr": "172.16.156.123:6822/1060879",
"state": "STATE_CLOSED",
"worker_name": "hb-worker",
"worker_id": 0
}
]
}
[root@node123 ~]#[root@node123 ~]# ceph daemon osd.1 connection dump hb_back_client
{
"established connections": [
{
"conn": "0x7fa9760e9000",
"peer_type": "osd",
"peer_name": "osd.12",
"peer_addr": "192.168.156.124:6813/2011020",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7fa9760de000",
"peer_type": "osd",
"peer_name": "osd.18",
"peer_addr": "192.168.156.124:6815/2011059",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7fa9760d4000",
"peer_type": "osd",
"peer_name": "osd.0",
"peer_addr": "192.168.156.124:6809/2010936",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7fa99ad34000",
"peer_type": "osd",
"peer_name": "osd.2",
"peer_addr": "192.168.156.122:6801/3705344",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7fa98dd24000",
"peer_type": "osd",
"peer_name": "osd.19",
"peer_addr": "192.168.156.123:6815/1060973",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7fa9760f3000",
"peer_type": "osd",
"peer_name": "osd.6",
"peer_addr": "192.168.156.124:6811/2010961",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7fa98dd39000",
"peer_type": "osd",
"peer_name": "osd.14",
"peer_addr": "192.168.156.122:6805/3705438",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7fa97606d000",
"peer_type": "osd",
"peer_name": "osd.6",
"peer_addr": "192.168.156.124:6803/1530563",
"state": "STATE_CLOSED",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7fa976063000",
"peer_type": "osd",
"peer_name": "osd.18",
"peer_addr": "192.168.156.124:6807/1530644",
"state": "STATE_CLOSED",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7fa98c925000",
"peer_type": "osd",
"peer_name": "osd.20",
"peer_addr": "192.168.156.122:6807/3705519",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7fa976081000",
"peer_type": "osd",
"peer_name": "osd.12",
"peer_addr": "192.168.156.124:6805/1530602",
"state": "STATE_CLOSED",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7fa976095000",
"peer_type": "osd",
"peer_name": "osd.7",
"peer_addr": "192.168.156.123:6811/1060879",
"state": "STATE_CLOSED",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7fa99ad1e000",
"peer_type": "osd",
"peer_name": "osd.8",
"peer_addr": "192.168.156.122:6803/3705373",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7fa97609f000",
"peer_type": "osd",
"peer_name": "osd.13",
"peer_addr": "192.168.156.123:6813/1060936",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
}
],
"accepting connections": [],
"deleted connections": [
{
"conn": "0x7fa976063000",
"peer_type": "osd",
"peer_name": "osd.18",
"peer_addr": "192.168.156.124:6807/1530644",
"state": "STATE_CLOSED",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7fa97606d000",
"peer_type": "osd",
"peer_name": "osd.6",
"peer_addr": "192.168.156.124:6803/1530563",
"state": "STATE_CLOSED",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7fa976081000",
"peer_type": "osd",
"peer_name": "osd.12",
"peer_addr": "192.168.156.124:6805/1530602",
"state": "STATE_CLOSED",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7fa976095000",
"peer_type": "osd",
"peer_name": "osd.7",
"peer_addr": "192.168.156.123:6811/1060879",
"state": "STATE_CLOSED",
"worker_name": "hb-worker",
"worker_id": 0
}
]
}[root@node123 ~]# ceph daemon osd.1 connection dump hb_front_server
{
"established connections": [
{
"conn": "0x7fa9c3471000",
"peer_type": "osd",
"peer_name": "osd.0",
"peer_addr": "172.16.156.124:0/2010936",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7fa9c342b000",
"peer_type": "osd",
"peer_name": "osd.19",
"peer_addr": "172.16.156.123:0/1060973",
"state": "STATE_CLOSED",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7fa9c3435000",
"peer_type": "osd",
"peer_name": "osd.13",
"peer_addr": "172.16.156.123:0/1060936",
"state": "STATE_CLOSED",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7fa9c3440000",
"peer_type": "osd",
"peer_name": "osd.7",
"peer_addr": "172.16.156.123:0/1060879",
"state": "STATE_CLOSED",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7fa976043000",
"peer_type": "osd",
"peer_name": "osd.20",
"peer_addr": "172.16.156.122:0/3705519",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7fa976077000",
"peer_type": "osd",
"peer_name": "osd.8",
"peer_addr": "172.16.156.122:0/3705373",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7fa97607c000",
"peer_type": "osd",
"peer_name": "osd.2",
"peer_addr": "172.16.156.122:0/3705344",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7fa976059000",
"peer_type": "osd",
"peer_name": "osd.6",
"peer_addr": "172.16.156.124:0/2010961",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7fa976090000",
"peer_type": "osd",
"peer_name": "osd.14",
"peer_addr": "172.16.156.122:0/3705438",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7fa9c345e000",
"peer_type": "osd",
"peer_name": "osd.18",
"peer_addr": "172.16.156.124:0/2011059",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7fa9c3469000",
"peer_type": "osd",
"peer_name": "osd.12",
"peer_addr": "172.16.156.124:0/2011020",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
}
],
"accepting connections": [],
"deleted connections": [
{
"conn": "0x7fa9c342b000",
"peer_type": "osd",
"peer_name": "osd.19",
"peer_addr": "172.16.156.123:0/1060973",
"state": "STATE_CLOSED",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7fa9c3435000",
"peer_type": "osd",
"peer_name": "osd.13",
"peer_addr": "172.16.156.123:0/1060936",
"state": "STATE_CLOSED",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7fa9c3440000",
"peer_type": "osd",
"peer_name": "osd.7",
"peer_addr": "172.16.156.123:0/1060879",
"state": "STATE_CLOSED",
"worker_name": "hb-worker",
"worker_id": 0
}
]
}[root@node123 ~]# ceph daemon osd.1 connection dump hb_back_server
{
"established connections": [
{
"conn": "0x7fa9c346d000",
"peer_type": "osd",
"peer_name": "osd.0",
"peer_addr": "192.168.156.124:0/2010936",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7fa9c343b000",
"peer_type": "osd",
"peer_name": "osd.7",
"peer_addr": "192.168.156.123:0/1060879",
"state": "STATE_CLOSED",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7fa976048000",
"peer_type": "osd",
"peer_name": "osd.14",
"peer_addr": "192.168.156.122:0/3705438",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7fa97605e000",
"peer_type": "osd",
"peer_name": "osd.2",
"peer_addr": "192.168.156.122:0/3705344",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7fa976054000",
"peer_type": "osd",
"peer_name": "osd.20",
"peer_addr": "192.168.156.122:0/3705519",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7fa9c3430000",
"peer_type": "osd",
"peer_name": "osd.13",
"peer_addr": "192.168.156.123:0/1060936",
"state": "STATE_CLOSED",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7fa9c3421000",
"peer_type": "osd",
"peer_name": "osd.18",
"peer_addr": "192.168.156.124:0/2011059",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7fa97608b000",
"peer_type": "osd",
"peer_name": "osd.8",
"peer_addr": "192.168.156.122:0/3705373",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7fa9c3427000",
"peer_type": "osd",
"peer_name": "osd.19",
"peer_addr": "192.168.156.123:0/1060973",
"state": "STATE_CLOSED",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7fa97604f000",
"peer_type": "osd",
"peer_name": "osd.6",
"peer_addr": "192.168.156.124:0/2010961",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7fa9c3462000",
"peer_type": "osd",
"peer_name": "osd.12",
"peer_addr": "192.168.156.124:0/2011020",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
}
],
"accepting connections": [],
"deleted connections": [
{
"conn": "0x7fa9c3427000",
"peer_type": "osd",
"peer_name": "osd.19",
"peer_addr": "192.168.156.123:0/1060973",
"state": "STATE_CLOSED",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7fa9c3430000",
"peer_type": "osd",
"peer_name": "osd.13",
"peer_addr": "192.168.156.123:0/1060936",
"state": "STATE_CLOSED",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7fa9c343b000",
"peer_type": "osd",
"peer_name": "osd.7",
"peer_addr": "192.168.156.123:0/1060879",
"state": "STATE_CLOSED",
"worker_name": "hb-worker",
"worker_id": 0
}
]
}==========================================================================
[root@node123 ~]# lsblk
NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
sda 8:0 0 447.1G 0 disk
????sda1 8:1 0 200M 0 part
????sda2 8:2 0 1G 0 part
????sda3 8:3 0 446G 0 part
????centos-swap 253:0 0 4G 0 lvm
????centos-home 253:1 0 392G 0 lvm
????centos-root 253:2 0 50G 0 lvm
sdb 8:16 0 447.1G 0 disk
????sdb1 8:17 0 512M 0 part /boot/efi
????sdb2 8:18 0 1G 0 part /boot
????sdb3 8:19 0 50G 0 part /var/log
????sdb4 8:20 0 2G 0 part
????sdb5 8:21 0 2M 0 part
????sdb6 8:22 0 393.6G 0 part /
nvme5n1 259:0 0 931.5G 0 disk /corefile
nvme4n1 259:1 0 931.5G 0 disk
????nvme4n1p1 259:34 0 100M 0 part /var/lib/ceph/osd/ceph-13
????nvme4n1p2 259:35 0 831G 0 part
????nvme4n1p3 259:36 0 100M 0 part /var/lib/ceph/osd/ceph-16
????nvme4n1p4 259:37 0 100G 0 part
nvme0n1 259:6 0 931.5G 0 disk
????nvme0n1p1 259:22 0 100M 0 part /var/lib/ceph/osd/ceph-7
????nvme0n1p2 259:23 0 831G 0 part
????nvme0n1p3 259:24 0 100M 0 part /var/lib/ceph/osd/ceph-10
????nvme0n1p4 259:25 0 100G 0 part
nvme3n1 259:7 0 931.5G 0 disk
nvme2n1 259:12 0 931.5G 0 disk
????nvme2n1p1 259:26 0 100M 0 part /var/lib/ceph/osd/ceph-1
????nvme2n1p2 259:27 0 831G 0 part
????nvme2n1p3 259:28 0 100M 0 part /var/lib/ceph/osd/ceph-4
????nvme2n1p4 259:29 0 100G 0 part
nvme1n1 259:13 0 931.5G 0 disk
????nvme1n1p1 259:30 0 100M 0 part /var/lib/ceph/osd/ceph-19
????nvme1n1p2 259:31 0 831G 0 part
????nvme1n1p3 259:32 0 100M 0 part /var/lib/ceph/osd/ceph-22
????nvme1n1p4 259:33 0 100G 0 part
[root@node123 ~]# ceph daemon dse.$(hostname) csd 22 messenger_dump
{
"cluster": {
"name": "osd.22",
"addr": "192.168.156.123:6806/1058487",
"network stack type": "posix",
"number of established connection": 11,
"number of accepting connection": 0,
"number of deleted connection": 0
},
"client": {
"name": "osd.22",
"addr": "172.16.156.123:6817/1058487",
"network stack type": "posix",
"number of established connection": 29,
"number of accepting connection": 1,
"number of deleted connection": 4
},
"hb_front_client": {
"name": "osd.22",
"addr": "172.16.156.123:0/1058487",
"network stack type": "posix",
"number of established connection": 10,
"number of accepting connection": 0,
"number of deleted connection": 0
},
"hb_back_client": {
"name": "osd.22",
"addr": "192.168.156.123:0/1058487",
"network stack type": "posix",
"number of established connection": 10,
"number of accepting connection": 0,
"number of deleted connection": 0
},
"hb_front_server": {
"name": "osd.22",
"addr": "172.16.156.123:6818/1058487",
"network stack type": "posix",
"number of established connection": 11,
"number of accepting connection": 0,
"number of deleted connection": 0
},
"hb_back_server": {
"name": "osd.22",
"addr": "192.168.156.123:6807/1058487",
"network stack type": "posix",
"number of established connection": 11,
"number of accepting connection": 0,
"number of deleted connection": 0
}
}
[root@node123 ~]# ps -efL | grep 1058487
ceph 1058464 1 1058487 0 310 13:50 ? 00:00:00 /opt/h3c/bin/dse -f --cluster ceph --id node123 --setuser ceph --setgroup ceph
root 1267164 3511408 1267164 0 1 14:19 pts/0 00:00:00 grep --color=auto 1058487
[root@node123 ~]#
[root@node123 ~]# ceph daemon dse.$(hostname) csd 22 connection_dump
# "cluster"
{
"connections": {
"established connections": [
{
"conn": "0x7f2a6ecbf000",
"peer_type": "osd",
"peer_name": "osd.23",
"peer_addr": "192.168.156.122:6814/3705358",
"state": "STATE_OPEN",
"worker_name": "cc-worker",
"worker_id": 2
},
{
"conn": "0x7f2a6e3cf000",
"peer_type": "osd",
"peer_name": "osd.17",
"peer_addr": "192.168.156.122:6812/3705352",
"state": "STATE_OPEN",
"worker_name": "cc-worker",
"worker_id": 1
},
{
"conn": "0x7f2a6e3b3000",
"peer_type": "osd",
"peer_name": "osd.11",
"peer_addr": "192.168.156.122:6810/3705346",
"state": "STATE_OPEN",
"worker_name": "cc-worker",
"worker_id": 3
},
{
"conn": "0x7f2a4b5b3000",
"peer_type": "osd",
"peer_name": "osd.16",
"peer_addr": "192.168.156.123:6804/1058481",
"state": "STATE_OPEN",
"worker_name": "cc-worker",
"worker_id": 3
},
{
"conn": "0x7f2a4b901000",
"peer_type": "osd",
"peer_name": "unknown.0",
"peer_addr": "192.168.156.123:6802/1058475",
"state": "STATE_OPEN",
"worker_name": "cc-worker",
"worker_id": 3
},
{
"conn": "0x7f2a6e54d000",
"peer_type": "osd",
"peer_name": "osd.21",
"peer_addr": "192.168.156.124:6806/2008360",
"state": "STATE_OPEN",
"worker_name": "cc-worker",
"worker_id": 1
},
{
"conn": "0x7f2a4b5b7000",
"peer_type": "osd",
"peer_name": "osd.4",
"peer_addr": "192.168.156.123:6800/1058469",
"state": "STATE_OPEN",
"worker_name": "cc-worker",
"worker_id": 3
},
{
"conn": "0x7f2a6e63f000",
"peer_type": "osd",
"peer_name": "osd.3",
"peer_addr": "192.168.156.124:6800/2008342",
"state": "STATE_OPEN",
"worker_name": "cc-worker",
"worker_id": 3
},
{
"conn": "0x7f2a6e639000",
"peer_type": "osd",
"peer_name": "osd.15",
"peer_addr": "192.168.156.124:6804/2008354",
"state": "STATE_OPEN",
"worker_name": "cc-worker",
"worker_id": 2
},
{
"conn": "0x7f2a6e3e5000",
"peer_type": "osd",
"peer_name": "osd.5",
"peer_addr": "192.168.156.122:6808/3705340",
"state": "STATE_OPEN",
"worker_name": "cc-worker",
"worker_id": 2
},
{
"conn": "0x7f2a6e781000",
"peer_type": "osd",
"peer_name": "osd.9",
"peer_addr": "192.168.156.124:6802/2008348",
"state": "STATE_OPEN",
"worker_name": "cc-worker",
"worker_id": 1
}
],
"accepting connections": [],
"deleted connections": []
}
}
# client
{
"connections": {
"established connections": [
{
"conn": "0x7f2a4b573000",
"peer_type": "client",
"peer_name": "client.?",
"peer_addr": "172.16.156.122:0/30985474",
"state": "STATE_OPEN",
"worker_name": "cp-worker",
"worker_id": 0
},
{
"conn": "0x7f2a4b56b000",
"peer_type": "client",
"peer_name": "client.?",
"peer_addr": "172.16.156.122:0/2650456294",
"state": "STATE_OPEN",
"worker_name": "cp-worker",
"worker_id": 0
},
{
"conn": "0x7f2a4b56f000",
"peer_type": "client",
"peer_name": "client.?",
"peer_addr": "172.16.156.122:0/1393244720",
"state": "STATE_OPEN",
"worker_name": "cp-worker",
"worker_id": 1
},
{
"conn": "0x7f2a4b561000",
"peer_type": "client",
"peer_name": "client.?",
"peer_addr": "172.16.156.122:0/4175774431",
"state": "STATE_OPEN",
"worker_name": "cp-worker",
"worker_id": 0
},
{
"conn": "0x7f2a4b57b000",
"peer_type": "client",
"peer_name": "client.?",
"peer_addr": "172.16.156.122:0/1621943635",
"state": "STATE_OPEN",
"worker_name": "cp-worker",
"worker_id": 1
},
{
"conn": "0x7f2a4b55d000",
"peer_type": "client",
"peer_name": "client.?",
"peer_addr": "172.16.156.122:0/1061568166",
"state": "STATE_OPEN",
"worker_name": "cp-worker",
"worker_id": 1
},
{
"conn": "0x7f2a4b535000",
"peer_type": "client",
"peer_name": "client.?",
"peer_addr": "172.16.156.124:0/664356854",
"state": "STATE_OPEN",
"worker_name": "cp-worker",
"worker_id": 1
},
{
"conn": "0x7f2a4b517000",
"peer_type": "client",
"peer_name": "client.?",
"peer_addr": "172.16.156.123:0/2275291650",
"state": "STATE_OPEN",
"worker_name": "cp-worker",
"worker_id": 1
},
{
"conn": "0x7f2a4b553000",
"peer_type": "client",
"peer_name": "client.?",
"peer_addr": "172.16.156.124:0/1083449978",
"state": "STATE_OPEN",
"worker_name": "cp-worker",
"worker_id": 1
},
{
"conn": "0x7f2a4b51b000",
"peer_type": "client",
"peer_name": "client.?",
"peer_addr": "172.16.156.123:0/3533738144",
"state": "STATE_OPEN",
"worker_name": "cp-worker",
"worker_id": 0
},
{
"conn": "0x7f2a4b521000",
"peer_type": "client",
"peer_name": "client.?",
"peer_addr": "172.16.156.123:0/1647920866",
"state": "STATE_OPEN",
"worker_name": "cp-worker",
"worker_id": 1
},
{
"conn": "0x7f2a4b507000",
"peer_type": "client",
"peer_name": "client.?",
"peer_addr": "172.16.156.123:0/1681816379",
"state": "STATE_OPEN",
"worker_name": "cp-worker",
"worker_id": 0
},
{
"conn": "0x7f2a4b0e3000",
"peer_type": "mgr",
"peer_name": "mgr.70684394",
"peer_addr": "172.16.156.122:6809/3705389",
"state": "STATE_OPEN",
"worker_name": "cp-worker",
"worker_id": 1
},
{
"conn": "0x7f2a4b567000",
"peer_type": "client",
"peer_name": "client.?",
"peer_addr": "172.16.156.122:0/3013228352",
"state": "STATE_OPEN",
"worker_name": "cp-worker",
"worker_id": 1
},
{
"conn": "0x7f2a4b50b000",
"peer_type": "client",
"peer_name": "client.?",
"peer_addr": "172.16.156.123:0/2157587208",
"state": "STATE_OPEN",
"worker_name": "cp-worker",
"worker_id": 1
},
{
"conn": "0x7f2a4b0e7000",
"peer_type": "mon",
"peer_name": "mon.0",
"peer_addr": "172.16.156.122:6789/0",
"state": "STATE_CLOSED",
"worker_name": "cp-worker",
"worker_id": 1
},
{
"conn": "0x7f2a4b52b000",
"peer_type": "client",
"peer_name": "client.?",
"peer_addr": "172.16.156.123:0/1216331179",
"state": "STATE_OPEN",
"worker_name": "cp-worker",
"worker_id": 1
},
{
"conn": "0x7f2a4b0eb000",
"peer_type": "mon",
"peer_name": "mon.1",
"peer_addr": "172.16.156.123:6789/0",
"state": "STATE_OPEN",
"worker_name": "cp-worker",
"worker_id": 0
},
{
"conn": "0x7f2a4b0f3000",
"peer_type": "mgr",
"peer_name": "mgr.69634228",
"peer_addr": "172.16.156.122:6827/3388246",
"state": "STATE_CLOSED",
"worker_name": "cp-worker",
"worker_id": 1
},
{
"conn": "0x7f2a4b0ef000",
"peer_type": "mon",
"peer_name": "mon.2",
"peer_addr": "172.16.156.124:6789/0",
"state": "STATE_CLOSED",
"worker_name": "cp-worker",
"worker_id": 0
},
{
"conn": "0x7f2a4b52f000",
"peer_type": "client",
"peer_name": "client.?",
"peer_addr": "172.16.156.124:0/1885133971",
"state": "STATE_OPEN",
"worker_name": "cp-worker",
"worker_id": 0
},
{
"conn": "0x7f2a4b543000",
"peer_type": "client",
"peer_name": "client.?",
"peer_addr": "172.16.156.124:0/1330139901",
"state": "STATE_OPEN",
"worker_name": "cp-worker",
"worker_id": 0
},
{
"conn": "0x7f2a4b539000",
"peer_type": "client",
"peer_name": "client.?",
"peer_addr": "172.16.156.124:0/836624565",
"state": "STATE_OPEN",
"worker_name": "cp-worker",
"worker_id": 0
},
{
"conn": "0x7f2a4b547000",
"peer_type": "client",
"peer_name": "client.?",
"peer_addr": "172.16.156.124:0/2591584495",
"state": "STATE_OPEN",
"worker_name": "cp-worker",
"worker_id": 1
},
{
"conn": "0x7f2a4b53f000",
"peer_type": "client",
"peer_name": "client.?",
"peer_addr": "172.16.156.124:0/394393686",
"state": "STATE_OPEN",
"worker_name": "cp-worker",
"worker_id": 1
},
{
"conn": "0x7f2a4b54b000",
"peer_type": "client",
"peer_name": "client.?",
"peer_addr": "172.16.156.124:0/1527811008",
"state": "STATE_OPEN",
"worker_name": "cp-worker",
"worker_id": 0
},
{
"conn": "0x7f2a4b50f000",
"peer_type": "client",
"peer_name": "client.?",
"peer_addr": "172.16.156.123:0/2526918123",
"state": "STATE_OPEN",
"worker_name": "cp-worker",
"worker_id": 0
},
{
"conn": "0x7f2a4b525000",
"peer_type": "client",
"peer_name": "client.?",
"peer_addr": "172.16.156.123:0/2081169342",
"state": "STATE_OPEN",
"worker_name": "cp-worker",
"worker_id": 0
},
{
"conn": "0x7f2a4b557000",
"peer_type": "client",
"peer_name": "client.?",
"peer_addr": "172.16.156.122:0/1148027175",
"state": "STATE_OPEN",
"worker_name": "cp-worker",
"worker_id": 0
}
],
"accepting connections": [
{
"conn": "0x7f2a4b503000",
"peer_type": "unknown",
"peer_name": "unknown.0",
"peer_addr": "-",
"state": "STATE_CLOSED",
"worker_name": "cp-worker",
"worker_id": 0
}
],
"deleted connections": [
{
"conn": "0x7f2a4b0e7000",
"peer_type": "mon",
"peer_name": "mon.0",
"peer_addr": "172.16.156.122:6789/0",
"state": "STATE_CLOSED",
"worker_name": "cp-worker",
"worker_id": 1
},
{
"conn": "0x7f2a4b0ef000",
"peer_type": "mon",
"peer_name": "mon.2",
"peer_addr": "172.16.156.124:6789/0",
"state": "STATE_CLOSED",
"worker_name": "cp-worker",
"worker_id": 0
},
{
"conn": "0x7f2a4b0f3000",
"peer_type": "mgr",
"peer_name": "mgr.69634228",
"peer_addr": "172.16.156.122:6827/3388246",
"state": "STATE_CLOSED",
"worker_name": "cp-worker",
"worker_id": 1
},
{
"conn": "0x7f2a4b503000",
"peer_type": "unknown",
"peer_name": "unknown.0",
"peer_addr": "-",
"state": "STATE_CLOSED",
"worker_name": "cp-worker",
"worker_id": 0
}
]
}
}
# hb_front_client
{
"connections": {
"established connections": [
{
"conn": "0x7f2a6ec69000",
"peer_type": "osd",
"peer_name": "osd.5",
"peer_addr": "172.16.156.122:6821/3705340",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7f2a6ec35000",
"peer_type": "osd",
"peer_name": "osd.17",
"peer_addr": "172.16.156.122:6825/3705352",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7f2a6ec89000",
"peer_type": "osd",
"peer_name": "osd.23",
"peer_addr": "172.16.156.122:6827/3705358",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7f2a6ec1f000",
"peer_type": "osd",
"peer_name": "osd.11",
"peer_addr": "172.16.156.122:6823/3705346",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7f2a6e4fb000",
"peer_type": "osd",
"peer_name": "osd.21",
"peer_addr": "172.16.156.124:6818/2008360",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7f2a6e6bf000",
"peer_type": "osd",
"peer_name": "osd.15",
"peer_addr": "172.16.156.124:6816/2008354",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7f2a6e6d7000",
"peer_type": "osd",
"peer_name": "osd.9",
"peer_addr": "172.16.156.124:6814/2008348",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7f2a6e6ef000",
"peer_type": "osd",
"peer_name": "osd.3",
"peer_addr": "172.16.156.124:6812/2008342",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7f2a6e6cf000",
"peer_type": "osd",
"peer_name": "osd.10",
"peer_addr": "172.16.156.123:6814/1058475",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7f2a6e6b7000",
"peer_type": "osd",
"peer_name": "osd.16",
"peer_addr": "172.16.156.123:6816/1058481",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
}
],
"accepting connections": [],
"deleted connections": []
}
}
# hb_back_client
{
"connections": {
"established connections": [
{
"conn": "0x7f2a6e769000",
"peer_type": "osd",
"peer_name": "osd.23",
"peer_addr": "192.168.156.122:6815/3705358",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7f2a6ea33000",
"peer_type": "osd",
"peer_name": "osd.21",
"peer_addr": "192.168.156.124:6807/2008360",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7f2a6ec2d000",
"peer_type": "osd",
"peer_name": "osd.17",
"peer_addr": "192.168.156.122:6813/3705352",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7f2a6e6f3000",
"peer_type": "osd",
"peer_name": "osd.3",
"peer_addr": "192.168.156.124:6801/2008342",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7f2a6e6db000",
"peer_type": "osd",
"peer_name": "osd.9",
"peer_addr": "192.168.156.124:6803/2008348",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7f2a6e6d3000",
"peer_type": "osd",
"peer_name": "osd.10",
"peer_addr": "192.168.156.123:6803/1058475",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7f2a6ec51000",
"peer_type": "osd",
"peer_name": "osd.11",
"peer_addr": "192.168.156.122:6811/3705346",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7f2a6e6c3000",
"peer_type": "osd",
"peer_name": "osd.15",
"peer_addr": "192.168.156.124:6805/2008354",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7f2a6ec41000",
"peer_type": "osd",
"peer_name": "osd.5",
"peer_addr": "192.168.156.122:6809/3705340",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7f2a6e6bb000",
"peer_type": "osd",
"peer_name": "osd.16",
"peer_addr": "192.168.156.123:6805/1058481",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
}
],
"accepting connections": [],
"deleted connections": []
}
}
# hb_front_server
{
"connections": {
"established connections": [
{
"conn": "0x7f2a4b5df000",
"peer_type": "osd",
"peer_name": "osd.23",
"peer_addr": "172.16.156.122:0/3705358",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7f2a4b5cf000",
"peer_type": "osd",
"peer_name": "osd.5",
"peer_addr": "172.16.156.122:0/3705340",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7f2a4b5af000",
"peer_type": "osd",
"peer_name": "osd.4",
"peer_addr": "172.16.156.123:0/1058469",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7f2a4b5a7000",
"peer_type": "osd",
"peer_name": "osd.10",
"peer_addr": "172.16.156.123:0/1058475",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7f2a4b5d7000",
"peer_type": "osd",
"peer_name": "osd.17",
"peer_addr": "172.16.156.122:0/3705352",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7f2a4b58f000",
"peer_type": "osd",
"peer_name": "osd.3",
"peer_addr": "172.16.156.124:0/2008342",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7f2a4b5ab000",
"peer_type": "osd",
"peer_name": "osd.16",
"peer_addr": "172.16.156.123:0/1058481",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7f2a4b593000",
"peer_type": "osd",
"peer_name": "osd.15",
"peer_addr": "172.16.156.124:0/2008354",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7f2a4b5bf000",
"peer_type": "osd",
"peer_name": "osd.21",
"peer_addr": "172.16.156.124:0/2008360",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7f2a4b597000",
"peer_type": "osd",
"peer_name": "osd.9",
"peer_addr": "172.16.156.124:0/2008348",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7f2a4b5c7000",
"peer_type": "osd",
"peer_name": "osd.11",
"peer_addr": "172.16.156.122:0/3705346",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
}
],
"accepting connections": [],
"deleted connections": []
}
}
# hb_back_server
{
"connections": {
"established connections": [
{
"conn": "0x7f2a4b5db000",
"peer_type": "osd",
"peer_name": "osd.23",
"peer_addr": "192.168.156.122:0/3705358",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7f2a4b59f000",
"peer_type": "osd",
"peer_name": "osd.16",
"peer_addr": "192.168.156.123:0/1058481",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7f2a4b5d3000",
"peer_type": "osd",
"peer_name": "osd.17",
"peer_addr": "192.168.156.122:0/3705352",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7f2a4b59b000",
"peer_type": "osd",
"peer_name": "osd.10",
"peer_addr": "192.168.156.123:0/1058475",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7f2a4b58b000",
"peer_type": "osd",
"peer_name": "osd.9",
"peer_addr": "192.168.156.124:0/2008348",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7f2a4b583000",
"peer_type": "osd",
"peer_name": "osd.3",
"peer_addr": "192.168.156.124:0/2008342",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7f2a4b587000",
"peer_type": "osd",
"peer_name": "osd.15",
"peer_addr": "192.168.156.124:0/2008354",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7f2a4b5cb000",
"peer_type": "osd",
"peer_name": "osd.5",
"peer_addr": "192.168.156.122:0/3705340",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7f2a4b5a3000",
"peer_type": "osd",
"peer_name": "osd.4",
"peer_addr": "192.168.156.123:0/1058469",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7f2a4b5bb000",
"peer_type": "osd",
"peer_name": "osd.21",
"peer_addr": "192.168.156.124:0/2008360",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
},
{
"conn": "0x7f2a4b5c3000",
"peer_type": "osd",
"peer_name": "osd.11",
"peer_addr": "192.168.156.122:0/3705346",
"state": "STATE_OPEN",
"worker_name": "hb-worker",
"worker_id": 0
}
],
"accepting connections": [],
"deleted connections": []
}
}
===========================================================================hostip : 172.16.156.122
engine.65539.0 engine_create_state: done
engine.65539.1 engine_create_state: done
engine.65539.2 engine_create_state: done
engine.65539.3 engine_create_state: done
engine.65542.0 engine_create_state: done
engine.65542.1 engine_create_state: done
engine.65542.2 engine_create_state: done
engine.65542.3 engine_create_state: done
hostip : 172.16.156.123
engine.65539.4 engine_create_state: done
engine.65539.5 engine_create_state: done
engine.65539.6 engine_create_state: done
engine.65539.7 engine_create_state: done
engine.65542.4 engine_create_state: done
engine.65542.5 engine_create_state: done
engine.65542.6 engine_create_state: done
engine.65542.7 engine_create_state: done
hostip : 172.16.156.124
engine.65539.8 engine_create_state: done
engine.65539.9 engine_create_state: done
engine.65539.a engine_create_state: done
engine.65539.b engine_create_state: done
engine.65542.8 engine_create_state: done
engine.65542.9 engine_create_state: done
engine.65542.a engine_create_state: done
engine.65542.b engine_create_state: done
[root@node123 ~]# ceph daemon dse.$(hostname) engine 65542.7 messenger_dump
{
"cluster": { [注] dse2dse_msgr
"name": "dse.0",
"addr": "172.16.156.123:6809/1058464",
"network stack type": "posix",
"number of established connection": 4,
"number of accepting connection": 2,
"number of deleted connection": 4
},
"public": {
"name": "dse.0",
"addr": "172.16.156.123:6808/1058464",
"network stack type": "posix",
"number of established connection": 10,
"number of accepting connection": 0,
"number of deleted connection": 0
},
"objecter": {
"name": "dse.70684099",
"addr": "172.16.156.123:6810/1058464",
"network stack type": "posix",
"number of established connection": 14,
"number of accepting connection": 0,
"number of deleted connection": 2
}
}[root@node123 ~]# ps -ef | grep 1058464
ceph 1058464 1 24 13:50 ? 00:08:01 /opt/h3c/bin/dse -f --cluster ceph --id node123 --setuser ceph --setgroup ceph
root 1294093 3511408 0 14:23 pts/0 00:00:00 grep --color=auto 1058464
[root@node123 ~]#
[root@node123 ~]# ceph daemon dse.$(hostname) engine 65542.7 connection_dump
# "cluster": [注] dse2dse_msgr
{
"connections": {
"established connections": [
{
"conn": "0x7f2a4b2a9000",
"peer_type": "dse",
"peer_name": "dse.0",
"peer_addr": "172.16.156.122:6818/3705334",
"state": "STATE_STANDBY",
"worker_name": "io-worker",
"worker_id": 1
},
{
"conn": "0x7f2a4b2ed000",
"peer_type": "dse",
"peer_name": "dse.0",
"peer_addr": "172.16.156.122:6817/3386390",
"state": "STATE_CLOSED",
"worker_name": "io-worker",
"worker_id": 2
},
{
"conn": "0x7f2a4b2ad000",
"peer_type": "dse",
"peer_name": "dse.0",
"peer_addr": "172.16.156.124:6809/2008338",
"state": "STATE_STANDBY",
"worker_name": "io-worker",
"worker_id": 2
},
{
"conn": "0x7f2a4b2f1000",
"peer_type": "dse",
"peer_name": "dse.0",
"peer_addr": "172.16.156.124:6817/1534398",
"state": "STATE_CLOSED",
"worker_name": "io-worker",
"worker_id": 1
}
],
"accepting connections": [
{
"conn": "0x7f2a4b48d000",
"peer_type": "dse",
"peer_name": "unknown.0",
"peer_addr": "172.16.156.124:6809/2008338",
"state": "STATE_CLOSED",
"worker_name": "io-worker",
"worker_id": 2
},
{
"conn": "0x7f2a4b7e7000",
"peer_type": "dse",
"peer_name": "unknown.0",
"peer_addr": "172.16.156.122:6818/3705334",
"state": "STATE_CLOSED",
"worker_name": "io-worker",
"worker_id": 2
}
],
"deleted connections": [
{
"conn": "0x7f2a4b2ed000",
"peer_type": "dse",
"peer_name": "dse.0",
"peer_addr": "172.16.156.122:6817/3386390",
"state": "STATE_CLOSED",
"worker_name": "io-worker",
"worker_id": 2
},
{
"conn": "0x7f2a4b2f1000",
"peer_type": "dse",
"peer_name": "dse.0",
"peer_addr": "172.16.156.124:6817/1534398",
"state": "STATE_CLOSED",
"worker_name": "io-worker",
"worker_id": 1
},
{
"conn": "0x7f2a4b48d000",
"peer_type": "dse",
"peer_name": "unknown.0",
"peer_addr": "172.16.156.124:6809/2008338",
"state": "STATE_CLOSED",
"worker_name": "io-worker",
"worker_id": 2
},
{
"conn": "0x7f2a4b7e7000",
"peer_type": "dse",
"peer_name": "unknown.0",
"peer_addr": "172.16.156.122:6818/3705334",
"state": "STATE_CLOSED",
"worker_name": "io-worker",
"worker_id": 2
}
]
}
}
# public
{
"connections": {
"established connections": [
{
"conn": "0x7f2662861000",
"peer_type": "client",
"peer_name": "client.70707860",
"peer_addr": "172.16.156.124:0/1339674865",
"state": "STATE_OPEN",
"worker_name": "eng-worker",
"worker_id": 1
},
{
"conn": "0x7f2662869000",
"peer_type": "client",
"peer_name": "client.70707857",
"peer_addr": "172.16.156.124:0/1071731267",
"state": "STATE_OPEN",
"worker_name": "eng-worker",
"worker_id": 7
},
{
"conn": "0x7f2a4b361000",
"peer_type": "client",
"peer_name": "client.70707869",
"peer_addr": "172.16.156.124:0/3299471168",
"state": "STATE_OPEN",
"worker_name": "eng-worker",
"worker_id": 5
},
{
"conn": "0x7f2a4b365000",
"peer_type": "client",
"peer_name": "client.70707866",
"peer_addr": "172.16.156.124:0/3207142384",
"state": "STATE_OPEN",
"worker_name": "eng-worker",
"worker_id": 6
},
{
"conn": "0x7f2a4b319000",
"peer_type": "client",
"peer_name": "client.70707851",
"peer_addr": "172.16.156.124:0/1079114638",
"state": "STATE_OPEN",
"worker_name": "eng-worker",
"worker_id": 4
},
{
"conn": "0x7f266285d000",
"peer_type": "client",
"peer_name": "client.70707826",
"peer_addr": "172.16.156.124:0/76342377",
"state": "STATE_OPEN",
"worker_name": "eng-worker",
"worker_id": 2
},
{
"conn": "0x7f2a4b343000",
"peer_type": "client",
"peer_name": "client.70707853",
"peer_addr": "172.16.156.124:0/419404416",
"state": "STATE_OPEN",
"worker_name": "eng-worker",
"worker_id": 6
},
{
"conn": "0x7f2662865000",
"peer_type": "client",
"peer_name": "client.70707854",
"peer_addr": "172.16.156.124:0/4161971061",
"state": "STATE_OPEN",
"worker_name": "eng-worker",
"worker_id": 0
},
{
"conn": "0x7f2a4b325000",
"peer_type": "client",
"peer_name": "client.70707863",
"peer_addr": "172.16.156.124:0/1016312339",
"state": "STATE_OPEN",
"worker_name": "eng-worker",
"worker_id": 7
},
{
"conn": "0x7f2a4b333000",
"peer_type": "client",
"peer_name": "client.70707872",
"peer_addr": "172.16.156.124:0/941002412",
"state": "STATE_OPEN",
"worker_name": "eng-worker",
"worker_id": 2
}
],
"accepting connections": [],
"deleted connections": []
}
}
# objecter
{
"connections": {
"established connections": [
{
"conn": "0x7f2a6edfd000",
"peer_type": "osd",
"peer_name": "osd.14",
"peer_addr": "172.16.156.122:6804/3705438",
"state": "STATE_OPEN",
"worker_name": "osdc-worker",
"worker_id": 3
},
{
"conn": "0x7f2a6e205000",
"peer_type": "osd",
"peer_name": "osd.1",
"peer_addr": "172.16.156.123:6819/1060851",
"state": "STATE_OPEN",
"worker_name": "osdc-worker",
"worker_id": 2
},
{
"conn": "0x7f2a6e209000",
"peer_type": "osd",
"peer_name": "osd.0",
"peer_addr": "172.16.156.124:6819/2010936",
"state": "STATE_OPEN",
"worker_name": "osdc-worker",
"worker_id": 7
},
{
"conn": "0x7f2a4b2b1000",
"peer_type": "osd",
"peer_name": "osd.18",
"peer_addr": "172.16.156.124:6825/2011059",
"state": "STATE_OPEN",
"worker_name": "osdc-worker",
"worker_id": 3
},
{
"conn": "0x7f2a4b2d9000",
"peer_type": "osd",
"peer_name": "osd.12",
"peer_addr": "172.16.156.124:6823/2011020",
"state": "STATE_OPEN",
"worker_name": "osdc-worker",
"worker_id": 2
},
{
"conn": "0x7f2a4b2b5000",
"peer_type": "osd",
"peer_name": "osd.20",
"peer_addr": "172.16.156.122:6806/3705519",
"state": "STATE_OPEN",
"worker_name": "osdc-worker",
"worker_id": 6
},
{
"conn": "0x7f2637105000",
"peer_type": "osd",
"peer_name": "osd.0",
"peer_addr": "172.16.156.124:6801/1530530",
"state": "STATE_CLOSED",
"worker_name": "osdc-worker",
"worker_id": 4
},
{
"conn": "0x7f2a6e743000",
"peer_type": "osd",
"peer_name": "osd.7",
"peer_addr": "172.16.156.123:6821/1060879",
"state": "STATE_OPEN",
"worker_name": "osdc-worker",
"worker_id": 1
},
{
"conn": "0x7f2a6e183000",
"peer_type": "osd",
"peer_name": "osd.6",
"peer_addr": "172.16.156.124:6821/2010961",
"state": "STATE_OPEN",
"worker_name": "osdc-worker",
"worker_id": 0
},
{
"conn": "0x7f2a4b2dd000",
"peer_type": "osd",
"peer_name": "osd.12",
"peer_addr": "172.16.156.124:6805/1530602",
"state": "STATE_CLOSED",
"worker_name": "osdc-worker",
"worker_id": 2
},
{
"conn": "0x7f2a4b2c5000",
"peer_type": "osd",
"peer_name": "osd.13",
"peer_addr": "172.16.156.123:6823/1060936",
"state": "STATE_OPEN",
"worker_name": "osdc-worker",
"worker_id": 5
},
{
"conn": "0x7f2a6e089000",
"peer_type": "osd",
"peer_name": "osd.8",
"peer_addr": "172.16.156.122:6802/3705373",
"state": "STATE_OPEN",
"worker_name": "osdc-worker",
"worker_id": 4
},
{
"conn": "0x7f2a4b2bd000",
"peer_type": "osd",
"peer_name": "osd.19",
"peer_addr": "172.16.156.123:6825/1060973",
"state": "STATE_OPEN",
"worker_name": "osdc-worker",
"worker_id": 0
},
{
"conn": "0x7f2a4b2b9000",
"peer_type": "osd",
"peer_name": "osd.2",
"peer_addr": "172.16.156.122:6800/3705344",
"state": "STATE_OPEN",
"worker_name": "osdc-worker",
"worker_id": 1
}
],
"accepting connections": [],
"deleted connections": [
{
"conn": "0x7f2637105000",
"peer_type": "osd",
"peer_name": "osd.0",
"peer_addr": "172.16.156.124:6801/1530530",
"state": "STATE_CLOSED",
"worker_name": "osdc-worker",
"worker_id": 4
},
{
"conn": "0x7f2a4b2dd000",
"peer_type": "osd",
"peer_name": "osd.12",
"peer_addr": "172.16.156.124:6805/1530602",
"state": "STATE_CLOSED",
"worker_name": "osdc-worker",
"worker_id": 2
}
]
}
}========================================================================================
其他OSD
[root@node124 26]# ceph daemon osd.18 messenger dump
{
"cluster": {
"name": "osd.18",
"addr": "192.168.156.124:6814/2011059",
"network stack type": "posix",
"number of established connection": 11,
"number of accepting connection": 3,
"number of deleted connection": 3
},
"client": {
"name": "osd.18",
"addr": "172.16.156.124:6825/2011059",
"network stack type": "posix",
"number of established connection": 7,
"number of accepting connection": 0,
"number of deleted connection": 2
},
"ms_objecter": {
"name": "osd.18",
"addr": "-",
"network stack type": "posix",
"number of established connection": 0,
"number of accepting connection": 0,
"number of deleted connection": 0
},
"hb_front_client": {
"name": "osd.18",
"addr": "172.16.156.124:0/2011059",
"network stack type": "posix",
"number of established connection": 10,
"number of accepting connection": 0,
"number of deleted connection": 0
},
"hb_back_client": {
"name": "osd.18",
"addr": "192.168.156.124:0/2011059",
"network stack type": "posix",
"number of established connection": 10,
"number of accepting connection": 0,
"number of deleted connection": 0
},
"hb_front_server": {
"name": "osd.18",
"addr": "172.16.156.124:6826/2011059",
"network stack type": "posix",
"number of established connection": 10,
"number of accepting connection": 0,
"number of deleted connection": 0
},
"hb_back_server": {
"name": "osd.18",
"addr": "192.168.156.124:6815/2011059",
"network stack type": "posix",
"number of established connection": 10,
"number of accepting connection": 0,
"number of deleted connection": 0
}
}
[root@node124 26]#
[root@node123 26]# ceph daemon osd.7 messenger dump
{
"cluster": {
"name": "osd.7",
"addr": "192.168.156.123:6810/1060879",
"network stack type": "posix",
"number of established connection": 11,
"number of accepting connection": 0,
"number of deleted connection": 0
},
"client": {
"name": "osd.7",
"addr": "172.16.156.123:6821/1060879",
"network stack type": "posix",
"number of established connection": 8,
"number of accepting connection": 0,
"number of deleted connection": 3
},
"ms_objecter": {
"name": "osd.7",
"addr": "-",
"network stack type": "posix",
"number of established connection": 0,
"number of accepting connection": 0,
"number of deleted connection": 0
},
"hb_front_client": {
"name": "osd.7",
"addr": "172.16.156.123:0/1060879",
"network stack type": "posix",
"number of established connection": 14,
"number of accepting connection": 0,
"number of deleted connection": 4
},
"hb_back_client": {
"name": "osd.7",
"addr": "192.168.156.123:0/1060879",
"network stack type": "posix",
"number of established connection": 14,
"number of accepting connection": 0,
"number of deleted connection": 4
},
"hb_front_server": {
"name": "osd.7",
"addr": "172.16.156.123:6822/1060879",
"network stack type": "posix",
"number of established connection": 11,
"number of accepting connection": 0,
"number of deleted connection": 1
},
"hb_back_server": {
"name": "osd.7",
"addr": "192.168.156.123:6811/1060879",
"network stack type": "posix",
"number of established connection": 11,
"number of accepting connection": 0,
"number of deleted connection": 1
}
}
[root@node123 26]#
[root@node122 26]# ceph daemon osd.2 messenger dump
{
"cluster": {
"name": "osd.2",
"addr": "192.168.156.122:6800/3705344",
"network stack type": "posix",
"number of established connection": 11,
"number of accepting connection": 1,
"number of deleted connection": 1
},
"client": {
"name": "osd.2",
"addr": "172.16.156.122:6800/3705344",
"network stack type": "posix",
"number of established connection": 7,
"number of accepting connection": 0,
"number of deleted connection": 2
},
"ms_objecter": {
"name": "osd.2",
"addr": "-",
"network stack type": "posix",
"number of established connection": 0,
"number of accepting connection": 0,
"number of deleted connection": 0
},
"hb_front_client": {
"name": "osd.2",
"addr": "172.16.156.122:0/3705344",
"network stack type": "posix",
"number of established connection": 11,
"number of accepting connection": 0,
"number of deleted connection": 1
},
"hb_back_client": {
"name": "osd.2",
"addr": "192.168.156.122:0/3705344",
"network stack type": "posix",
"number of established connection": 11,
"number of accepting connection": 0,
"number of deleted connection": 1
},
"hb_front_server": {
"name": "osd.2",
"addr": "172.16.156.122:6801/3705344",
"network stack type": "posix",
"number of established connection": 11,
"number of accepting connection": 0,
"number of deleted connection": 3
},
"hb_back_server": {
"name": "osd.2",
"addr": "192.168.156.122:6801/3705344",
"network stack type": "posix",
"number of established connection": 11,
"number of accepting connection": 0,
"number of deleted connection": 3
}
}
[root@node122 26]## 其他CSD
[root@node122 26]# ceph daemon dse.$(hostname) csd 23 messenger_dump
{
"cluster": {
"name": "osd.23",
"addr": "192.168.156.122:6814/3705358",
"network stack type": "posix",
"number of established connection": 11,
"number of accepting connection": 1,
"number of deleted connection": 1
},
"client": {
"name": "osd.23",
"addr": "172.16.156.122:6826/3705358",
"network stack type": "posix",
"number of established connection": 12,
"number of accepting connection": 0,
"number of deleted connection": 2
},
"hb_front_client": {
"name": "osd.23",
"addr": "172.16.156.122:0/3705358",
"network stack type": "posix",
"number of established connection": 11,
"number of accepting connection": 0,
"number of deleted connection": 0
},
"hb_back_client": {
"name": "osd.23",
"addr": "192.168.156.122:0/3705358",
"network stack type": "posix",
"number of established connection": 11,
"number of accepting connection": 0,
"number of deleted connection": 0
},
"hb_front_server": {
"name": "osd.23",
"addr": "172.16.156.122:6827/3705358",
"network stack type": "posix",
"number of established connection": 8,
"number of accepting connection": 0,
"number of deleted connection": 0
},
"hb_back_server": {
"name": "osd.23",
"addr": "192.168.156.122:6815/3705358",
"network stack type": "posix",
"number of established connection": 8,
"number of accepting connection": 0,
"number of deleted connection": 0
}
}
[root@node122 26]# ceph daemon dse.$(hostname) csd 17 messenger_dump
{
"cluster": {
"name": "osd.17",
"addr": "192.168.156.122:6812/3705352",
"network stack type": "posix",
"number of established connection": 11,
"number of accepting connection": 1,
"number of deleted connection": 1
},
"client": {
"name": "osd.17",
"addr": "172.16.156.122:6824/3705352",
"network stack type": "posix",
"number of established connection": 12,
"number of accepting connection": 0,
"number of deleted connection": 2
},
"hb_front_client": {
"name": "osd.17",
"addr": "172.16.156.122:0/3705352",
"network stack type": "posix",
"number of established connection": 10,
"number of accepting connection": 0,
"number of deleted connection": 0
},
"hb_back_client": {
"name": "osd.17",
"addr": "192.168.156.122:0/3705352",
"network stack type": "posix",
"number of established connection": 10,
"number of accepting connection": 0,
"number of deleted connection": 0
},
"hb_front_server": {
"name": "osd.17",
"addr": "172.16.156.122:6825/3705352",
"network stack type": "posix",
"number of established connection": 11,
"number of accepting connection": 0,
"number of deleted connection": 0
},
"hb_back_server": {
"name": "osd.17",
"addr": "192.168.156.122:6813/3705352",
"network stack type": "posix",
"number of established connection": 11,
"number of accepting connection": 0,
"number of deleted connection": 0
}
}
[root@node122 26]#
[root@node123 26]# ceph daemon dse.$(hostname) csd 16 messenger_dump
{
"cluster": {
"name": "osd.16",
"addr": "192.168.156.123:6804/1058481",
"network stack type": "posix",
"number of established connection": 11,
"number of accepting connection": 1,
"number of deleted connection": 1
},
"client": {
"name": "osd.16",
"addr": "172.16.156.123:6815/1058481",
"network stack type": "posix",
"number of established connection": 29,
"number of accepting connection": 0,
"number of deleted connection": 3
},
"hb_front_client": {
"name": "osd.16",
"addr": "172.16.156.123:0/1058481",
"network stack type": "posix",
"number of established connection": 10,
"number of accepting connection": 0,
"number of deleted connection": 0
},
"hb_back_client": {
"name": "osd.16",
"addr": "192.168.156.123:0/1058481",
"network stack type": "posix",
"number of established connection": 10,
"number of accepting connection": 0,
"number of deleted connection": 0
},
"hb_front_server": {
"name": "osd.16",
"addr": "172.16.156.123:6816/1058481",
"network stack type": "posix",
"number of established connection": 11,
"number of accepting connection": 0,
"number of deleted connection": 0
},
"hb_back_server": {
"name": "osd.16",
"addr": "192.168.156.123:6805/1058481",
"network stack type": "posix",
"number of established connection": 11,
"number of accepting connection": 0,
"number of deleted connection": 0
}
}
[root@node123 26]#
[root@node124 26]# ceph daemon dse.$(hostname) csd 21 messenger_dump
{
"cluster": {
"name": "osd.21",
"addr": "192.168.156.124:6806/2008360",
"network stack type": "posix",
"number of established connection": 11,
"number of accepting connection": 2,
"number of deleted connection": 2
},
"client": {
"name": "osd.21",
"addr": "172.16.156.124:6817/2008360",
"network stack type": "posix",
"number of established connection": 29,
"number of accepting connection": 1,
"number of deleted connection": 4
},
"hb_front_client": {
"name": "osd.21",
"addr": "172.16.156.124:0/2008360",
"network stack type": "posix",
"number of established connection": 11,
"number of accepting connection": 0,
"number of deleted connection": 1
},
"hb_back_client": {
"name": "osd.21",
"addr": "192.168.156.124:0/2008360",
"network stack type": "posix",
"number of established connection": 11,
"number of accepting connection": 0,
"number of deleted connection": 1
},
"hb_front_server": {
"name": "osd.21",
"addr": "172.16.156.124:6818/2008360",
"network stack type": "posix",
"number of established connection": 11,
"number of accepting connection": 0,
"number of deleted connection": 0
},
"hb_back_server": {
"name": "osd.21",
"addr": "192.168.156.124:6807/2008360",
"network stack type": "posix",
"number of established connection": 11,
"number of accepting connection": 4,
"number of deleted connection": 4
}
}
[root@node124 26]#
[root@node122 26]# ceph daemon dse.$(hostname) csd 5 messenger_dump
{
"cluster": {
"name": "osd.5",
"addr": "192.168.156.122:6808/3705340",
"network stack type": "posix",
"number of established connection": 11,
"number of accepting connection": 1,
"number of deleted connection": 1
},
"client": {
"name": "osd.5",
"addr": "172.16.156.122:6820/3705340",
"network stack type": "posix",
"number of established connection": 12,
"number of accepting connection": 0,
"number of deleted connection": 2
},
"hb_front_client": {
"name": "osd.5",
"addr": "172.16.156.122:0/3705340",
"network stack type": "posix",
"number of established connection": 10,
"number of accepting connection": 0,
"number of deleted connection": 0
},
"hb_back_client": {
"name": "osd.5",
"addr": "192.168.156.122:0/3705340",
"network stack type": "posix",
"number of established connection": 10,
"number of accepting connection": 0,
"number of deleted connection": 0
},
"hb_front_server": {
"name": "osd.5",
"addr": "172.16.156.122:6821/3705340",
"network stack type": "posix",
"number of established connection": 11,
"number of accepting connection": 0,
"number of deleted connection": 0
},
"hb_back_server": {
"name": "osd.5",
"addr": "192.168.156.122:6809/3705340",
"network stack type": "posix",
"number of established connection": 11,
"number of accepting connection": 0,
"number of deleted connection": 0
}
}
[root@node122 26]#
[root@node122 26]# ceph daemon dse.$(hostname) csd 17 messenger_dump
{
"cluster": {
"name": "osd.17",
"addr": "192.168.156.122:6812/3705352",
"network stack type": "posix",
"number of established connection": 11,
"number of accepting connection": 1,
"number of deleted connection": 1
},
"client": {
"name": "osd.17",
"addr": "172.16.156.122:6824/3705352",
"network stack type": "posix",
"number of established connection": 12,
"number of accepting connection": 0,
"number of deleted connection": 2
},
"hb_front_client": {
"name": "osd.17",
"addr": "172.16.156.122:0/3705352",
"network stack type": "posix",
"number of established connection": 10,
"number of accepting connection": 0,
"number of deleted connection": 0
},
"hb_back_client": {
"name": "osd.17",
"addr": "192.168.156.122:0/3705352",
"network stack type": "posix",
"number of established connection": 10,
"number of accepting connection": 0,
"number of deleted connection": 0
},
"hb_front_server": {
"name": "osd.17",
"addr": "172.16.156.122:6825/3705352",
"network stack type": "posix",
"number of established connection": 11,
"number of accepting connection": 0,
"number of deleted connection": 0
},
"hb_back_server": {
"name": "osd.17",
"addr": "192.168.156.122:6813/3705352",
"network stack type": "posix",
"number of established connection": 11,
"number of accepting connection": 0,
"number of deleted connection": 0
}
}[root@node122 26]# ceph daemon dse.$(hostname) engine 65539.0 messenger_dump
{
"cluster": {
"name": "dse.0",
"addr": "172.16.156.122:6818/3705334",
"network stack type": "posix",
"number of established connection": 2,
"number of accepting connection": 2,
"number of deleted connection": 2
},
"public": {
"name": "dse.0",
"addr": "172.16.156.122:6810/3705334",
"network stack type": "posix",
"number of established connection": 1,
"number of accepting connection": 0,
"number of deleted connection": 1
},
"objecter": {
"name": "dse.70684427",
"addr": "172.16.156.122:6819/3705334",
"network stack type": "posix",
"number of established connection": 13,
"number of accepting connection": 0,
"number of deleted connection": 1
}
}
[root@node122 26]#[root@node124 26]# ceph daemon dse.$(hostname) engine 65539.8 messenger_dump
{
"cluster": {
"name": "dse.0",
"addr": "172.16.156.124:6809/2008338",
"network stack type": "posix",
"number of established connection": 4,
"number of accepting connection": 2,
"number of deleted connection": 4
},
"public": {
"name": "dse.0",
"addr": "172.16.156.124:6801/2008338",
"network stack type": "posix",
"number of established connection": 2,
"number of accepting connection": 0,
"number of deleted connection": 2
},
"objecter": {
"name": "dse.70684199",
"addr": "172.16.156.124:6810/2008338",
"network stack type": "posix",
"number of established connection": 12,
"number of accepting connection": 0,
"number of deleted connection": 0
}
}
[root@node124 26]#
ceph 网络问题常用排查手段
原文链接：https://blog.csdn.net/underzerotem/article/details/106577334
检测Ceph环境中是否存在网络问题，常用方法使用及总结如下：
Iperf3：IPerf3使用过程中会将网络压满，会影响业务，只能和技服充分沟通后再使用
服务端：iperf3 -s -p 4000 -f M
iperf3 -s(服务端) -p 4000(端口号) -f 开关指定生成报告的数据格式类型
客户端：
iperf3 -c 192.168.26.51 -f K
iperf3 -c 192.168.26.51(服务端的ip) -f K -i 1(1s 刷新1次) -t 60(运行60s) -p (port)
iperf3 -c 192.168.26.51 -f K -w 200K
-w 设置窗口大小和套接字缓冲区的大小
在服务器发送和客户端接收的反向模式下运行，可以添加-R开关
iperf3 -c 192.168.26.51 -f K -w 200K -R
双向运行测试，意味着我们要同时测量两个方向上的宽带 -d：
iperf3 -c 192.168.26.51 -f K -w 200K -d
如果要在客户端输出中获取服务器结果，可以使用“–get-server-output”选项，此时我们便会在客户端192.168.26.131主机上获取两份数据。
iperf3 -c 192.168.26.51 -f K -w 200K --get-server-output
使用-P选项设置并行客户端流的数量，它们同时运行
iperf3 -c 192.168.26.51 -f K -w 200K -P 2
1、可以看到带宽不稳，波动较大
2、Retr过高，重传意味着丢包，tcp发生丢包会严重影响通信性能
3、窗口值过小，还没有到1MB就开始重传，滑动窗口太小，会降低性能
2、查询ethX网口收发包统计，注意-S为大写
ethtool -S eth2 | grep errors
[root@node1 lfl]# ethtool -S eth2 | grep errors
rx_crc_errors: 0
rx_missed_errors: 0
tx_aborted_errors: 0
tx_carrier_errors: 0
tx_window_errors: 0
rx_long_length_errors: 0
rx_short_length_errors: 0
rx_align_errors: 0
rx_errors: 0
tx_errors: 0
rx_length_errors: 0
rx_over_errors: 0
rx_frame_errors: 0
rx_fifo_errors: 0
tx_fifo_errors: 0
tx_heartbeat_errors: 0
排查bond2（业务网口）的各个网口是否存在错误信息。
3、netstat -i | column –t 查看网口接口信息
主要观察RX-ERR 和 TX-ERR项
相关各项解释如下：
Iface：网络接口名称
MTU：最大传输单元
RX-OK：接收时，正确的数据包数
RX-ERR：接收时，产生错误的数据包数
RX-DRP：接收时，丢弃的数据包数
RX-OVR：接收时，由于过速而丢失的数据包数
TX-OK：发送时，正确的数据包数
TX-ERR：发送时，产生错误的数据包数
TX-DRP：发送时，丢弃的数据包数
TX-OVR：发送时，由于过速而丢失的数据包数
4、
watch “cat /proc/net/snmp | grep Tcp | awk ‘{print $13,$14,$16}’| column -t”
InCsumErrors - 接收的带有校验和错误的报文数
TCP 接收的错误报文数
TCP 接收的目前所有建立连接的错误报文数
标红的各项不停的几十几十的涨说明网络存在问题
5、tcpdump分析抓取相关网口的cap包，抓取后拖入wireshark中进行分析
Tcpdump抓取相关包的方法如下：
抓取相关包：
tcpdump -i eth0 -s 0 -w file.pcap
可以指定端口进行抓包：tcpdump -i eth0 port 22
读取抓包文件：
tcpdump -r file.pcap
