准备基本环境
| 名称 |
ip地址 |
cpu |
内存 |
es监听端口 |
| redis-651 |
10.0.2.1 |
8c |
64G |
9200 |
| redis-652 |
10.0.2.2 |
8c |
64G |
9200 |
| redis-653 |
10.0.2.3 |
8c |
64G |
9200 |
搭建集群
# 更新hosts文件
cat /etc/hosts
10.0.2.1 es-node-1
10.0.2.2 es-node-2
10.0.2.3 es-node-3
# 创建es用户
useradd -d /export/es es
su - es
[root@es-node-1 deeplangdevops]# vim /etc/security/limits.conf
root soft nofile 65535
root hard nofile 65535
* soft nofile 65535
* hard nofile 65535
es soft memlock unlimited
es hard memlock unlimited
[root@es-node-1 deeplangdevops]# ulimit -n
65535
[es@es-node-1 config]$ cat /etc/sysctl.conf
vm.swappiness = 0
kernel.sysrq = 1
net.ipv4.neigh.default.gc_stale_time = 120
# see details in https://help.aliyun.com/knowledge_detail/39428.html
net.ipv4.conf.all.rp_filter = 0
net.ipv4.conf.default.rp_filter = 0
net.ipv4.conf.default.arp_announce = 2
net.ipv4.conf.lo.arp_announce = 2
net.ipv4.conf.all.arp_announce = 2
# see details in https://help.aliyun.com/knowledge_detail/41334.html
net.ipv4.tcp_max_tw_buckets = 5000
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_max_syn_backlog = 1024
net.ipv4.tcp_synack_retries = 2
net.ipv4.tcp_slow_start_after_idle = 0
vm.max_map_count = 655350
[es@es-node-1 config]$ sysctl -p
# 下载包并解压
[es@es-node-1 ~]$ cd /export
[es@es-node-1 ~]$ wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.17.8-linux-x86_64.tar.gz
[es@es-node-1 ~]$ tar -xf elasticsearch-7.17.8-linux-x86_64.tar.gz
# 创建数据目录并准备配置文件
[es@es-node-1 ~]$ mkdir /export/elasticsearch-7.17.8/data/
[es@es-node-1 ~]$ cd /export/elasticsearch-7.17.8/config/
[root@es-node-1 config]# vim elasticsearch.yml
cluster.name: my-application
node.name: es-node-1
node.master: true
node.data: true
path.data: /export/elasticsearch-7.17.8/data
path.logs: /export/elasticsearch-7.17.8/logs
network.host: 0.0.0.0
http.port: 9200
transport.tcp.port: 9300
transport.tcp.compress: true
discovery.seed_hosts: ["10.0.2.1:9300", "10.0.2.2:9300", "10.0.2.3:9300"]
cluster.initial_master_nodes: ["es-node-1", "es-node-2", "es-node-3"]
discovery.zen.minimum_master_nodes: 2
gateway.recover_after_nodes: 2
gateway.recover_after_time: 5m
gateway.expected_nodes: 2
search.max_buckets: 90000000
indices.query.bool.max_clause_count: 10240
cluster.routing.allocation.same_shard.host: true
bootstrap.memory_lock: true
http.cors.enabled: true
http.cors.allow-origin: "*"
http.max_content_length: 2147483647b
# 主要参数解释：
discovery.zen.minimum_master_nodes: 2 # 为了避免脑裂，集群节点数最少为 半数+1
#只要指定数量的节点加入集群，就开始进行恢复
gateway.recover_after_nodes: 2
#如果期望的节点数量没有达标，那么会等待一定的时间，然后就开始进行shard recovery，默认是等待5m
gateway.recover_after_time: 5m
#要求必须有多少个节点在集群中，当加入集群中的节点数量达到这个期望数值之后，每个node的local shard的恢复就会理解开始，默认的值是0，也就是不会做任何的等待
gateway.expected_nodes: 2
#查询结果在分片上找到的条目超过了限定的10000个，官网限制在10000是为了其性能考虑的。需要调大search.max_buckets这个参数。
search.max_buckets: 90000000
#es的查询参数限制，默认是限制只能传入1024个参数
indices.query.bool.max_clause_count: 10240
#将阻止主副本分片被分配到同一台物理机，提高可用性。
cluster.routing.allocation.same_shard.host：true
#ES默认开启了内存地址锁定，为了避免内存交换提高性能。但是Centos6不支持SecComp功能，启动会报错，Centos7支持
bootstrap.memory_lock: true
#设置是否压缩tcp传输时的数据，默认为false，不压缩。
transport.tcp.compress: true
# 是否支持跨域
http.cors.enabled: true
# *表示支持所有域名
http.cors.allow-origin: "*"
[es@es-node-1 config]$ vim jvm.options
-Xms8g
-Xmx8g
# 更新hosts文件
cat /etc/hosts
10.0.2.1 es-node-1
10.0.2.2 es-node-2
10.0.2.3 es-node-3
# 创建es用户
useradd -d /export/es es
su - es
[root@es-node-2 deeplangdevops]# vim /etc/security/limits.conf
root soft nofile 65535
root hard nofile 65535
* soft nofile 65535
* hard nofile 65535
es soft memlock unlimited
es hard memlock unlimited
[root@es-node-2 deeplangdevops]# ulimit -n
65535
[es@es-node-2 config]$ cat /etc/sysctl.conf
vm.swappiness = 0
kernel.sysrq = 1
net.ipv4.neigh.default.gc_stale_time = 120
# see details in https://help.aliyun.com/knowledge_detail/39428.html
net.ipv4.conf.all.rp_filter = 0
net.ipv4.conf.default.rp_filter = 0
net.ipv4.conf.default.arp_announce = 2
net.ipv4.conf.lo.arp_announce = 2
net.ipv4.conf.all.arp_announce = 2
# see details in https://help.aliyun.com/knowledge_detail/41334.html
net.ipv4.tcp_max_tw_buckets = 5000
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_max_syn_backlog = 1024
net.ipv4.tcp_synack_retries = 2
net.ipv4.tcp_slow_start_after_idle = 0
vm.max_map_count = 655350
[es@es-node-2 config]$ sysctl -p
# 下载包并解压
[es@es-node-2 ~]$ cd /export
[es@es-node-2 ~]$ wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.17.8-linux-x86_64.tar.gz
[es@es-node-2 ~]$ tar -xf elasticsearch-7.17.8-linux-x86_64.tar.gz
# 创建数据目录并准备配置文件
[es@es-node-2 ~]$ mkdir /export/elasticsearch-7.17.8/data/
[es@es-node-2 ~]$ cd /export/elasticsearch-7.17.8/config/
[root@es-node-2 config]# vim elasticsearch.yml
cluster.name: my-application
node.name: es-node-2
node.master: true
node.data: true
path.data: /export/elasticsearch-7.17.8/data
path.logs: /export/elasticsearch-7.17.8/logs
network.host: 0.0.0.0
http.port: 9200
transport.tcp.port: 9300
transport.tcp.compress: true
discovery.seed_hosts: ["10.0.2.1:9300", "10.0.2.2:9300", "10.0.2.3:9300"]
cluster.initial_master_nodes: ["es-node-1", "es-node-2", "es-node-3"]
discovery.zen.minimum_master_nodes: 2
gateway.recover_after_nodes: 2
gateway.recover_after_time: 5m
gateway.expected_nodes: 2
search.max_buckets: 90000000
indices.query.bool.max_clause_count: 10240
cluster.routing.allocation.same_shard.host: true
bootstrap.memory_lock: true
http.cors.enabled: true
http.cors.allow-origin: "*"
http.max_content_length: 2147483647b
[es@es-node-2 config]$ vim jvm.options
-Xms8g
-Xmx8g
# 更新hosts文件
cat /etc/hosts
10.0.2.1 es-node-1
10.0.2.2 es-node-2
10.0.2.3 es-node-3
# 创建es用户
useradd -d /export/es es
su - es
[root@es-node-3 deeplangdevops]# vim /etc/security/limits.conf
root soft nofile 65535
root hard nofile 65535
* soft nofile 65535
* hard nofile 65535
es soft memlock unlimited
es hard memlock unlimited
[root@es-node-3 deeplangdevops]# ulimit -n
65535
[es@es-node-3 config]$ cat /etc/sysctl.conf
vm.swappiness = 0
kernel.sysrq = 1
net.ipv4.neigh.default.gc_stale_time = 120
# see details in https://help.aliyun.com/knowledge_detail/39428.html
net.ipv4.conf.all.rp_filter = 0
net.ipv4.conf.default.rp_filter = 0
net.ipv4.conf.default.arp_announce = 2
net.ipv4.conf.lo.arp_announce = 2
net.ipv4.conf.all.arp_announce = 2
# see details in https://help.aliyun.com/knowledge_detail/41334.html
net.ipv4.tcp_max_tw_buckets = 5000
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_max_syn_backlog = 1024
net.ipv4.tcp_synack_retries = 2
net.ipv4.tcp_slow_start_after_idle = 0
vm.max_map_count = 655350
[es@es-node-3 config]$ sysctl -p
# 下载包并解压
[es@es-node-3 ~]$ cd /export
[es@es-node-3 ~]$ wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.17.8-linux-x86_64.tar.gz
[es@es-node-3 ~]$ tar -xf elasticsearch-7.17.8-linux-x86_64.tar.gz
# 创建数据目录并准备配置文件
[es@es-node-3 ~]$ mkdir /export/elasticsearch-7.17.8/data/
[es@es-node-3 ~]$ cd /export/elasticsearch-7.17.8/config/
[root@es-node-3 config]# vim elasticsearch.yml
cluster.name: my-application
node.name: es-node-3
node.master: true
node.data: true
path.data: /export/elasticsearch-7.17.8/data
path.logs: /export/elasticsearch-7.17.8/logs
network.host: 0.0.0.0
http.port: 9200
transport.tcp.port: 9300
transport.tcp.compress: true
discovery.seed_hosts: ["10.0.2.1:9300", "10.0.2.2:9300", "10.0.2.3:9300"]
cluster.initial_master_nodes: ["es-node-1", "es-node-2", "es-node-3"]
discovery.zen.minimum_master_nodes: 2
gateway.recover_after_nodes: 2
gateway.recover_after_time: 5m
gateway.expected_nodes: 2
search.max_buckets: 90000000
indices.query.bool.max_clause_count: 10240
cluster.routing.allocation.same_shard.host: true
bootstrap.memory_lock: true
http.cors.enabled: true
http.cors.allow-origin: "*"
http.max_content_length: 2147483647b
[es@es-node-3 config]$ vim jvm.options
-Xms8g
-Xmx8g
# 3个节点启动 es集群
nohup /export/elasticsearch-7.17.8/bin/elasticsearch >> /export/elasticsearch-7.17.8/logs/elasticsearch.log &
# 在 10.0.2.1 一个节点上操作
1.生成 ca证书
[es@es-node-1 elasticsearch-7.17.8]$ /export/elasticsearch-7.17.8/bin/elasticsearch-certutil ca
Please enter the desired output file [elastic-stack-ca.p12]: #这里直接回车即可
Enter password for elastic-stack-ca.p12 : #这里直接回车即可，不要设置密码
设置完毕后，会在/export/elasticsearch-7.17.8 下看到新生成的文件：
elastic-stack-ca.p12
2.生成p12秘钥
[es@es-node-1 elasticsearch-7.17.8]$ /export/elasticsearch-7.17.8/bin/elasticsearch-certutil cert --ca elastic-stack-ca.p12
下面三项直接回车即可：
Enter password for CA (elastic-stack-ca.p12) :
Please enter the desired output file [elastic-certificates.p12]:
Enter password for elastic-certificates.p12 : #这里直接回车即可，不要设置密码，否则后面ES会启动不了
Certificates written to /export/elasticsearch-7.17.8/elastic-certificates.p12
设置完毕后，会在/export/elasticsearch-7.17.8下看到新生成的文件：
elastic-certificates.p12
3. 拷贝p12秘钥文件
[es@es-node-1 elasticsearch-7.17.8]$ cd /export/elasticsearch-7.17.8/config/
[es@es-node-1 config]$ mkdir certs
[es@es-node-1 config]$ cp /export/elasticsearch-7.17.8/elastic-certificates.p12 certs/
4. 将p12认证文件拷贝到其他(10.0.2.2、10.0.2.3)节点上
在其他节点上先创建下certs目录：
[es@es-node-2 elasticsearch-7.17.8]$ cd /export/elasticsearch-7.17.8/config/
[es@es-node-2 config]$ mkdir certs
[es@es-node-3 elasticsearch-7.17.8]$ cd /export/elasticsearch-7.17.8/config/
[es@es-node-3 config]$ mkdir certs
5. 在 10.0.2.1 上操作拷贝文件
scp /export/elasticsearch-7.17.8/config/certs/elastic-certificates.p12 10.0.2.2:/export/elasticsearch-7.17.8/config/certs/elastic-certificates.p12
scp /export/elasticsearch-7.17.8/config/certs/elastic-certificates.p12 10.0.2.3:/export/elasticsearch-7.17.8/config/certs/elastic-certificates.p12
6.修改所有ES节点的配置文件( 3个节点都要增加以下配置内容)
[es@es-node-1 elasticsearch-7.17.8]$ vim config/elasticsearch.yml
xpack.security.enabled: true
xpack.security.transport.ssl.enabled: true
xpack.security.transport.ssl.verification_mode: certificate
xpack.security.transport.ssl.keystore.path: certs/elastic-certificates.p12
xpack.security.transport.ssl.truststore.path: certs/elastic-certificates.p12
7. 重启各ES节点
8. 设置密码
想要成功设置密码的话，必须确保集群状态正常才行,否则密码设置会失败。
在其中一个节点上设置密码即可：
在下面输入自定义的密码:
[es@es-node-1 ~]$ /export/elasticsearch-7.17.8/bin/elasticsearch-setup-passwords interactive
Initiating the setup of passwords for reserved users elastic,apm_system,kibana,logstash_system,beats_system,remote_monitoring_user.
You will be prompted to enter passwords as the process progresses.
Please confirm that you would like to continue [y/N]y
Enter password for [elastic]:
Reenter password for [elastic]:
Enter password for [apm_system]:
Reenter password for [apm_system]:
Enter password for [kibana]:
Reenter password for [kibana]:
Enter password for [logstash_system]:
Reenter password for [logstash_system]:
Enter password for [beats_system]:
Reenter password for [beats_system]:
Enter password for [remote_monitoring_user]:
Reenter password for [remote_monitoring_user]:
Changed password for user [apm_system]
Changed password for user [kibana]
Changed password for user [logstash_system]
Changed password for user [beats_system]
Changed password for user [remote_monitoring_user]
Changed password for user [elastic]
# 再次通过 kibana 访问es集群需要输入密码,搭建完成。
功能使用汇总
# 执行命令
GET _cat/shards?v
index shard prirep state docs store ip node
zonghe_coll-000001 1 p STARTED 665304 7.3gb 10.0.2.3 es-node-3
zonghe_coll-000001 1 r STARTED 665304 7.4gb 10.0.2.1 es-node-1
zonghe_coll-000001 1 r STARTED 665304 7.4gb 10.0.2.2 es-node-2
zonghe_coll-000001 2 r STARTED 663990 7.8gb 10.0.2.3 es-node-3
zonghe_coll-000001 2 r STARTED 663990 7.8gb 10.0.2.1 es-node-1
zonghe_coll-000001 2 p STARTED 663990 7.8gb 10.0.2.2 es-node-2
zonghe_coll-000001 0 r STARTED 664786 7.6gb 10.0.2.3 es-node-3
zonghe_coll-000001 0 p STARTED 664786 7.6gb 10.0.2.1 es-node-1
zonghe_coll-000001 0 r STARTED 664786 7.6gb 10.0.2.2 es-node-2
# 参数说明
index：所有名称
shard：分片数
prirep：分片类型，p=pri=primary为主分片，r=rep=replicas为复制分片
state：分片状态，STARTED为正常分片，INITIALIZING为异常分片
docs：记录数
store：存储大小
ip：es节点ip
node：es节点名称
#
- 例如一份 21.8G的数据，要写入es 集群的话会分成3个部分，也就是3分片，上面结果显示 0 p STARTED 664786 7.6gb 10.0.2.1 es-node-1 和 1 p STARTED 665304 7.3gb 10.0.2.3 es-node-3 、和 2 p STARTED 663990 7.8gb 10.0.2.2 es-node-2 分别落到了3台机器上，都是主分片数据
- 0号分片主分片在node1上， 他 的副本分片分别在node2 和 node3 上
- 1号分片主分片在 node3上，他的副本分片分别在 node1 和 node2上
- 2号分片主分片在 node2上，他的副本分片分别在 node1 和node3上
- 综上，0号主分7.6gb + 1号主分片 7.3gb + 2号分片主分片7.8gb = 22.7G
