k8s下创建mongodb副本集
环境:
OS:Centos 7
k8s:
[root@master mongo_yaml]# kubectl version
Client Version: v1.28.2
Kustomize Version: v5.0.4-0.20230601165947-6ce0bf390ce3
Server Version: v1.28.13
拓扑结构:1master 2nodes
#################################部署nfs服务器##################################
我们这里采用nfs做为pv,使用master节点做nfs服务器,当然nfs服务器也可以使用k8s外部的集群部署也可以.
1.安装nfs
[root@master ~]#yum -y install nfs-utils
[root@master ~]#yum -y install rpcbind
2.要创建nfs共享文件夹
[root@nfs data]# mkdir -p /data/mongo-1
[root@nfs data]# mkdir -p /data/mongo-2
[root@nfs data]# mkdir -p /data/mongo-3
[root@nfs data]# mkdir -p /data/mongo-1/data
[root@nfs data]# mkdir -p /data/mongo-2/data
[root@nfs data]# mkdir -p /data/mongo-3/data
[root@nfs data]# mkdir -p /data/mongo-1/key
[root@nfs data]# mkdir -p /data/mongo-2/key
[root@nfs data]# mkdir -p /data/mongo-3/key
创建3个副本的mongo的副本集,每个副本对应使用mongo-1,mongo-2,mongo-3 三个目录
3.编辑 /etc/exports 文件
[root@nfs data]# vim /etc/exports
#添加如下内容
/data/ *(rw,sync,no_root_squash)
4.保存退出后执行如下命令重启服务
systemctl restart rpcbind
systemctl restart nfs.service
systemctl enable nfs.service
5.执行 exportfs -v 命令可以显示出所有的共享目录
[root@master /]# exportfs -v
/data <world>(sync,wdelay,hide,no_subtree_check,sec=sys,rw,secure,no_root_squash,no_all_squash)
6.其他的 Node 节点上需要执行如下命令安装 nfs-utils 客户端
yum -y install nfs-utils
7.其他的 Node 节点上可执行如下命令查看 Master 节点上共享的文件夹
[root@node1 /]# showmount -e 192.168.1.102
Export list for 192.168.1.102:
/data *
[root@node2 etcd]# showmount -e 192.168.1.102
Export list for 192.168.1.102:
/data *
############################将keyfile放入到docker镜像中######################
参考如下连接
https://www.cnblogs.com/hxlasky/p/18741205
下面在部署的时候需要使用新的镜像，具体根据实际情况修改
###############################创建名称空间########################
[root@master mongo_yaml]# kubectl create namespace ns-mongo
###############################部署pv##############################
1.创建yaml保留目录
mkdir -p /root/my_yaml/mongo_cluster
2.创建pv
[root@master1 mongo]# vi /root/my_yaml/mongo_cluster/pv.yaml
# 添加以下内容
apiVersion: v1
kind: PersistentVolume
metadata:
name: mongo-pv-1
namespace: ns-mongo
spec:
accessModes:
- ReadWriteOnce
capacity:
storage: 2Gi
nfs:
path: /data/mongo-1
readOnly: false
server: 192.168.1.102
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: mongo-pv-2
namespace: ns-mongo
spec:
accessModes:
- ReadWriteOnce
capacity:
storage: 2Gi
nfs:
path: /data/mongo-2
readOnly: false
server: 192.168.1.102
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: mongo-pv-3
namespace: ns-mongo
spec:
accessModes:
- ReadWriteOnce
capacity:
storage: 2Gi
nfs:
path: /data/mongo-3
readOnly: false
server: 192.168.1.102
3.执行创建
[root@master mongo_cluster]# cd /root/my_yaml/mongo_cluster
[root@master1 mongo]# kubectl apply -f pv.yaml
persistentvolume/mongo-pv-1 created
persistentvolume/mongo-pv-2 created
persistentvolume/mongo-pv-3 created
4.查看
[root@master mongo_cluster]# kubectl get pv -n ns-mongo
NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE
mongo-pv-1 2Gi RWO Retain Available 15s
mongo-pv-2 2Gi RWO Retain Available 15s
mongo-pv-3 2Gi RWO Retain Available
###############################准备configmap文件##############################
1.准备yaml文件
apiVersion: v1
data:
mongod.conf: |+
dbpath=/mongo/data
logpath=/mongo/mongodb.log
pidfilepath=/mongo/key/master.pid
directoryperdb=true
logappend=true
bind_ip=0.0.0.0
port=27017
replSet=rs0
##keyFile=/data/config/mongodb-keyfile
##auth=true
kind: ConfigMap
metadata:
name: mongodb-conf
namespace: ns-mongo
这里我们先将keyFile和auth注释掉，等创建好了副本集和创建用户后再启用
2.创建
[root@master1 mongo]#cd /root/my_yaml/mongo_cluster
[root@master1 mongo]# kubectl apply -f config.yaml
3.查看
[root@master mongo_cluster]# kubectl get ConfigMap -n ns-mongo
NAME DATA AGE
mongodb-conf 1 7s
4.查看内容
kubectl describe cm mongodb-conf -n ns-mongo
######################创建service##################################
1.[root@master1 mongo]# vi /root/my_yaml/mongo_cluster/service.yaml
#输入以下内容
apiVersion: v1
kind: Service
metadata:
namespace: ns-mongo
name: mongo-1 #需要与podname同名
labels:
name: mongo
spec:
ports:
- name: mongo-port
port: 27017
clusterIP: None
selector:
name: mongo-1
---
apiVersion: v1
kind: Service
metadata:
namespace: ns-mongo
name: mongo-2 #需要与podname同名
labels:
name: mongo
spec:
ports:
- name: mongo-port
port: 27017
clusterIP: None
selector:
name: mongo-2
---
apiVersion: v1
kind: Service
metadata:
namespace: ns-mongo
name: mongo-3 #需要与podname同名
labels:
name: mongo
spec:
ports:
- name: mongo-port
port: 27017
clusterIP: None
selector:
name: mongo-3
2.创建
[root@master1 mongo]#cd /root/my_yaml/mongo_cluster
[root@master1 mongo]#kubectl apply -f service.yaml
3.查看
[root@master mongo_cluster]# kubectl get svc -n ns-mongo
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
mongo-1 ClusterIP None <none> 27017/TCP 24s
mongo-2 ClusterIP None <none> 27017/TCP 24s
mongo-3 ClusterIP None <none> 27017/TCP 24s
##################################创建MongoDB服务####################################
1.编辑创建mongo的YAML文件(StatefulSet)
[root@master1 mongo]# vi /root/my_yaml/mongo_cluster/mongo-sts.yaml
#添加如下内容
apiVersion: apps/v1
kind: StatefulSet
metadata:
namespace: ns-mongo
name: mongo-1
spec:
selector:
matchLabels:
name: mongo-1
serviceName: "mongo-1"
replicas: 1
podManagementPolicy: Parallel
template:
metadata:
labels:
name: mongo-1
app: mongo-cluster
spec:
containers:
- name: mongo
image: registry.cn-shenzhen.aliyuncs.com/hxlk8s/mongo:4.0.28.1
imagePullPolicy: IfNotPresent
command:
- mongod
- "-f"
- "/etc/mongod.conf"
ports:
- containerPort: 27017
volumeMounts:
- name: mongo-cnf-volume
mountPath: /etc/mongod.conf/
subPath: mongod.conf
- name: mongo-dir
mountPath: /mongo
volumes:
- name: mongo-cnf-volume #映射configMap信息
configMap:
name: mongodb-conf
items:
- key: mongod.conf
path: mongod.conf
volumeClaimTemplates:
- metadata:
name: mongo-dir
spec:
accessModes: [ "ReadWriteOnce" ]
resources:
requests:
storage: 2Gi
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
namespace: ns-mongo
name: mongo-2
spec:
selector:
matchLabels:
name: mongo-2
serviceName: "mongo-2"
replicas: 1
podManagementPolicy: Parallel
template:
metadata:
labels:
name: mongo-2
app: mongo-cluster
spec:
containers:
- name: mongo
image: registry.cn-shenzhen.aliyuncs.com/hxlk8s/mongo:4.0.28.1
imagePullPolicy: IfNotPresent
command:
- mongod
- "-f"
- "/etc/mongod.conf"
ports:
- containerPort: 27017
volumeMounts:
- name: mongo-cnf-volume
mountPath: /etc/mongod.conf/
subPath: mongod.conf
- name: mongo-dir
mountPath: /mongo
volumes:
- name: mongo-cnf-volume #映射configMap信息
configMap:
name: mongodb-conf
items:
- key: mongod.conf
path: mongod.conf
volumeClaimTemplates:
- metadata:
name: mongo-dir
spec:
accessModes: [ "ReadWriteOnce" ]
resources:
requests:
storage: 2Gi
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
namespace: ns-mongo
name: mongo-3
spec:
selector:
matchLabels:
name: mongo-3
serviceName: "mongo-3"
replicas: 1
podManagementPolicy: Parallel
template:
metadata:
labels:
name: mongo-3
app: mongo-cluster
spec:
containers:
- name: mongo
image: registry.cn-shenzhen.aliyuncs.com/hxlk8s/mongo:4.0.28.1
imagePullPolicy: IfNotPresent
command:
- mongod
- "-f"
- "/etc/mongod.conf"
ports:
- containerPort: 27017
volumeMounts:
- name: mongo-cnf-volume
mountPath: /etc/mongod.conf/
subPath: mongod.conf
- name: mongo-dir
mountPath: /mongo
volumes:
- name: mongo-cnf-volume #映射configMap信息
configMap:
name: mongodb-conf
items:
- key: mongod.conf
path: mongod.conf
volumeClaimTemplates:
- metadata:
name: mongo-dir
spec:
accessModes: [ "ReadWriteOnce" ]
resources:
requests:
storage: 2Gi
2.创建
[root@master mongo_cluster]# cd /root/my_yaml/mongo_cluster
[root@master mongo_cluster]# kubectl apply -f mongo-sts.yaml
3.查看
[root@master mongo_cluster]# kubectl get sts -n ns-mongo
NAME READY AGE
mongo-1 1/1 7s
mongo-2 1/1 7s
mongo-3 1/1 7s
#################################初始化mongo集群#########################
1.查看当前的pod
[root@master mongo_cluster]# kubectl get pods -n ns-mongo
NAME READY STATUS RESTARTS AGE
mongo-1-0 1/1 Running 0 33s
mongo-2-0 1/1 Running 0 33s
mongo-3-0 1/1 Running 0 33s
2.进入到第一个pod中
[root@master ~]# kubectl exec -it mongo-1-0 -n ns-mongo -- /bin/bash
3.初始化集群
mongo
use admin;
rs.initiate({ _id:"rs0", // replSet指定的名称
members:[{ _id:0, host:"mongo-1.ns-mongo.svc.cluster.local:27017" // 主节点ip与端口,
}]
})
将mongo-2和mongo-3加入到集群
rs0:PRIMARY> rs.add("mongo-2.ns-mongo.svc.cluster.local:27017")
rs0:PRIMARY> rs.add("mongo-3.ns-mongo.svc.cluster.local:27017")
4.查看集群状态
rs0:PRIMARY> rs.status();
{
"set" : "rs0",
"date" : ISODate("2025-02-24T09:32:15.828Z"),
"myState" : 1,
"term" : NumberLong(1),
"syncingTo" : "",
"syncSourceHost" : "",
"syncSourceId" : -1,
"heartbeatIntervalMillis" : NumberLong(2000),
"optimes" : {
"lastCommittedOpTime" : {
"ts" : Timestamp(1740389527, 1),
"t" : NumberLong(1)
},
"readConcernMajorityOpTime" : {
"ts" : Timestamp(1740389527, 1),
"t" : NumberLong(1)
},
"appliedOpTime" : {
"ts" : Timestamp(1740389527, 1),
"t" : NumberLong(1)
},
"durableOpTime" : {
"ts" : Timestamp(1740389527, 1),
"t" : NumberLong(1)
}
},
"lastStableCheckpointTimestamp" : Timestamp(1740389527, 1),
"electionCandidateMetrics" : {
"lastElectionReason" : "electionTimeout",
"lastElectionDate" : ISODate("2025-02-24T09:31:07.159Z"),
"electionTerm" : NumberLong(1),
"lastCommittedOpTimeAtElection" : {
"ts" : Timestamp(0, 0),
"t" : NumberLong(-1)
},
"lastSeenOpTimeAtElection" : {
"ts" : Timestamp(1740389467, 1),
"t" : NumberLong(-1)
},
"numVotesNeeded" : 1,
"priorityAtElection" : 1,
"electionTimeoutMillis" : NumberLong(10000),
"newTermStartDate" : ISODate("2025-02-24T09:31:07.165Z"),
"wMajorityWriteAvailabilityDate" : ISODate("2025-02-24T09:31:07.232Z")
},
"members" : [
{
"_id" : 0,
"name" : "mongo-1.ns-mongo.svc.cluster.local:27017",
"health" : 1,
"state" : 1,
"stateStr" : "PRIMARY",
"uptime" : 705,
"optime" : {
"ts" : Timestamp(1740389527, 1),
"t" : NumberLong(1)
},
"optimeDate" : ISODate("2025-02-24T09:32:07Z"),
"syncingTo" : "",
"syncSourceHost" : "",
"syncSourceId" : -1,
"infoMessage" : "could not find member to sync from",
"electionTime" : Timestamp(1740389467, 2),
"electionDate" : ISODate("2025-02-24T09:31:07Z"),
"configVersion" : 3,
"self" : true,
"lastHeartbeatMessage" : ""
},
{
"_id" : 1,
"name" : "mongo-2.ns-mongo.svc.cluster.local:27017",
"health" : 1,
"state" : 2,
"stateStr" : "SECONDARY",
"uptime" : 40,
"optime" : {
"ts" : Timestamp(1740389527, 1),
"t" : NumberLong(1)
},
"optimeDurable" : {
"ts" : Timestamp(1740389527, 1),
"t" : NumberLong(1)
},
"optimeDate" : ISODate("2025-02-24T09:32:07Z"),
"optimeDurableDate" : ISODate("2025-02-24T09:32:07Z"),
"lastHeartbeat" : ISODate("2025-02-24T09:32:14.518Z"),
"lastHeartbeatRecv" : ISODate("2025-02-24T09:32:15.079Z"),
"pingMs" : NumberLong(0),
"lastHeartbeatMessage" : "",
"syncingTo" : "mongo-1.ns-mongo.svc.cluster.local:27017",
"syncSourceHost" : "mongo-1.ns-mongo.svc.cluster.local:27017",
"syncSourceId" : 0,
"infoMessage" : "",
"configVersion" : 3
},
{
"_id" : 2,
"name" : "mongo-3.ns-mongo.svc.cluster.local:27017",
"health" : 1,
"state" : 2,
"stateStr" : "SECONDARY",
"uptime" : 33,
"optime" : {
"ts" : Timestamp(1740389527, 1),
"t" : NumberLong(1)
},
"optimeDurable" : {
"ts" : Timestamp(1740389527, 1),
"t" : NumberLong(1)
},
"optimeDate" : ISODate("2025-02-24T09:32:07Z"),
"optimeDurableDate" : ISODate("2025-02-24T09:32:07Z"),
"lastHeartbeat" : ISODate("2025-02-24T09:32:14.538Z"),
"lastHeartbeatRecv" : ISODate("2025-02-24T09:32:13.993Z"),
"pingMs" : NumberLong(0),
"lastHeartbeatMessage" : "",
"syncingTo" : "mongo-2.ns-mongo.svc.cluster.local:27017",
"syncSourceHost" : "mongo-2.ns-mongo.svc.cluster.local:27017",
"syncSourceId" : 1,
"infoMessage" : "",
"configVersion" : 3
}
],
"ok" : 1,
"operationTime" : Timestamp(1740389527, 1),
"$clusterTime" : {
"clusterTime" : Timestamp(1740389527, 1),
"signature" : {
"hash" : BinData(0,"AAAAAAAAAAAAAAAAAAAAAAAAAAA="),
"keyId" : NumberLong(0)
}
}
}
创建用户
use admin
db.createUser({user:"root",pwd:"root123",roles:["root"]}); --创建用户
db.auth("root","root123"); --设置用户登陆权限,密码一定要和创建用户时输入的密码相同
show users; --查看创建的用户
4.添加外部访问
[root@master mongo]# vi /root/my_yaml/mongo_cluster/nodeport.yaml
#添加如下内容
#master nodeport service
apiVersion: v1
kind: Service
metadata:
name: mongo-1-front-service
labels:
name: mongo-1
namespace: ns-mongo
spec:
selector:
name: mongo-1
type: NodePort
externalTrafficPolicy: Cluster
ports:
- name: mongo-http
nodePort: 30881
port: 27017
protocol: TCP
targetPort: 27017
---
#slave nodeport service
apiVersion: v1
kind: Service
metadata:
name: mongo-2-front-service
labels:
name: mongo-2
namespace: ns-mongo
spec:
selector:
name: mongo-2
type: NodePort
externalTrafficPolicy: Cluster
ports:
- name: mongo-http
nodePort: 30882
port: 27017
protocol: TCP
targetPort: 27017
---
#slave nodeport service
apiVersion: v1
kind: Service
metadata:
name: mongo-3-front-service
labels:
name: mongo-3
namespace: ns-mongo
spec:
selector:
name: mongo-3
type: NodePort
externalTrafficPolicy: Cluster
ports:
- name: mongo-http
nodePort: 30883
port: 27017
protocol: TCP
targetPort: 27017
3.应用
[root@master mongo_cluster]# cd /root/my_yaml/mongo_cluster
[root@master mongo_cluster]# kubectl apply -f nodeport.yaml
4.查看
[root@master mongo_cluster]# kubectl get svc -n ns-mongo
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
mongo-1 ClusterIP None <none> 27017/TCP 16m
mongo-1-front-service NodePort 10.101.45.116 <none> 27017:30881/TCP 16s
mongo-2 ClusterIP None <none> 27017/TCP 16m
mongo-2-front-service NodePort 10.98.129.5 <none> 27017:30882/TCP 16s
mongo-3 ClusterIP None <none> 27017/TCP 16m
mongo-3-front-service NodePort 10.99.103.247 <none> 27017:30883/TCP 16s
5.外部访问(任意一个node节点都可以访问)
/opt/mongodb/bin/mongo 192.168.1.102:30881
###############################启用密码登录方法1###################################
1.修改config.yaml启用密码认证
apiVersion: v1
data:
mongod.conf: |+
dbpath=/mongo/data
logpath=/mongo/mongodb.log
pidfilepath=/mongo/key/master.pid
directoryperdb=true
logappend=true
bind_ip=0.0.0.0
port=27017
replSet=rs0
keyFile=/data/config/mongodb-keyfile
auth=true
kind: ConfigMap
metadata:
name: mongodb-conf
namespace: ns-mongo
3.重新部署各应用
kubectl delete -f mongo-sts.yaml
kubectl delete -f service.yaml
kubectl delete -f config.yaml
kubectl apply -f config.yaml
kubectl apply -f service.yaml
kubectl apply -f mongo-sts.yaml
因为已经做了持久化，之前创建的账号数据不会丢失
4.密码登录
mongo -u root -p root123 --authenticationDatabase admin
###############################启用密码登录方法2###################################
1.安全认证参数不写在config
而是写在StatefulSet里
如下：
[root@master test1]# more 5-mongo-sts.yaml
apiVersion: apps/v1
kind: StatefulSet
metadata:
namespace: ns-mongo
name: mongo-1
spec:
selector:
matchLabels:
name: mongo-1
serviceName: "mongo-1"
replicas: 1
podManagementPolicy: Parallel
template:
metadata:
labels:
name: mongo-1
app: mongo-cluster
spec:
containers:
- name: mongo
image: registry.cn-shenzhen.aliyuncs.com/hxlk8s/mongo:4.0.28.1
imagePullPolicy: IfNotPresent
command:
- mongod
- "-f"
- "/etc/mongod.conf"
- "--keyFile"
- /data/config/mongodb-keyfile
ports:
- containerPort: 27017
volumeMounts:
- name: mongo-cnf-volume
mountPath: /etc/mongod.conf/
subPath: mongod.conf
- name: mongo-dir
mountPath: /mongo
volumes:
- name: mongo-cnf-volume #映射configMap信息
configMap:
name: mongodb-conf
items:
- key: mongod.conf
path: mongod.conf
volumeClaimTemplates:
- metadata:
name: mongo-dir
spec:
accessModes: [ "ReadWriteOnce" ]
resources:
requests:
storage: 2Gi
副本集创建完成后直接对StatefulSet重新部署即可，不涉及到其他的组件重新部署
【推荐】国内首个AI IDE，深度理解中文开发场景，立即下载体验Trae
【推荐】编程新体验，更懂你的AI，立即体验豆包MarsCode编程助手
【推荐】抖音旗下AI助手豆包，你的智能百科全书，全免费不限次数
【推荐】轻量又高性能的 SSH 工具 IShell：AI 加持，快人一步
· DeepSeek 开源周回顾「GitHub 热点速览」
· 物流快递公司核心技术能力-地址解析分单基础技术分享
· .NET 10首个预览版发布：重大改进与新特性概览！
· AI与.NET技术实操系列（二）：开始使用ML.NET
· 单线程的Redis速度为什么快？
2021-02-25 ios文件挂载到目录
2020-02-25 goldengate采用rman进行初始化
2020-02-25 该死的crontab环境变量问题
2019-02-25 输入时间参数获取rds备份集信息