Terraform 部署 Kubernetes
Terraform 部署 Kubernetes
依赖
k8s集群服务
创建 namespaces
查看当前 namespaces
[root@node devops]# kubectl get ns
NAME STATUS AGE
default Active 2d18h
devops Active 2d4h
kube-node-lease Active 2d18h
kube-public Active 2d18h
kube-system Active 2d18h
编写 Terraform 代码
- 查看
k8s provider
文档地址：https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs
- 文件结构
./
├── main.tf # 定义 k8s config 信息，namepaces 信息
└── versions.tf # 定义 provider 版本
- 定义
provider版本
## versions.tf
# 定义使用的 k8s provider 版本
terraform {
required_providers {
kubernetes = {
source = "hashicorp/kubernetes"
version = "2.12.1"
}
}
}
provider "kubernetes" {
# Configuration options
}
- 定义
network
## main.tf
provider "kubernetes" {
# Configuration options
# k8s 集群配置文件信息
config_path = "/root/.kube/config"
# context 信息
config_context = "context-cluster1"
# 别名
alias = "clustera"
insecure = true
}
resource "kubernetes_namespace" "jenkins" {
provider = kubernetes.clustera
metadata {
name = "jenkins"
}
}
- 创建
namespace
[root@node devops]# terraform fmt
[root@node devops]# terraform validate
Success! The configuration is valid.
[root@node devops]# terraform plan
Terraform used the selected providers to generate the following execution plan. Resource actions are indicated with the following symbols:
+ create
Terraform will perform the following actions:
# kubernetes_namespace.jenkins will be created
+ resource "kubernetes_namespace" "jenkins" {
+ id = (known after apply)
+ metadata {
+ generation = (known after apply)
+ name = "jenkins"
+ resource_version = (known after apply)
+ uid = (known after apply)
}
}
Plan: 1 to add, 0 to change, 0 to destroy.
[root@node devops]# terraform apply --auto-approve
Terraform used the selected providers to generate the following execution plan. Resource actions are indicated with the following symbols:
+ create
Terraform will perform the following actions:
# kubernetes_namespace.jenkins will be created
+ resource "kubernetes_namespace" "jenkins" {
+ id = (known after apply)
+ metadata {
+ generation = (known after apply)
+ name = "jenkins"
+ resource_version = (known after apply)
+ uid = (known after apply)
}
}
Plan: 1 to add, 0 to change, 0 to destroy.
kubernetes_namespace.jenkins: Creating...
kubernetes_namespace.jenkins: Creation complete after 0s [id=jenkins]
Apply complete! Resources: 1 added, 0 changed, 0 destroyed.
- 查看
namespaces信息
[root@node devops]# kubectl get ns
NAME STATUS AGE
default Active 2d18h
devops Active 2d4h
jenkins Active 3s # 新创建的 namespace
kube-node-lease Active 2d18h
kube-public Active 2d18h
kube-system Active 2d18h
创建 Deployment
查看 jenkins 名称空间 Deployment
[root@node devops]# kubectl -n jenkins get pods
No resources found in jenkins namespace.
编写 Terraform 代码
- 查看
k8s provider
文档地址：https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs
- 文件结构
./
├── jenins.tf # 定义 deployment信息
├── main.tf # 定义 k8s config 信息，namepaces 信息
└── versions.tf # 定义 provider 版本
main.tfversion.tf配置见上文namepaces创建
- 创建
Deployment
## jenkins.tf
resource "kubernetes_deployment_v1" "jenkins" {
# 使用的 k8s 配置文件信息，引用 main.tf 中定义别名
provider = kubernetes.clustera
metadata {
name = "jenkins"
labels = {
app = "jenkins"
}
# deployment 资源的名称空间，引用 main.tf 中定义
namespace = kubernetes_namespace.jenkins.id
}
spec {
# 资源副本数
replicas = 1
selector {
match_labels = {
app = "jenkins"
}
}
template {
metadata {
labels = {
app = "jenkins"
}
}
spec {
container {
image = "jenkins/jenkins:2.401.3-lts"
name = "jenkins"
image_pull_policy = "IfNotPresent"
port {
container_port = 8080
}
resources {
limits = {
cpu = "1000m"
memory = "4096Mi"
}
requests = {
cpu = "250m"
memory = "1024Mi"
}
}
# liveness_probe {
# http_get {
# path = "/"
# port = 8080
# }
# initial_delay_seconds = 30
# period_seconds = 3
# }
}
}
}
}
}
- 创建
Deployment
[root@node devops]# terraform plan
kubernetes_namespace.jenkins: Refreshing state... [id=jenkins]
Terraform used the selected providers to generate the following execution plan. Resource actions are indicated with the following symbols:
+ create
Terraform will perform the following actions:
# kubernetes_deployment_v1.jenkins will be created
+ resource "kubernetes_deployment_v1" "jenkins" {
+ id = (known after apply)
+ wait_for_rollout = true
+ metadata {
+ generation = (known after apply)
+ labels = {
+ "app" = "jenkins"
}
+ name = "jenkins"
+ namespace = "jenkins"
+ resource_version = (known after apply)
+ uid = (known after apply)
}
+ spec {
+ min_ready_seconds = 0
+ paused = false
+ progress_deadline_seconds = 600
+ replicas = "1"
+ revision_history_limit = 10
+ selector {
+ match_labels = {
+ "app" = "jenkins"
}
}
+ template {
+ metadata {
+ generation = (known after apply)
+ labels = {
+ "app" = "jenkins"
}
+ name = (known after apply)
+ resource_version = (known after apply)
+ uid = (known after apply)
}
+ spec {
+ automount_service_account_token = true
+ dns_policy = "ClusterFirst"
+ enable_service_links = true
+ host_ipc = false
+ host_network = false
+ host_pid = false
+ hostname = (known after apply)
+ node_name = (known after apply)
+ restart_policy = "Always"
+ service_account_name = (known after apply)
+ share_process_namespace = false
+ termination_grace_period_seconds = 30
+ container {
+ image = "jenkins/jenkins:2.401.3-lts"
+ image_pull_policy = "IfNotPresent"
+ name = "jenkins"
+ stdin = false
+ stdin_once = false
+ termination_message_path = "/dev/termination-log"
+ termination_message_policy = (known after apply)
+ tty = false
+ port {
+ container_port = 8080
+ protocol = "TCP"
}
+ resources {
+ limits = {
+ "cpu" = "1000m"
+ "memory" = "4096Mi"
}
+ requests = {
+ "cpu" = "250m"
+ "memory" = "1024Mi"
}
}
}
}
}
}
}
Plan: 1 to add, 0 to change, 0 to destroy.
Note: You didn't use the -out option to save this plan, so Terraform can't guarantee to take exactly these actions if you run "terraform apply" now.
[root@node devops]# terraform apply --auto-approve
kubernetes_namespace.jenkins: Refreshing state... [id=jenkins]
Terraform used the selected providers to generate the following execution plan. Resource actions are indicated with the following symbols:
+ create
Terraform will perform the following actions:
# kubernetes_deployment_v1.jenkins will be created
+ resource "kubernetes_deployment_v1" "jenkins" {
+ id = (known after apply)
+ wait_for_rollout = true
+ metadata {
+ generation = (known after apply)
+ labels = {
+ "app" = "jenkins"
}
+ name = "jenkins"
+ namespace = "jenkins"
+ resource_version = (known after apply)
+ uid = (known after apply)
}
+ spec {
+ min_ready_seconds = 0
+ paused = false
+ progress_deadline_seconds = 600
+ replicas = "1"
+ revision_history_limit = 10
+ selector {
+ match_labels = {
+ "app" = "jenkins"
}
}
+ template {
+ metadata {
+ generation = (known after apply)
+ labels = {
+ "app" = "jenkins"
}
+ name = (known after apply)
+ resource_version = (known after apply)
+ uid = (known after apply)
}
+ spec {
+ automount_service_account_token = true
+ dns_policy = "ClusterFirst"
+ enable_service_links = true
+ host_ipc = false
+ host_network = false
+ host_pid = false
+ hostname = (known after apply)
+ node_name = (known after apply)
+ restart_policy = "Always"
+ service_account_name = (known after apply)
+ share_process_namespace = false
+ termination_grace_period_seconds = 30
+ container {
+ image = "jenkins/jenkins:2.401.3-lts"
+ image_pull_policy = "IfNotPresent"
+ name = "jenkins"
+ stdin = false
+ stdin_once = false
+ termination_message_path = "/dev/termination-log"
+ termination_message_policy = (known after apply)
+ tty = false
+ port {
+ container_port = 8080
+ protocol = "TCP"
}
+ resources {
+ limits = {
+ "cpu" = "1000m"
+ "memory" = "4096Mi"
}
+ requests = {
+ "cpu" = "250m"
+ "memory" = "1024Mi"
}
}
}
}
}
}
}
Plan: 1 to add, 0 to change, 0 to destroy.
kubernetes_deployment_v1.jenkins: Creating...
kubernetes_deployment_v1.jenkins: Creation complete after 1s [id=jenkins/jenkins]
Apply complete! Resources: 1 added, 0 changed, 0 destroyed.
- 查看
jenkins名称空间Deployment
[root@node devops]# kubectl -n jenkins get pods
NAME READY STATUS RESTARTS AGE
jenkins-7756db549c-vxg8l 1/1 Running 0 75s
[root@node devops]# kubectl -n jenkins get deployments.apps -o wide
NAME READY UP-TO-DATE AVAILABLE AGE CONTAINERS IMAGES SELECTOR
jenkins 1/1 1 1 83s jenkins jenkins/jenkins:2.401.3-lts app=jenkins
创建 Service
查看 jenkins 名称空间 Service
[root@node devops]# kubectl -n jenkins get svc
No resources found in jenkins namespace.
编写 Terraform 代码
- 查看
k8s provider
文档地址：https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs
- 文件结构
./
├── jenins.tf # 定义 deployment 信息，service 信息
├── main.tf # 定义 k8s config 信息，namepaces 信息
└── versions.tf # 定义 provider 版本
main.tfversion.tf配置见上文namepaces创建
- 创建
Service
## jenins.tf
# deployment 信息
......
# service 信息
resource "kubernetes_service_v1" "jenkins" {
provider = kubernetes.clustera
metadata {
name = "jenkins-service"
namespace = kubernetes_namespace.jenkins.id
}
spec {
# service 标签选择，使用 deployment 中定义的标签
selector = {
app = kubernetes_deployment_v1.jenkins.metadata[0].labels.app
}
port {
port = 8080
target_port = 8080
}
type = "ClusterIP"
}
}
- 创建
Service
[root@node devops]# terraform plan
kubernetes_namespace.jenkins: Refreshing state... [id=jenkins]
kubernetes_deployment_v1.jenkins: Refreshing state... [id=jenkins/jenkins]
Terraform used the selected providers to generate the following execution plan. Resource actions are indicated with the following symbols:
+ create
Terraform will perform the following actions:
# kubernetes_service_v1.jenkins will be created
+ resource "kubernetes_service_v1" "jenkins" {
+ id = (known after apply)
+ status = (known after apply)
+ wait_for_load_balancer = true
+ metadata {
+ generation = (known after apply)
+ name = "jenkins-service"
+ namespace = "jenkins"
+ resource_version = (known after apply)
+ uid = (known after apply)
}
+ spec {
+ allocate_load_balancer_node_ports = true
+ cluster_ip = (known after apply)
+ cluster_ips = (known after apply)
+ external_traffic_policy = (known after apply)
+ health_check_node_port = (known after apply)
+ internal_traffic_policy = (known after apply)
+ ip_families = (known after apply)
+ ip_family_policy = (known after apply)
+ publish_not_ready_addresses = false
+ selector = {
+ "app" = "jenkins"
}
+ session_affinity = "None"
+ type = "ClusterIP"
+ port {
+ node_port = (known after apply)
+ port = 8080
+ protocol = "TCP"
+ target_port = "8080"
}
}
}
Plan: 1 to add, 0 to change, 0 to destroy.
Note: You didn't use the -out option to save this plan, so Terraform can't guarantee to take exactly these actions if you run "terraform apply" now.
[root@node devops]# terraform apply --auto-approve
kubernetes_namespace.jenkins: Refreshing state... [id=jenkins]
kubernetes_deployment_v1.jenkins: Refreshing state... [id=jenkins/jenkins]
Terraform used the selected providers to generate the following execution plan. Resource actions are indicated with the following symbols:
+ create
Terraform will perform the following actions:
# kubernetes_service_v1.jenkins will be created
+ resource "kubernetes_service_v1" "jenkins" {
+ id = (known after apply)
+ status = (known after apply)
+ wait_for_load_balancer = true
+ metadata {
+ generation = (known after apply)
+ name = "jenkins-service"
+ namespace = "jenkins"
+ resource_version = (known after apply)
+ uid = (known after apply)
}
+ spec {
+ allocate_load_balancer_node_ports = true
+ cluster_ip = (known after apply)
+ cluster_ips = (known after apply)
+ external_traffic_policy = (known after apply)
+ health_check_node_port = (known after apply)
+ internal_traffic_policy = (known after apply)
+ ip_families = (known after apply)
+ ip_family_policy = (known after apply)
+ publish_not_ready_addresses = false
+ selector = {
+ "app" = "jenkins"
}
+ session_affinity = "None"
+ type = "ClusterIP"
+ port {
+ node_port = (known after apply)
+ port = 8080
+ protocol = "TCP"
+ target_port = "8080"
}
}
}
Plan: 1 to add, 0 to change, 0 to destroy.
kubernetes_service_v1.jenkins: Creating...
kubernetes_service_v1.jenkins: Creation complete after 0s [id=jenkins/jenkins-service]
Apply complete! Resources: 1 added, 0 changed, 0 destroyed.
- 查看
jenkins名称空间Service
[root@node devops]# kubectl -n jenkins get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
jenkins-service ClusterIP 10.68.95.56 <none> 8080/TCP 23s
[root@node devops]# kubectl -n jenkins get pods -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
jenkins-7756db549c-vxg8l 1/1 Running 0 15m 172.20.167.136 192.168.0.101 <none> <none>
[root@node devops]# kubectl -n jenkins describe svc jenkins-service
Name: jenkins-service
Namespace: jenkins
Labels: <none>
Annotations: <none>
Selector: app=jenkins
Type: ClusterIP
IP Family Policy: SingleStack
IP Families: IPv4
IP: 10.68.95.56
IPs: 10.68.95.56
Port: <unset> 8080/TCP
TargetPort: 8080/TCP
Endpoints: 172.20.167.136:8080
Session Affinity: None
Events: <none>
创建 Ingress
查看 jenkins 名称空间 Ingress
[root@node devops]# kubectl -n jenkins get ingress
No resources found in jenkins namespace.
编写 Terraform 代码
- 查看
k8s provider
文档地址：https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs
- 文件结构
./
├── jenins.tf # 定义 deployment 信息，service 信息，ingress 信息
├── main.tf # 定义 k8s config 信息，namepaces 信息
└── versions.tf # 定义 provider 版本
main.tfversion.tf配置见上文namepaces创建
- 创建
Ingress
## jenins.tf
# deployment 信息
......
# service 信息
......
# ingress 信息
resource "kubernetes_ingress_v1" "jenkins_ingress" {
provider = kubernetes.clustera
metadata {
name = "jenkins-ingress"
namespace = kubernetes_namespace.jenkins.id
}
spec {
default_backend {
service {
name = kubernetes_service_v1.jenkins.metadata[0].name
port {
number = 8080
}
}
}
rule {
host = "jenkins.evescn.com"
http {
path {
backend {
service {
name = kubernetes_service_v1.jenkins.metadata[0].name
port {
number = 8080
}
}
}
path_type = "Prefix"
path = "/"
}
}
}
}
}
- 创建
Ingress
[root@node devops]# terraform plan
kubernetes_namespace.jenkins: Refreshing state... [id=jenkins]
kubernetes_deployment_v1.jenkins: Refreshing state... [id=jenkins/jenkins]
kubernetes_service_v1.jenkins: Refreshing state... [id=jenkins/jenkins-service]
kubernetes_ingress_v1.jenkins_ingress: Refreshing state... [id=jenkins/jenkins-ingress]
Terraform used the selected providers to generate the following execution plan. Resource actions are indicated with the following symbols:
+ create
Terraform will perform the following actions:
# kubernetes_ingress_v1.jenkins_ingress will be created
+ resource "kubernetes_ingress_v1" "jenkins_ingress" {
+ id = (known after apply)
+ status = (known after apply)
+ metadata {
+ generation = (known after apply)
+ name = "jenkins-ingress"
+ namespace = "jenkins"
+ resource_version = (known after apply)
+ uid = (known after apply)
}
+ spec {
+ default_backend {
+ service {
+ name = "jenkins-service"
+ port {
+ number = 8080
}
}
}
+ rule {
+ host = "jenkins.evescn.com"
+ http {
+ path {
+ path = "/"
+ path_type = "Prefix"
+ backend {
+ service {
+ name = "jenkins-service"
+ port {
+ number = 8080
}
}
}
}
}
}
}
}
Plan: 1 to add, 0 to change, 0 to destroy.
Note: You didn't use the -out option to save this plan, so Terraform can't guarantee to take exactly these actions if you run "terraform apply" now.
[root@node devops]# terraform apply --auto-approve
kubernetes_namespace.jenkins: Refreshing state... [id=jenkins]
kubernetes_deployment_v1.jenkins: Refreshing state... [id=jenkins/jenkins]
kubernetes_service_v1.jenkins: Refreshing state... [id=jenkins/jenkins-service]
kubernetes_ingress_v1.jenkins_ingress: Refreshing state... [id=jenkins/jenkins-ingress]
Terraform used the selected providers to generate the following execution plan. Resource actions are indicated with the following symbols:
+ create
Terraform will perform the following actions:
# kubernetes_ingress_v1.jenkins_ingress will be created
+ resource "kubernetes_ingress_v1" "jenkins_ingress" {
+ id = (known after apply)
+ status = (known after apply)
+ metadata {
+ generation = (known after apply)
+ name = "jenkins-ingress"
+ namespace = "jenkins"
+ resource_version = (known after apply)
+ uid = (known after apply)
}
+ spec {
+ default_backend {
+ service {
+ name = "jenkins-service"
+ port {
+ number = 8080
}
}
}
+ rule {
+ host = "jenkins.evescn.com"
+ http {
+ path {
+ path = "/"
+ path_type = "Prefix"
+ backend {
+ service {
+ name = "jenkins-service"
+ port {
+ number = 8080
}
}
}
}
}
}
}
}
Plan: 1 to add, 0 to change, 0 to destroy.
kubernetes_ingress_v1.jenkins_ingress: Creating...
kubernetes_ingress_v1.jenkins_ingress: Creation complete after 0s [id=jenkins/jenkins-ingress]
Apply complete! Resources: 1 added, 0 changed, 0 destroyed.
- 查看
jenkins名称空间Ingress
[root@node devops]# kubectl -n jenkins get ingress
NAME CLASS HOSTS ADDRESS PORTS AGE
jenkins-ingress <none> jenkins.evescn.com 80 22s
[root@node devops]# kubectl -n jenkins get pods -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
jenkins-7756db549c-vxg8l 1/1 Running 0 15m 172.20.167.136 192.168.0.101 <none> <none>
[root@node devops]# kubectl -n jenkins describe ingress jenkins-ingress
Name: jenkins-ingress
Labels: <none>
Namespace: jenkins
Address:
Default backend: jenkins-service:8080 (172.20.167.136:8080)
Rules:
Host Path Backends
---- ---- --------
jenkins.evescn.com
/ jenkins-service:8080 (172.20.167.136:8080)
Annotations: <none>
Events: <none>
