基于 Terraform 构建企业内部 kubernetes 集群

terraform {
  required_version = ">= 1.6.0"

  required_providers {
    aws = {
      source  = "hashicorp/aws"
      version = "~> 5.40"
    }
    random = {
      source  = "hashicorp/random"
      version = "~> 3.6"
    }
    tls = {
      source  = "hashicorp/tls"
      version = "~> 4.0"
    }
  }

  backend "s3" {
    bucket         = "your-company-terraform-state"
    key            = "rke2-cluster/terraform.tfstate"
    region         = "ap-southeast-1"
    encrypt        = true
    dynamodb_table = "terraform-state-lock"
  }
}

provider "aws" {
  region = var.aws_region

  default_tags {
    tags = {
      Project     = "rke2-k8s-cluster"
      Environment = var.environment
      ManagedBy   = "Terraform"
    }
  }
}

# VPC
resource "aws_vpc" "main" {
  cidr_block           = var.vpc_cidr          # 10.0.0.0/16
  enable_dns_hostnames = true
  enable_dns_support   = true

  tags = {
    Name = "${var.cluster_name}-vpc"
  }
}

# 互联网网关
resource "aws_internet_gateway" "main" {
  vpc_id = aws_vpc.main.id

  tags = {
    Name = "${var.cluster_name}-igw"
  }
}

# 公有子网（Bastion / NLB）
resource "aws_subnet" "public" {
  count             = length(var.availability_zones)
  vpc_id            = aws_vpc.main.id
  cidr_block        = cidrsubnet(var.vpc_cidr, 8, count.index)
  availability_zone = var.availability_zones[count.index]

  map_public_ip_on_launch = true

  tags = {
    Name = "${var.cluster_name}-public-${count.index + 1}"
    "kubernetes.io/role/elb" = "1"
  }
}

# 私有子网（Control Plane / Worker Nodes）
resource "aws_subnet" "private" {
  count             = length(var.availability_zones)
  vpc_id            = aws_vpc.main.id
  cidr_block        = cidrsubnet(var.vpc_cidr, 8, count.index + 10)
  availability_zone = var.availability_zones[count.index]

  tags = {
    Name = "${var.cluster_name}-private-${count.index + 1}"
    "kubernetes.io/role/internal-elb" = "1"
  }
}

# NAT 网关（每个 AZ 一个，生产高可用要求）
resource "aws_eip" "nat" {
  count  = length(var.availability_zones)
  domain = "vpc"
}

resource "aws_nat_gateway" "main" {
  count         = length(var.availability_zones)
  allocation_id = aws_eip.nat[count.index].id
  subnet_id     = aws_subnet.public[count.index].id

  tags = {
    Name = "${var.cluster_name}-nat-${count.index + 1}"
  }
}

# 私有子网路由表
resource "aws_route_table" "private" {
  count  = length(var.availability_zones)
  vpc_id = aws_vpc.main.id

  route {
    cidr_block     = "0.0.0.0/0"
    nat_gateway_id = aws_nat_gateway.main[count.index].id
  }

  tags = {
    Name = "${var.cluster_name}-private-rt-${count.index + 1}"
  }
}

resource "aws_route_table_association" "private" {
  count          = length(var.availability_zones)
  subnet_id      = aws_subnet.private[count.index].id
  route_table_id = aws_route_table.private[count.index].id
}

# Control Plane 安全组
resource "aws_security_group" "control_plane" {
  name        = "${var.cluster_name}-control-plane-sg"
  description = "RKE2 Control Plane Security Group"
  vpc_id      = var.vpc_id

  # RKE2 API Server
  ingress {
    from_port   = 6443
    to_port     = 6443
    protocol    = "tcp"
    cidr_blocks = [var.vpc_cidr]
    description = "Kubernetes API Server"
  }

  # RKE2 Supervisor API
  ingress {
    from_port   = 9345
    to_port     = 9345
    protocol    = "tcp"
    cidr_blocks = [var.vpc_cidr]
    description = "RKE2 Supervisor API"
  }

  # etcd
  ingress {
    from_port       = 2379
    to_port         = 2380
    protocol        = "tcp"
    self            = true
    description     = "etcd client and peer"
  }

  # Calico BGP
  ingress {
    from_port   = 179
    to_port     = 179
    protocol    = "tcp"
    cidr_blocks = [var.vpc_cidr]
    description = "Calico BGP"
  }

  # Calico VXLAN
  ingress {
    from_port   = 4789
    to_port     = 4789
    protocol    = "udp"
    cidr_blocks = [var.vpc_cidr]
    description = "Calico VXLAN"
  }

  # Kubelet
  ingress {
    from_port   = 10250
    to_port     = 10250
    protocol    = "tcp"
    cidr_blocks = [var.vpc_cidr]
    description = "Kubelet"
  }

  egress {
    from_port   = 0
    to_port     = 0
    protocol    = "-1"
    cidr_blocks = ["0.0.0.0/0"]
  }

  tags = {
    Name = "${var.cluster_name}-control-plane-sg"
  }
}

# Worker Node 安全组
resource "aws_security_group" "worker" {
  name        = "${var.cluster_name}-worker-sg"
  description = "RKE2 Worker Node Security Group"
  vpc_id      = var.vpc_id

  # NodePort 范围
  ingress {
    from_port   = 30000
    to_port     = 32767
    protocol    = "tcp"
    cidr_blocks = [var.vpc_cidr]
    description = "NodePort Services"
  }

  # Kubelet
  ingress {
    from_port   = 10250
    to_port     = 10250
    protocol    = "tcp"
    cidr_blocks = [var.vpc_cidr]
    description = "Kubelet"
  }

  # Calico BGP & VXLAN
  ingress {
    from_port   = 179
    to_port     = 179
    protocol    = "tcp"
    cidr_blocks = [var.vpc_cidr]
  }

  ingress {
    from_port   = 4789
    to_port     = 4789
    protocol    = "udp"
    cidr_blocks = [var.vpc_cidr]
  }

  egress {
    from_port   = 0
    to_port     = 0
    protocol    = "-1"
    cidr_blocks = ["0.0.0.0/0"]
  }

  tags = {
    Name = "${var.cluster_name}-worker-sg"
  }
}

# Control Plane IAM Role
resource "aws_iam_role" "control_plane" {
  name = "${var.cluster_name}-control-plane-role"

  assume_role_policy = jsonencode({
    Version = "2012-10-17"
    Statement = [{
      Action    = "sts:AssumeRole"
      Effect    = "Allow"
      Principal = { Service = "ec2.amazonaws.com" }
    }]
  })
}

resource "aws_iam_role_policy" "control_plane" {
  name = "${var.cluster_name}-control-plane-policy"
  role = aws_iam_role.control_plane.id

  policy = jsonencode({
    Version = "2012-10-17"
    Statement = [
      {
        Effect = "Allow"
        Action = [
          "ec2:DescribeInstances",
          "ec2:DescribeRegions",
          "ec2:DescribeRouteTables",
          "ec2:DescribeSecurityGroups",
          "ec2:DescribeSubnets",
          "ec2:DescribeVolumes",
          "ec2:DescribeVpcs",
          "ec2:CreateSecurityGroup",
          "ec2:CreateTags",
          "ec2:CreateVolume",
          "ec2:ModifyInstanceAttribute",
          "ec2:AttachVolume",
          "ec2:DetachVolume",
          "elasticloadbalancing:*",
          "ecr:GetAuthorizationToken",
          "ecr:BatchCheckLayerAvailability",
          "ecr:GetDownloadUrlForLayer",
          "ecr:GetRepositoryPolicy",
          "ecr:DescribeRepositories",
          "ecr:ListImages",
          "ecr:BatchGetImage",
          "s3:GetObject",
          "s3:ListBucket"
        ]
        Resource = "*"
      }
    ]
  })
}

resource "aws_iam_instance_profile" "control_plane" {
  name = "${var.cluster_name}-control-plane-profile"
  role = aws_iam_role.control_plane.name
}

# Worker Node IAM Role（权限收窄）
resource "aws_iam_role" "worker" {
  name = "${var.cluster_name}-worker-role"

  assume_role_policy = jsonencode({
    Version = "2012-10-17"
    Statement = [{
      Action    = "sts:AssumeRole"
      Effect    = "Allow"
      Principal = { Service = "ec2.amazonaws.com" }
    }]
  })
}

resource "aws_iam_role_policy" "worker" {
  name = "${var.cluster_name}-worker-policy"
  role = aws_iam_role.worker.id

  policy = jsonencode({
    Version = "2012-10-17"
    Statement = [
      {
        Effect = "Allow"
        Action = [
          "ec2:DescribeInstances",
          "ec2:DescribeRegions",
          "ecr:GetAuthorizationToken",
          "ecr:BatchCheckLayerAvailability",
          "ecr:GetDownloadUrlForLayer",
          "ecr:GetRepositoryPolicy",
          "ecr:DescribeRepositories",
          "ecr:ListImages",
          "ecr:BatchGetImage"
        ]
        Resource = "*"
      }
    ]
  })
}

resource "aws_iam_instance_profile" "worker" {
  name = "${var.cluster_name}-worker-profile"
  role = aws_iam_role.worker.name
}

# 生成集群 Token
resource "random_password" "rke2_token" {
  length  = 64
  special = false
}

# 将 Token 存入 SSM Parameter Store
resource "aws_ssm_parameter" "rke2_token" {
  name  = "/${var.cluster_name}/rke2-token"
  type  = "SecureString"
  value = random_password.rke2_token.result
}

# 首个 Server 节点（Init 节点）
resource "aws_instance" "control_plane_init" {
  ami                    = var.ami_id
  instance_type          = "m5.xlarge"
  subnet_id              = var.private_subnet_ids[0]
  vpc_security_group_ids = [var.control_plane_sg_id]
  iam_instance_profile   = var.control_plane_instance_profile
  key_name               = var.key_pair_name

  root_block_device {
    volume_size = 100
    volume_type = "gp3"
    encrypted   = true
  }

  user_data = base64encode(templatefile("${path.module}/../../scripts/rke2-server-init.sh", {
    rke2_version     = var.rke2_version
    cluster_name     = var.cluster_name
    rke2_token       = random_password.rke2_token.result
    aws_region       = var.aws_region
    pod_cidr         = var.pod_cidr
    service_cidr     = var.service_cidr
  }))

  tags = {
    Name = "${var.cluster_name}-control-plane-init"
    "kubernetes.io/cluster/${var.cluster_name}" = "owned"
    "rke2-node-role" = "server"
  }
}

# NLB（API Server 高可用入口）
resource "aws_lb" "api_server" {
  name               = "${var.cluster_name}-api-nlb"
  internal           = true
  load_balancer_type = "network"
  subnets            = var.private_subnet_ids
}

resource "aws_lb_target_group" "api_server" {
  name     = "${var.cluster_name}-api-tg"
  port     = 6443
  protocol = "TCP"
  vpc_id   = var.vpc_id

  health_check {
    protocol            = "HTTPS"
    path                = "/healthz"
    port                = "6443"
    healthy_threshold   = 3
    unhealthy_threshold = 3
  }
}

resource "aws_lb_listener" "api_server" {
  load_balancer_arn = aws_lb.api_server.arn
  port              = 6443
  protocol          = "TCP"

  default_action {
    type             = "forward"
    target_group_arn = aws_lb_target_group.api_server.arn
  }
}

# 追加 2 个 Server 节点
resource "aws_instance" "control_plane_join" {
  count                  = 2
  ami                    = var.ami_id
  instance_type          = "m5.xlarge"
  subnet_id              = var.private_subnet_ids[count.index + 1]
  vpc_security_group_ids = [var.control_plane_sg_id]
  iam_instance_profile   = var.control_plane_instance_profile
  key_name               = var.key_pair_name

  root_block_device {
    volume_size = 100
    volume_type = "gp3"
    encrypted   = true
  }

  user_data = base64encode(templatefile("${path.module}/../../scripts/rke2-server-join.sh", {
    rke2_version  = var.rke2_version
    rke2_token    = random_password.rke2_token.result
    server_url    = "https://${aws_lb.api_server.dns_name}:9345"
  }))

  depends_on = [aws_instance.control_plane_init]

  tags = {
    Name = "${var.cluster_name}-control-plane-${count.index + 2}"
    "kubernetes.io/cluster/${var.cluster_name}" = "owned"
    "rke2-node-role" = "server"
  }
}

#!/bin/bash
set -eux

# 系统初始化
swapoff -a
sed -i '/swap/d' /etc/fstab

# 内核参数
cat >> /etc/sysctl.d/99-k8s.conf <<EOF
net.bridge.bridge-nf-call-iptables  = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward                 = 1
EOF
sysctl --system

# 安装 RKE2
curl -sfL https://get.rke2.io | INSTALL_RKE2_VERSION="${rke2_version}" sh -

# 配置 RKE2 Server
mkdir -p /etc/rancher/rke2

cat > /etc/rancher/rke2/config.yaml <<EOF
token: "${rke2_token}"
tls-san:
  - "${server_lb_dns}"
cluster-cidr: "${pod_cidr}"
service-cidr: "${service_cidr}"
cni: none
disable:
  - rke2-canal
cloud-provider-name: aws
node-label:
  - "node.kubernetes.io/role=control-plane"
EOF

# 启动 RKE2 Server
systemctl enable rke2-server.service
systemctl start rke2-server.service

# 等待 kubeconfig 生成
until [ -f /etc/rancher/rke2/rke2.yaml ]; do sleep 5; done

# 配置 kubectl
mkdir -p /root/.kube
cp /etc/rancher/rke2/rke2.yaml /root/.kube/config
export KUBECONFIG=/etc/rancher/rke2/rke2.yaml
ln -sf /var/lib/rancher/rke2/bin/kubectl /usr/local/bin/kubectl

echo "RKE2 Server Init Completed"

# 启动模板
resource "aws_launch_template" "worker" {
  name_prefix   = "${var.cluster_name}-worker-"
  image_id      = var.ami_id
  instance_type = "m5.2xlarge"
  key_name      = var.key_pair_name

  vpc_security_group_ids = [var.worker_sg_id]

  iam_instance_profile {
    name = var.worker_instance_profile
  }

  block_device_mappings {
    device_name = "/dev/sda1"
    ebs {
      volume_size           = 200
      volume_type           = "gp3"
      encrypted             = true
      delete_on_termination = true
    }
  }

  user_data = base64encode(templatefile("${path.module}/../../scripts/rke2-agent.sh", {
    rke2_version = var.rke2_version
    rke2_token   = var.rke2_token
    server_url   = var.server_url
  }))

  tag_specifications {
    resource_type = "instance"
    tags = {
      Name = "${var.cluster_name}-worker"
      "kubernetes.io/cluster/${var.cluster_name}" = "owned"
      "rke2-node-role" = "agent"
    }
  }

  lifecycle {
    create_before_destroy = true
  }
}

# Auto Scaling Group
resource "aws_autoscaling_group" "worker" {
  name                = "${var.cluster_name}-worker-asg"
  desired_capacity    = 50
  min_size            = 10
  max_size            = 60
  vpc_zone_identifier = var.private_subnet_ids

  launch_template {
    id      = aws_launch_template.worker.id
    version = "$Latest"
  }

  health_check_type         = "EC2"
  health_check_grace_period = 300

  tag {
    key                 = "Name"
    value               = "${var.cluster_name}-worker"
    propagate_at_launch = true
  }

  tag {
    key                 = "kubernetes.io/cluster/${var.cluster_name}"
    value               = "owned"
    propagate_at_launch = true
  }
}

#!/bin/bash
set -eux

swapoff -a
sed -i '/swap/d' /etc/fstab

cat >> /etc/sysctl.d/99-k8s.conf <<EOF
net.bridge.bridge-nf-call-iptables  = 1
net.ipv4.ip_forward                 = 1
EOF
sysctl --system

curl -sfL https://get.rke2.io | INSTALL_RKE2_TYPE="agent" INSTALL_RKE2_VERSION="${rke2_version}" sh -

mkdir -p /etc/rancher/rke2

cat > /etc/rancher/rke2/config.yaml <<EOF
server: "${server_url}"
token: "${rke2_token}"
cloud-provider-name: aws
node-label:
  - "node.kubernetes.io/role=worker"
EOF

systemctl enable rke2-agent.service
systemctl start rke2-agent.service

echo "RKE2 Agent Join Completed"

apiVersion: operator.tigera.io/v1
kind: Installation
metadata:
  name: default
spec:
  calicoNetwork:
    ipPools:
    - blockSize: 26
      cidr: 192.168.0.0/16
      encapsulation: VXLAN
      natOutgoing: Enabled
      nodeSelector: all()
  nodeAddressAutodetectionV4:
    interface: eth0
---
apiVersion: operator.tigera.io/v1
kind: APIServer
metadata:
  name: default
spec: {}

# 1. 安装 Tigera Operator
kubectl create -f https://raw.githubusercontent.com/projectcalico/calico/v3.27.0/manifests/tigera-operator.yaml

# 2. 应用自定义配置
kubectl create -f calico/custom-resources.yaml

# 3. 等待 Calico 组件就绪
kubectl wait --for=condition=Ready pods -n calico-system --all --timeout=300s

# 4. 验证节点状态
kubectl get nodes

# 初始化
terraform init

# 预览变更
terraform plan -var-file="terraform.tfvars"

# 执行部署
terraform apply -var-file="terraform.tfvars" -auto-approve

# 查看输出
terraform output

# 检查节点状态
kubectl get nodes -o wide

# 检查核心组件
kubectl get pods -n kube-system
kubectl get pods -n calico-system

# 检查集群信息
kubectl cluster-info

# 部署测试 Deployment
kubectl create deployment nginx-test --image=nginx --replicas=3
kubectl get pods -o wide
kubectl delete deployment nginx-test

# 通过 Helm 安装 kube-prometheus-stack
helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
helm repo update

helm upgrade --install kube-prometheus-stack prometheus-community/kube-prometheus-stack \
  --namespace monitoring \
  --create-namespace \
  --set prometheus.prometheusSpec.retention=15d \
  --set prometheus.prometheusSpec.storageSpec.volumeClaimTemplate.spec.storageClassName=gp3 \
  --set prometheus.prometheusSpec.storageSpec.volumeClaimTemplate.spec.resources.requests.storage=100Gi \
  --set grafana.adminPassword="your-secure-password" \
  --set alertmanager.enabled=true

helm upgrade --install loki grafana/loki-stack \
  --namespace monitoring \
  --set promtail.enabled=true \
  --set loki.persistence.enabled=true \
  --set loki.persistence.size=50Gi

apiVersion: monitoring.coreos.com/v1alpha1
kind: AlertmanagerConfig
metadata:
  name: cluster-alerts
  namespace: monitoring
spec:
  route:
    receiver: dingtalk-webhook
    groupBy: ['alertname', 'namespace']
    groupWait: 30s
    groupInterval: 5m
    repeatInterval: 4h
  receivers:
    - name: dingtalk-webhook
      webhookConfigs:
        - url: 'http://dingtalk-webhook-adapter:8060/dingtalk/webhook/send'
          sendResolved: true