Kubeasz 部署 K8s 生产方案（存储：Longhorn）

一、环境规划

1.1 节点规划

生产建议：Master 至少 3 节点保证高可用；Worker 根据业务负载规划；Longhorn 推荐每个 Worker 挂载一块独立数据盘（如 /dev/vdb），避免与系统盘争用 IO。

1.2 系统要求

二、部署前准备

2.1 基础环境初始化（所有节点）

# 关闭防火墙
systemctl stop firewalld && systemctl disable firewalld

# 关闭 Swap
swapoff -a
sed -i '/swap/d' /etc/fstab

# 关闭 SELinux
setenforce 0
sed -i 's/SELINUX=enforcing/SELINUX=disabled/' /etc/selinux/config

# 加载内核模块（Longhorn 依赖）
cat >> /etc/modules-load.d/longhorn.conf <<EOF
iscsi_tcp
dm_crypt
overlay
br_netfilter
EOF

# 加载内核模块
modprobe iscsi_tcp
modprobe dm_crypt
modprobe overlay
modprobe br_netfilter

# 内核参数优化
cat >> /etc/sysctl.d/99-k8s.conf <<EOF
net.bridge.bridge-nf-call-iptables  = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward                 = 1
net.ipv4.conf.all.forwarding        = 1
vm.max_map_count                    = 262144
fs.file-max                         = 1000000
fs.inotify.max_user_instances       = 8192
fs.inotify.max_user_watches         = 524288
EOF

sysctl --system

# 安装必要工具
apt update && apt install -y curl wget socat conntrack ipvsadm ipset jq

2.2 iSCSI 客户端安装（Longhorn 必需，所有 Worker 节点）

# Ubuntu
apt install -y open-iscsi nfs-common

# 启动 iscsid
systemctl enable iscsid
systemctl start iscsid

三、Kubeasz 部署 K8s

3.1 安装 Kubeasz

# 在部署机（可选独立部署机或 master-01）
export RELEASE=v3.6.5  # 检查最新版本: https://github.com/easz/kubeasz/releases

# 下载 kubeasz
curl -L https://github.com/easz/kubeasz/releases/download/${RELEASE}/ezdown -o /usr/local/bin/ezdown
chmod +x /usr/local/bin/ezdown

# 初始化目录
ezdown -D

# 下载离线镜像（可选，加速部署）
ezdown -X v1.28.2

3.2 配置集群

# 创建集群配置
cd /etc/kubeasz
ezctl new k8s-prod

3.3 编辑集群配置文件

hosts 文件 (/etc/kubeasz/clusters/k8s-prod/hosts)：

# 部署节点
[deploy]
192.168.1.11

# etcd 节点（生产建议独立部署或使用 master 节点）
[etcd]
192.168.1.11
192.168.1.12
192.168.1.13

# master 节点
[master]
192.168.1.11
192.168.1.12
192.168.1.13

# worker 节点
[node]
192.168.1.21
192.168.1.22
192.168.1.23

# 负载均衡（VIP）
[ex_lb]
192.168.1.10

# 配置变量
[all:vars]
# k8s 版本
CLUSTER_NAME="k8s-prod"
CONTAINER_RUNTIME="containerd"
K8S_VER="v1.28.2"

# 网络插件（推荐 calico）
CLUSTER_NETWORK="calico"
NETWORK_CALICO="calico-v3.26"

# Service CIDR
SERVICE_CIDR="10.68.0.0/16"
POOLID_IPV4="172.20.0.0/16"

# API Server VIP
MASTER_IP="192.168.1.10"
LB_DOMAIN="lb.k8s.local"

# 认证
CA_EXPIRY="87600h"
CERT_EXPIRY="8760h"

# 审计日志
AUDIT_ENABLED=true

config.yml (/etc/kubeasz/clusters/k8s-prod/config.yml)：

# 关键配置项
# 容器运行时
container_runtime:
  type: containerd
  version: "1.7.8"

# etcd 配置
etcd:
  data_dir: "/var/lib/etcd"
  snapshot_count: 10000
  heartbeat_interval: 250
  election_timeout: 5000

# kubelet 配置
kubelet:
  max_pods: 200
  pod_infra_container_image: "registry.k8s.io/pause:3.9"
  kube_reserved:
    cpu: "500m"
    memory: "512Mi"
  system_reserved:
    cpu: "500m"
    memory: "512Mi"
  eviction_hard:
    memory.available: "200Mi"
    nodefs.available: "10%"
    imagefs.available: "15%"

# 开启审计
audit:
  enabled: true
  log_path: "/var/log/kubernetes/audit.log"
  max_age: 30
  max_backups: 10
  max_size: 100

3.4 执行部署

# 一键部署（推荐分步执行以便排错）
cd /etc/kubeasz

# 步骤1：初始化环境
ezctl setup k8s-prod 01.prepare

# 步骤2：部署 etcd
ezctl setup k8s-prod 02.etcd

# 步骤3：部署容器运行时
ezctl setup k8s-prod 03.containerd

# 步骤4：部署 master 组件
ezctl setup k8s-prod 04.kube-master

# 步骤5：部署网络插件
ezctl setup k8s-prod 05.kube-node

# 步骤6：部署 worker 节点
ezctl setup k8s-prod 06.network

# 步骤7：部署核心服务
ezctl setup k8s-prod 07.cluster-addon

# 步骤8：部署负载均衡
ezctl setup k8s-prod 08.lb

# 步骤9：部署存储（Longhorn，见第四节）
# 手动执行

3.5 验证集群

kubectl get nodes
kubectl get cs
kubectl get pods -n kube-system

四、Longhorn 存储部署

4.1 准备数据盘（所有 Worker 节点）

# 查看数据盘（假设 /dev/vdb 为 Longhorn 专用数据盘）
lsblk

# 格式化数据盘
mkfs.ext4 -F /dev/vdb

# 挂载到 Longhorn 数据目录
mkdir -p /data/longhorn
mount /dev/vdb /data/longhorn

# 写入 fstab 持久化
echo '/dev/vdb /data/longhorn ext4 defaults 0 2' >> /etc/fstab

4.2 安装 Longhorn

# 创建命名空间
kubectl create namespace longhorn-system

# 方式一：使用 Helm 安装（推荐）
helm repo add longhorn https://charts.longhorn.io
helm repo update

helm install longhorn longhorn/longhorn \
  --namespace longhorn-system \
  --version 1.5.3 \
  --set defaultSettings.defaultDataPath=/data/longhorn \
  --set defaultSettings.storageOverProvisioningPercentage=200 \
  --set defaultSettings.storageMinimalAvailablePercentage=10 \
  --set defaultSettings.replicaAutoBalance=best-effort \
  --set defaultSettings.backupTarget="" \
  --set persistence.defaultClass=true \
  --set persistence.defaultClassReplicaCount=3 \
  --set longhornManager.resources.requests.cpu=100m \
  --set longhornManager.resources.requests.memory=128Mi \
  --set longhornDriver.resources.requests.cpu=100m \
  --set longhornDriver.resources.requests.memory=128Mi

4.3 Longhorn 生产优化配置

部署完成后，在 Longhorn UI 或 CRD 中调整以下设置：

# longhorn-setting.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: longhorn-default-setting
  namespace: longhorn-system
data:
  default-setting.yaml: |
    # 默认副本数
    default-replica-count: 3
    # 默认数据路径
    default-data-path: /data/longhorn
    # 存储超配比例（200%）
    storage-over-provisioning-percentage: 200
    # 最小可用空间比例（低于10%停止调度）
    storage-minimal-available-percentage: 10
    # 副本自动均衡
    replica-auto-balance: best-effort
    # 节点排水策略
    node-drain-policy: block-if-contains-last-replica
    # 自动删除 PVC 时删除卷
    deleting-confirmation-flag: true
    # 快照数量限制
    snapshot-count-max: 250
    # 备份目标（生产建议配置 NFS/S3）
    # backup-target: nfs://192.168.1.100:/backup/longhorn
    # 自动 salvage
    auto-salvage: true
    # 自动删除无效副本
    auto-delete-when-volume-is-purged: true
    # 调度策略
    replica-soft-anti-affinity: false
    replica-zone-soft-anti-affinity: true
    # 引擎并发
    concurrent-automatic-engine-upgrade-per-node-limit: 3

4.4 创建 StorageClass

# longhorn-storageclass.yaml
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: longhorn
  annotations:
    storageclass.kubernetes.io/is-default-class: "true"
provisioner: driver.longhorn.io
allowVolumeExpansion: true
reclaimPolicy: Retain          # 生产建议 Retain，防止误删
volumeBindingMode: Immediate
parameters:
  numberOfReplicas: "3"
  staleReplicaTimeout: "30"
  fromBackup: ""
  fsType: "ext4"
  dataLocality: "disabled"
  # 数据本地性优化（可选，同节点优先）
  # dataLocality: "best-effort"
  unmapMarkSnapChainRemoved: "ignored"
---
# 高性能场景 StorageClass
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: longhorn-fast
provisioner: driver.longhorn.io
allowVolumeExpansion: true
reclaimPolicy: Retain
volumeBindingMode: Immediate
parameters:
  numberOfReplicas: "2"
  staleReplicaTimeout: "30"
  fsType: "ext4"
  dataLocality: "best-effort"

kubectl apply -f longhorn-storageclass.yaml

4.5 配置备份目标（NFS 示例）

# 在 NFS 服务器上
# mkdir -p /backup/longhorn && chmod 777 /backup/longhorn

# Longhorn UI 中设置：
# Settings > General > Backup Target
# nfs://192.168.1.100:/backup/longhorn

五、验证与测试

5.1 验证 Longhorn

# 检查 Longhorn 组件
kubectl get pods -n longhorn-system

# 检查节点磁盘状态
kubectl get nodes -o json | jq '.items[].metadata.name'

# 访问 Longhorn UI（通过 kubectl proxy 或 Ingress）
kubectl port-forward -n longhorn-system svc/longhorn-frontend 8080:80
# 浏览器访问 http://localhost:8080

5.2 创建 PVC 测试

# test-pvc.yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: test-longhorn-pvc
spec:
  accessModes:
    - ReadWriteOnce
  storageClassName: longhorn
  resources:
    requests:
      storage: 5Gi
---
apiVersion: v1
kind: Pod
metadata:
  name: test-pod
spec:
  containers:
    - name: test
      image: busybox
      command: ["sh", "-c", "echo 'Longhorn works!' > /data/test.txt && sleep 3600"]
      volumeMounts:
        - name: data
          mountPath: /data
  volumes:
    - name: data
      persistentVolumeClaim:
        claimName: test-longhorn-pvc

kubectl apply -f test-pvc.yaml
kubectl get pvc test-longhorn-pvc
kubectl exec test-pod -- cat /data/test.txt

六、生产加固建议

监控示例（Prometheus 告警规则）

# longhorn-alert-rules.yaml
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: longhorn-alerts
  namespace: longhorn-system
spec:
  groups:
    - name: longhorn.rules
      rules:
        - alert: LonghornNodeDiskUsageHigh
          expr: longhorn_node_storage_usage_bytes / longhorn_node_storage_capacity_bytes > 0.8
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "Longhorn 节点磁盘使用率超过 80%"
        - alert: LonghornVolumeDegraded
          expr: longhorn_volume_robustness != "healthy"
          for: 5m
          labels:
            severity: critical
          annotations:
            summary: "Longhorn 卷状态异常: {
   { $labels.name }}"