跨云部署K8s - kubeadm 安装

环境信息

机器三台(ip 为虚拟ip，真实ip为公网ip)

部署

• 设置hostaname（所有机器）

# 在 master 上设置 hostname
hostnamectl set-hostname k8s-master1
# 在 node1 上设置 hostname
hostnamectl set-hostname k8s-node1
# 在 node2 上设置 hostname
hostnamectl set-hostname k8s-node2
# 在每台机器上添加 host 解析 使用公网ip
192.168.1.2 k8s-master1
192.168.1.3 k8s-node1
192.168.1.4 k8s-node2

• 配置yum 源并安装docker (所有机器)

yum install -y yum-utils
yum-config-manager \
--add-repo \
http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
yum install -y docker-ce-20.10.7 docker-ce-cli-20.10.7  containerd.io-1.4.6

cat > /etc/docker/daemon.json <<EOF
{
  "registry-mirrors": ["https://lb17zmlc.mirror.aliyuncs.com"],
  "exec-opts": ["native.cgroupdriver=cgroupfs"],
  "log-driver": "json-file",
  "log-opts": {
    "max-size": "100m"
  },
  "storage-driver": "overlay2"
}
EOF

systemctl enable docker
systemctl start docker

• 环境初始化(所有机器)

# 将 SELinux 设置为 permissive 模式（相当于将其禁用）
setenforce 0
sed -i 's/^SELINUX=enforcing$/SELINUX=permissive/' /etc/selinux/config

#关闭swap
swapoff -a  
sed -ri 's/.*swap.*/#&/' /etc/fstab

#允许 iptables 检查桥接流量
cat <<EOF | sudo tee /etc/modules-load.d/k8s.conf
br_netfilter
EOF

cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
EOF
sysctl --system

echo 'net.ipv4.ip_forward=1 ' >>/etc/sysctl.conf
echo 'net.ipv4.tcp_tw_recycle=1 ' >>/etc/sysctl.conf
echo 'net.ipv4.tcp_tw_reuse=1 ' >>/etc/sysctl.conf
sysctl -p

• 下载镜像(所有机器)

tee ./images.sh <<-'EOF'
#!/bin/bash
images=(
kube-apiserver:v1.20.9
kube-proxy:v1.20.9
kube-controller-manager:v1.20.9
kube-scheduler:v1.20.9
coredns:1.7.0
etcd:3.4.13-0
pause:3.2
)
for imageName in ${images[@]} ; do
docker pull registry.cn-hangzhou.aliyuncs.com/lfy_k8s_images/$imageName
done
EOF

chmod +x ./images.sh && ./images.sh

• 安装 kubelet、kubeadm、kubectl(所有机器)

cat <<EOF | sudo tee /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=http://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=http://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg
   http://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
exclude=kubelet kubeadm kubectl
EOF


yum install -y kubelet-1.20.9 kubeadm-1.20.9 kubectl-1.20.9 --disableexcludes=kubernetes

systemctl enable --now kubelet

• 建立虚拟机网卡（所有机器这个很重要一定要做）

这个重启失败的话可以看下报错，有些云厂商的基础镜像有问题，就比如我遇到的一个重启时候失败，报没有eth1、2、3的网卡，到网卡目录 /etc/sysconfig/network-scripts/ 里查看发现多了很多个多余的网卡配置，直接删掉就好了。

# 替换你的公网IP进去
cat > /etc/sysconfig/network-scripts/ifcfg-eth0:1 <<EOF
BOOTPROTO=static
DEVICE=eth0:1
IPADDR=你的公网IP
PREFIX=32
TYPE=Ethernet
USERCTL=no
ONBOOT=yes
EOF
# 重启网卡
systemctl restart network
# 查看eth0网卡上是否有多一个公网ip
ip a

修改kubelet的参数（所有机器）

# 此文件安装kubeadm后就存在了
vim /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf

# 注意，这步很重要，如果不做，节点仍然会使用内网IP注册进集群
# 在末尾添加参数 --node-ip=公网IP
ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS --node-ip=xx.xx.xx.xx

添加配置文件（master）

# step1 添加配置文件，注意替换下面的IP
cat > kubeadm-config.yaml <<EOF
apiVersion: kubeadm.k8s.io/v1beta2
kind: ClusterConfiguration
kubernetesVersion: v1.20.9
apiServer:
  certSANs:    #填写所有kube-apiserver节点的hostname、IP、VIP
  - k8s-master1    #请替换为hostname
  - 192.168.1.2   #请替换为公网IP
  - xx.xx.xx.xx  #请替换为私网IP
  - 10.96.0.1   #不要替换，此IP是API的集群地址，部分服务会用到
controlPlaneEndpoint: 192.168.1.2:6443 #替换为master的公网IP
imageRepository: registry.cn-hangzhou.aliyuncs.com/lfy_k8s_images
networking:
  podSubnet: 10.244.0.0/16
  serviceSubnet: 10.96.0.0/12
--- 
apiVersion: kubeproxy-config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
featureGates:
  SupportIPVSProxyMode: true
mode: ipvs
EOF

# step2 如果是1核心或者1G内存的请在末尾添加参数（--ignore-preflight-errors=all），否则会初始化失败
# 同时注意，此步骤成功后，会打印重要信息
kubeadm init --config=kubeadm-config.yaml --ignore-preflight-errors=all

注意这里会出现一个 重要信息 kubeadm join 192.168.1.2:6443 --token XXX --discovery-token-ca-cert-hash XXX 的相关信息这个要copy下来，后边节点加入到集群会需要，当然如果时间久了，token会失效的，可以使用该命令重新生成。 kubeadm token create --print-join-command

• master操作

# 信息1 上面初始化成功后，将会生成kubeconfig文件，用于请求api服务器，请执行下面操作
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

# 也可以执行以下命令，该步骤和集群部署无关，是一个方便我们执行命令的操作，方便我们使用kubectl 像linux命令一样可以 tab
yum install -y bash-completion
source /usr/share/bash-completion/bash_completion
source <(kubectl completion bash)
echo "source <(kubectl completion bash)" >> ~/.bashrc

• 修改api参数(master操作)

# 修改两个信息，添加--bind-address和修改--advertise-address
vim /etc/kubernetes/manifests/kube-apiserver.yaml

apiVersion: v1
kind: Pod
metadata:
  annotations:
    kubeadm.kubernetes.io/kube-apiserver.advertise-address.endpoint: 10.0.20.8:6443
  creationTimestamp: null
  labels:
    component: kube-apiserver
    tier: control-plane
  name: kube-apiserver
  namespace: kube-system
spec:
  containers:
  - command:
    - kube-apiserver
    - --advertise-address=xx.xx.xx.xx # 修改为公网IP
    - --bind-address=0.0.0.0  # 新增参数
    - --allow-privileged=true
    - --authorization-mode=Node,RBAC
    - --client-ca-file=/etc/kubernetes/pki/ca.crt
    - --enable-admission-plugins=NodeRestriction
    - --enable-bootstrap-token-auth=true
    - --etcd-cafile=/etc/kubernetes/pki/etcd/ca.crt
    - --etcd-certfile=/etc/kubernetes/pki/apiserver-etcd-client.crt
    - --etcd-keyfile=/etc/kubernetes/pki/apiserver-etcd-client.key
    - --etcd-servers=https://127.0.0.1:2379
    - --insecure-port=0
    - --kubelet-client-certificate=/etc/kubernetes/pki/apiserver-kubelet-client.crt
    - --kubelet-client-key=/etc/kubernetes/pki/apiserver-kubelet-client.key
    - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
    - --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.crt
    - --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client.key
    - --requestheader-allowed-names=front-proxy-client
    - --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt
    - --requestheader-extra-headers-prefix=X-Remote-Extra-
    - --requestheader-group-headers=X-Remote-Group
    - --requestheader-username-headers=X-Remote-User
    - --secure-port=6443
    - --service-account-issuer=https://kubernetes.default.svc.cluster.local
    - --service-account-key-file=/etc/kubernetes/pki/sa.pub
    - --service-account-signing-key-file=/etc/kubernetes/pki/sa.key
    - --service-cluster-ip-range=10.96.0.0/12
    - --tls-cert-file=/etc/kubernetes/pki/apiserver.crt
    - --tls-private-key-file=/etc/kubernetes/pki/apiserver.key
    - --feature-gates=RemoveSelfLink=false # 如果有打算使用 nfs 需要加上这个，不然nfs 创建不了 一直pending
    image: registry.cn-hangzhou.aliyuncs.com/lfy_k8s_images/kube-apiserver:v1.20.9
    imagePullPolicy: IfNotPresent
    livenessProbe:
      failureThreshold: 8
      httpGet:
        host: 10.0.20.8
        path: /livez
        port: 6443
        scheme: HTTPS
      initialDelaySeconds: 10
      periodSeconds: 10
      timeoutSeconds: 15
    name: kube-apiserver
    readinessProbe:
      failureThreshold: 3
      httpGet:
        host: 10.0.20.8
        path: /readyz
        port: 6443
        scheme: HTTPS
      periodSeconds: 1
      timeoutSeconds: 15
    resources:
      requests:
        cpu: 250m
    startupProbe:
      failureThreshold: 24
      httpGet:
        host: 10.0.20.8
        path: /livez
        port: 6443
        scheme: HTTPS
      initialDelaySeconds: 10
      periodSeconds: 10
      timeoutSeconds: 15
    volumeMounts:
    - mountPath: /etc/ssl/certs
      name: ca-certs
      readOnly: true
    - mountPath: /etc/pki
      name: etc-pki
      readOnly: true
    - mountPath: /etc/kubernetes/pki
      name: k8s-certs
      readOnly: true
  hostNetwork: true
  priorityClassName: system-node-critical
  volumes:
  - hostPath:
      path: /etc/ssl/certs
      type: DirectoryOrCreate
    name: ca-certs
  - hostPath:
      path: /etc/pki
      type: DirectoryOrCreate
    name: etc-pki
  - hostPath:
      path: /etc/kubernetes/pki
      type: DirectoryOrCreate
    name: k8s-certs
status: {
   }

配置文件路径：/etc/kubernetes/manifests/kube-apiserver.yaml

spec:
  containers:
  - command:
    - kube-apiserver
    - --advertise-address=192.168.210.20
    - --.......　　#省略多行内容
    - --feature-gates=RemoveSelfLink=false　　#添加此行

• 开始节点加入集群（节点操作）

# 在两台节点上执行kube init 后输出的加入节点的命令
kubeadm join 192.168.1.2:6443 --token XXX --discovery-token-ca-cert-hash XXX

• 部署 flannel 网络

---
apiVersion: policy/v1beta1
kind: PodSecurityPolicy
metadata:
  name: psp.flannel.unprivileged
  annotations:
    seccomp.security.alpha.kubernetes.io/allowedProfileNames: docker/default
    seccomp.security.alpha.kubernetes.io/defaultProfileName: docker/default
    apparmor.security.beta.kubernetes.io/allowedProfileNames: runtime/default
    apparmor.security.beta.kubernetes.io/defaultProfileName: runtime/default
spec:
  privileged: false
  volumes:
  - configMap
  - secret
  - emptyDir
  - hostPath
  allowedHostPaths:
  - pathPrefix: "/etc/cni/net.d"
  - pathPrefix: "/etc/kube-flannel"
  - pathPrefix: "/run/flannel"
  readOnlyRootFilesystem: false
  # Users and groups
  runAsUser:
    rule: RunAsAny
  supplementalGroups:
    rule: RunAsAny
  fsGroup:
    rule: RunAsAny
  # Privilege Escalation
  allowPrivilegeEscalation: false
  defaultAllowPrivilegeEscalation: false
  # Capabilities
  allowedCapabilities: ['NET_ADMIN', 'NET_RAW']
  defaultAddCapabilities: []
  requiredDropCapabilities: []
  # Host namespaces
  hostPID: false
  hostIPC: false
  hostNetwork: true
  hostPorts:
  - min: 0
    max: 65535
  # SELinux
  seLinux:
    # SELinux is unused in CaaSP
    rule: 'RunAsAny'
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: flannel
rules:
- apiGroups: ['extensions']
  resources: ['podsecuritypolicies']
  verbs: ['use']
  resourceNames: ['psp.flannel.unprivileged']
- apiGroups:
  - ""
  resources:
  - pods
  verbs:
  - get
- apiGroups:
  - ""
  resources:
  - nodes
  verbs:
  - list
  - watch
- apiGroups:
  - ""
  resources:
  - nodes/status
  verbs:
  - patch
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: flannel
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: flannel
subjects:
- kind: ServiceAccount
  name: flannel
  namespace: kube-system
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: flannel
  namespace: kube-system
---
kind: ConfigMap
apiVersion: v1
metadata:
  name: kube-flannel-cfg
  namespace: kube-system
  labels:
    tier: node
    app: flannel
data:
  cni-conf.json: |
    {
      "name": "cbr0",
      "cniVersion": "0.3.1",
      "plugins": [
        {
          "type": "flannel",
          "delegate": {
            "hairpinMode": true,
            "isDefaultGateway": true
          }
        },
        {
          "type": "portmap",
          "capabilities": {
            "portMappings": true
          }
        }
      ]
    }
  net-conf.json: |
    {
      "Network": "10.244.0.0/16",
      "Backend": {
        "Type": "vxlan"
      }
    }
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: kube-flannel-ds
  namespace: kube-system
  labels:
    tier: node
    app: flannel
spec:
  selector:
    matchLabels:
      app: flannel
  template:
    metadata:
      labels:
        tier: node
        app: flannel
    spec:
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
            - matchExpressions:
              - key: kubernetes.io/os
                operator: In
                values:
                - linux
      hostNetwork: true
      priorityClassName: system-node-critical
      tolerations:
      - operator: Exists
        effect: NoSchedule
      serviceAccountName: flannel
      initContainers:
      - name: install-cni
        image: quay.io/coreos/flannel:v0.14.0
        command:
        - cp
        args:
        - -f
        - /etc/kube-flannel/cni-conf.json
        - /etc/cni/net.d/10-flannel.conflist
        volumeMounts:
        - name: cni
          mountPath: /etc/cni/net.d
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
      containers:
      - name: kube-flannel
        image: quay.io/coreos/flannel:v0.14.0
        command:
        - /opt/bin/flanneld
        args:
        - --ip-masq
        - --public-ip=$(PUBLIC_IP) 
        - --iface=eth0             
        - --kube-subnet-mgr
        args:
        - --ip-masq
        - --kube-subnet-mgr
        resources:
          requests:
            cpu: "100m"
            memory: "50Mi"
          limits:
            cpu: "100m"
            memory: "50Mi"
        securityContext:
          privileged: false
          capabilities:
            add: ["NET_ADMIN", "NET_RAW"]
        env:
        - name: PUBLIC_IP     
          valueFrom:          
            fieldRef:          
              fieldPath: status.podIP
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        volumeMounts:
        - name: run
          mountPath: /run/flannel
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
      volumes:
      - name: run
        hostPath:
          path: /run/flannel
      - name: cni
        hostPath:
          path: /etc/cni/net.d
      - name: flannel-cfg
        configMap:
          name: kube-flannel-cfg

• 创建并查看状态

# 部署
kubectl apply -f flannel.yaml
#查看状态
kubectl get pods -A| grep flannel

• 修改node 信息，flannel 默认获取到的node ip 是内网ip，需要修改下 node annotations里的信息

# 修改node信息
kubectl edit node k8s-node2

metadata:
  annotations:
    flannel.alpha.coreos.com/backend-data: '{"VNI":1,"VtepMAC":"fa:9e:d9:2a:77:15"}'
    flannel.alpha.coreos.com/backend-type: vxlan
    flannel.alpha.coreos.com/kube-subnet-manager: "true"
    flannel.alpha.coreos.com/public-ip: 192.169.1.4  #默认为内网ip，修改为公网ip

• 检查下 proxy的mode 是否为 ipvs，如果不是的话，不同节点的pod 会ping 不通

kubectl -n kube-system get cm kube-proxy -o yaml|grep ipvs

# 如果不是的话，执行以下操作，是的话忽略。

kubectl -n kube-system edit cm kube-proxy

# 添加 
mode: "ipvs"

#结果如：
data:
  config.conf: |-
    apiVersion: kubeproxy.config.k8s.io/v1alpha1
    bindAddress: 0.0.0.0
    bindAddressHardFail: false
    mode: "ipvs"

• 测试

在两个node 上各自部署一个deployment、service，来测试网络是否通,

# 打标签，也可以直接选择node
kubectl label nodes k8s-node1 node=node1
kubectl label nodes k8s-node2 node=node2

[root@k8s-master1 test]# cat dp-node1.yaml 
apiVersion: apps/v1
kind: Deployment
metadata:
  name: nginx-node1
spec:
  selector:
    matchLabels:
      app: nginx-node1
  replicas: 1
  template:
    metadata:
      labels:
        app: nginx-node1
    spec:
      containers:
      - name: nginx
        image: nginx:alpine
        ports:
        - containerPort: 80
      nodeSelector:
        node: "node1" 

[root@k8s-master1 test]# cat svc-node1.yaml 
apiVersion: v1
kind: Service
metadata:
  name: nginx-service-node1
spec:
  selector:
    app: nginx-node1
  ports:
  - protocol: TCP
    port: 80
    targetPort: 80

[root@k8s-master1 test]# cat dp-node2.yaml 
apiVersion: apps/v1
kind: Deployment
metadata:
  name: nginx-node2
spec:
  selector:
    matchLabels:
      app: nginx-node2
  replicas: 1
  template:
    metadata:
      labels:
        app: nginx-node2
    spec:
      containers:
      - name: nginx
        image: nginx:alpine
        ports:
        - containerPort: 80
      nodeSelector:
        node: "node2"

[root@k8s-master1 test]# cat svc-node2.yaml 
apiVersion: v1
kind: Service
metadata:
  name: nginx-service-node2
spec:
  selector:
    app: nginx-node2
  ports:
  - protocol: TCP
    port: 80
    targetPort: 80


# 部署完成后，kubectl get svc        kubectl get pods -o wide |grep nginx 获取 service 和 pod的 ip 在pod里ping即可