基于Rocky系统kubeadm部署k8s1.28.0主从集群(cri-dockerd)

1. kubernetes集群规划

主机IP	主机名	主机配置	角色
192.168.100.3	master1	2C/4G	管理节点
192.168.100.4	node1	2C/4G	工作节点
192.168.100.5	node2	2C/4G	工作节点

2. 集群前期环境准备

（1）配置网卡

设置手动IP地址

[root@localhost ~]# vi /etc/NetworkManager/system-connections/ens32.nmconnection
[connection]
id=ens32
uuid=2cff8ce1-b35d-3f0b-83cf-1c04ccee2a43
type=ethernet
autoconnect-priority=-999
interface-name=ens32
timestamp=1733093521

[ethernet]

[ipv4]
method=manual
address1=192.168.100.3/24,192.168.100.2
dns=223.5.5.5;8.8.8.8

[ipv6]
addr-gen-mode=eui64
method=auto

[proxy]

重启网卡服务

[root@localhost ~]# systemctl restart NetworkManager

访问测试

[root@localhost ~]# ping -c 4 bing.com
PING bing.com (13.107.21.200) 56(84) bytes of data.
64 bytes from 13.107.21.200 (13.107.21.200): icmp_seq=1 ttl=128 time=68.1 ms
64 bytes from 13.107.21.200 (13.107.21.200): icmp_seq=2 ttl=128 time=68.6 ms
64 bytes from 13.107.21.200 (13.107.21.200): icmp_seq=3 ttl=128 time=68.3 ms
64 bytes from 13.107.21.200 (13.107.21.200): icmp_seq=4 ttl=128 time=69.5 ms

--- bing.com ping statistics ---
4 packets transmitted, 4 received, 0% packet loss, time 3005ms
rtt min/avg/max/mdev = 68.105/68.612/69.538/0.558 ms

（2）配置root远程登录

编辑sshd配置文件

[root@localhost ~]# vi /etc/ssh/sshd_config
...
### 修改第40行
#PermitRootLogin prohibit-password
### 修改为以下内容
PermitRootLogin yes

重启sshd服务

[root@localhost ~]# systemctl restart sshd

（3）初始化脚本

[root@localhost ~]# vi init.sh
#!/bin/bash
echo "——>>> 关闭防火墙与SELinux <<<——"
sleep 3
systemctl disable firewalld --now &> /dev/null
setenforce 0
sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config

echo "——>>> 配置阿里仓库 <<<——"
sleep 3
sed -e 's|^#mirrorlist=|mirrorlist=|g' \
    -e 's|^baseurl=https://mirrors.aliyun.com/rockylinux|#baseurl=http://dl.rockylinux.org/$contentdir|g' \
    -i.bak \
    /etc/yum.repos.d/rocky*.repo

echo "——>>> 设置时区并同步时间 <<<——"
sleep 3
timedatectl set-timezone Asia/Shanghai
yum install -y chrony &> /dev/null
systemctl enable chronyd --now &> /dev/null
sed -i '/^server/s/^/# /' /etc/chrony.conf
sed -i '/^# server 3.centos.pool.ntp.org iburst/a\server ntp1.aliyun.com iburst\nserver ntp2.aliyun.com iburst\nserver ntp3.aliyun.com iburst' /etc/chrony.conf
systemctl restart chronyd &> /dev/null
chronyc sources &> /dev/null

echo "——>>> 设置系统最大打开文件数 <<<——"
sleep 3
if ! grep "* soft nofile 65535" /etc/security/limits.conf &>/dev/null; then
cat >> /etc/security/limits.conf << EOF
* soft nofile 65535   # 软限制
* hard nofile 65535   # 硬限制
EOF
fi

echo "——>>> 系统内核优化 <<<——"
sleep 3
cat >> /etc/sysctl.conf << EOF
net.ipv4.tcp_syncookies = 1             # 防范SYN洪水攻击，0为关闭
net.ipv4.tcp_max_tw_buckets = 20480     # 此项参数可以控制TIME_WAIT套接字的最大数量，避免Squid服务器被大量的TIME_WAIT套接字拖死
net.ipv4.tcp_max_syn_backlog = 20480    # 表示SYN队列的长度，默认为1024，加大队列长度为8192，可以容纳更多等待连接的网络连接数
net.core.netdev_max_backlog = 262144    # 每个网络接口 接受数据包的速率比内核处理这些包的速率快时，允许发送到队列的数据包的最大数目
net.ipv4.tcp_fin_timeout = 20           # FIN-WAIT-2状态的超时时间，避免内核崩溃
EOF

echo "——>>> 减少SWAP使用 <<<——"
sleep 3
echo "0" > /proc/sys/vm/swappiness

echo "——>>> 安装系统性能分析工具及其他 <<<——"
sleep 3
yum install -y vim net-tools lsof wget lrzsz &> /dev/null

执行初始化脚本

[root@localhost ~]# sh init.sh 
——>>> 关闭防火墙与SELinux <<<——
——>>> 配置阿里仓库 <<<——
——>>> 设置时区并同步时间 <<<——
——>>> 设置系统最大打开文件数 <<<——
——>>> 系统内核优化 <<<——
——>>> 减少SWAP使用 <<<——
——>>> 安装系统性能分析工具及其他 <<<——

（4）配置主机名

[root@localhost ~]# hostnamectl set-hostname k8s-master
[root@localhost ~]# bash
[root@k8s-master ~]# 

（5）配置主机映射

cat >> /etc/hosts << EOF
192.168.100.3 k8s-master
192.168.100.4 k8s-node1
192.168.100.5 k8s-node2
EOF

3. Docker环境安装

（1）开启bridge网桥过滤

bridge(桥接) 是 Linux 系统中的一种虚拟网络设备，它充当一个虚拟的交换机，为集群内的容器提供网络通信功能，容器就可以通过这个 bridge 与其他容器或外部网络通信了。

[root@master ~]# cat > /etc/sysctl.d/k8s.conf <<EOF
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
EOF

由于开启bridge功能，需要加载br_netfilter模块来允许在bridge设备上的数据包经过iptables防火墙处理

[root@master ~]# modprobe br_netfilter && lsmod | grep br_netfilter

加载配置文件

[root@master ~]# sysctl -p /etc/sysctl.d/k8s.conf

（2）安装Docker

[root@master ~]# curl -o /etc/yum.repos.d/docker-ce.repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
[root@master ~]# yum install -y yum-utils device-mapper-persistent-data lvm2
[root@master ~]# yum list docker-ce --showduplicates | sort -r
[root@k8s-master ~]# yum list docker-ce --showduplicates | sort -r
Last metadata expiration check: 0:42:47 ago on Mon 02 Dec 2024 12:26:17 PM CST.
Installed Packages
docker-ce.x86_64               3:27.3.1-1.el9                  docker-ce-stable 
docker-ce.x86_64               3:27.3.1-1.el9                  @docker-ce-stable
docker-ce.x86_64               3:27.3.0-1.el9                  docker-ce-stable 
docker-ce.x86_64               3:27.2.1-1.el9                  docker-ce-stable 
docker-ce.x86_64               3:27.2.0-1.el9                  docker-ce-stable 
docker-ce.x86_64               3:27.1.2-1.el9                  docker-ce-stable 
docker-ce.x86_64               3:27.1.1-1.el9                  docker-ce-stable 
docker-ce.x86_64               3:27.1.0-1.el9                  docker-ce-stable 
docker-ce.x86_64               3:27.0.3-1.el9                  docker-ce-stable 
docker-ce.x86_64               3:27.0.2-1.el9                  docker-ce-stable 
docker-ce.x86_64               3:27.0.1-1.el9                  docker-ce-stable 
docker-ce.x86_64               3:26.1.4-1.el9                  docker-ce-stable 
docker-ce.x86_64               3:26.1.3-1.el9                  docker-ce-stable 
...
[root@master ~]# yum -y install docker-ce
[root@master ~]# systemctl enable docker --now
[root@k8s-master ~]# docker version
Client: Docker Engine - Community
 Version:           27.3.1
 API version:       1.47
 Go version:        go1.22.7
 Git commit:        ce12230
 Built:             Fri Sep 20 11:42:48 2024
 OS/Arch:           linux/amd64
 Context:           default

Server: Docker Engine - Community
 Engine:
  Version:          27.3.1
  API version:      1.47 (minimum version 1.24)
  Go version:       go1.22.7
  Git commit:       41ca978
  Built:            Fri Sep 20 11:41:09 2024
  OS/Arch:          linux/amd64
  Experimental:     false
 containerd:
  Version:          1.7.24
  GitCommit:        88bf19b2105c8b17560993bee28a01ddc2f97182
 runc:
  Version:          1.2.2
  GitCommit:        v1.2.2-0-g7cb3632
 docker-init:
  Version:          0.19.0
  GitCommit:        de40ad0

（3）配置镜像加速器和Cgroup驱动

sudo mkdir -p /etc/docker
sudo tee /etc/docker/daemon.json << 'EOF'
{
  "insecure-registries": ["0.0.0.0/0"],
  "registry-mirrors": [
    "https://docker.aityp.com",
    "https://docker.m.daocloud.io",
    "https://reg-mirror.qiniu.com",
    "https://k8s.m.daocloud.io",
    "https://elastic.m.daocloud.io",
    "https://gcr.m.daocloud.io",
    "https://ghcr.m.daocloud.io",
    "https://k8s-gcr.m.daocloud.io",
    "https://mcr.m.daocloud.io",
    "https://nvcr.m.daocloud.io",
    "https://quay.m.daocloud.io",
    "https://jujucharms.m.daocloud.io",
    "https://rocks-canonical.m.daocloud.io",
    "https://d3p1s1ji.mirror.aliyuncs.com"
  ],
  "exec-opts": [
    "native.cgroupdriver=systemd"
  ],
  "max-concurrent-downloads": 10,
  "max-concurrent-uploads": 5,
  "log-opts": {
    "max-size": "300m",
    "max-file": "2"
  },
  "live-restore": true,
  "log-level": "debug"
}
EOF

sudo systemctl daemon-reload
sudo systemctl restart docker

（4）安装cri-dockerd

Docker与Kubernetes通信的中间程序

K8s的1.24版本以后移除了docker-shim，而Docker Engine默认又不支持CRI规范，因而二者将无法直接完成整合，为此，Mirantis和Docker联合创建了cri-dockerd项目，用于为Docker Engine提供一个能够支持到CRI规范的垫片，从而能够让Kubernetes基于CRI控制Docker ，所以想在K8s的1.24版本及以后的版本中使用docker，需要安装cri-dockerd，然后K8s集群通过cri-dockerd联系到docker（注意每个节点都要安装）

项目地址：https://github.com/Mirantis/cri-dockerd

[root@master ~]# wget ftp://ftp.icm.edu.pl/vol/rzm7/linux-centos-vault/7.6.1810/cr/x86_64/Packages/libcgroup-0.41-21.el7.x86_64.rpm
[root@master ~]# wget https://github.com/Mirantis/cri-dockerd/releases/download/v0.3.2/cri-dockerd-0.3.2-3.el7.x86_64.rpm
[root@master ~]# rpm -ivh libcgroup-0.41-21.el7.x86_64.rpm
[root@master ~]# rpm -ivh cri-dockerd-0.3.2-3.el7.x86_64.rpm

（5）编辑cri-docker.service文件

[root@master ~]# vim /usr/lib/systemd/system/cri-docker.service 
...
[Service]
Type=notify
ExecStart=/usr/bin/cri-dockerd --container-runtime-endpoint fd:// --pod-infra-container-image=registry.aliyuncs.com/google_containers/pause:3.9 --network-plugin=cni --cni-bin-dir=/opt/cni/bin --cni-cache-dir=/var/lib/cni/cache --cni-conf-dir=/etc/cni/net.d
...

重启服务

systemctl daemon-reload
systemctl restart cri-docker.service
systemctl enable cri-docker.service

4. 配置阿里云YUM源

（1）添加k8s源

[root@master ~]# cat > /etc/yum.repos.d/kubernetes.repo << EOF
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
enabled=1
gpgcheck=1
repo_gpgcheck=1
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF

（2）安装k8s工具

[root@master ~]# yum install -y kubelet-1.28.0 kubeadm-1.28.0 kubectl-1.28.0

• kubeadm：用于初始化集群，并配置集群所需的组件并生成对应的安全证书和令牌；
• kubelet：负责与 Master 节点通信，并根据 Master 节点的调度决策来创建、更新和删除 Pod，同时维护 Node 节点上的容器状态；
• kubectl：用于管理k8集群的一个命令行工具；

设置kubelet开启自启，不需要直接开启初始化过程会启动

[root@master ~]# systemctl enable kubelet

（3）关闭swap分区

临时关闭

[root@master ~]# swapoff -a

永久关闭

[root@master ~]# sed -ri 's/.*swap.*/#&/' /etc/fstab

（4）初始化集群

命令行方式

kubeadm init \
  --apiserver-advertise-address=192.168.100.3 \
  --image-repository registry.aliyuncs.com/google_containers \
  --kubernetes-version v1.28.0 \
  --service-cidr=10.96.0.0/12 \
  --pod-network-cidr=10.244.0.0/16 \
  --cri-socket=unix:///var/run/cri-dockerd.sock

yaml文件方式

[root@master ~]# kubeadm config print init-defaults > kubeadm-config.yaml
[root@master ~]# cat kubeadm-config.yaml
apiVersion: kubeadm.k8s.io/v1beta3
bootstrapTokens:
- groups:
  - system:bootstrappers:kubeadm:default-node-token
  token: abcdef.0123456789abcdef
  ttl: 24h0m0s
  usages:
  - signing
  - authentication
kind: InitConfiguration
localAPIEndpoint:
  advertiseAddress: 192.168.100.3		### 本地IP地址
  bindPort: 6443
nodeRegistration:
  criSocket: unix:///var/run/cri-dockerd.sock		### 修改容器运行时
  imagePullPolicy: IfNotPresent
  name: master		### 修改主机名
  taints: null
---
apiServer:
  timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta3
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controllerManager: {}
dns: {}
etcd:
  local:
    dataDir: /var/lib/etcd
imageRepository: registry.cn-hangzhou.aliyuncs.com/google_containers	### 修改仓库地址
kind: ClusterConfiguration
kubernetesVersion: 1.28.0
networking:
  dnsDomain: cluster.local
  serviceSubnet: 10.96.0.0/12
  podSubnet: 10.244.0.0/16  ### 添加 podSubnet 配置
scheduler: {}

初始化集群

[root@master1 ~]# kubeadm init --config kubeadm-config.yaml --upload-certs

#选项说明：
--upload-certs   //初始化过程将生成证书，并将其上传到etcd存储中，否则节点无法加入集群

初始化失败使用以下命令重置

[root@master1 ~]# kubeadm reset --cri-socket /var/run/cri-dockerd.sock

（5）配置认证文件

mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

export KUBECONFIG=/etc/kubernetes/admin.conf

使用kubectl工具查看节点状态

[root@master ~]# kubectl get nodes
NAME        STATUS   ROLES      AGE  VERSION
k8s-master   NotReady  control-plane    20s  v1.28.0

注：由于网络插件还没有部署，节点会处于"NotReady"状态

（6）将node节点加入集群

kubeadm join 192.168.100.3:6443 --token rtrcwi.cyn9p08jznxgmciq \
	--discovery-token-ca-cert-hash sha256:93f37dba1f78e73ec5e343baeda001b481bc120efaefa962887991ac0da42216 \
	--cri-socket=unix:///var/run/cri-dockerd.sock

（7）去除污点

[root@master ~]# kubectl taint nodes master node-role.kubernetes.io/control-plane-

5. 配置Calico网络组件

（1）下载配置文件

wget https://docs.projectcalico.org/manifests/tigera-operator.yaml
wget https://docs.projectcalico.org/manifests/custom-resources.yaml

（2）编辑配置文件

[root@master ~]# vim custom-resources.yaml 
# This section includes base Calico installation configuration.
# For more information, see: https://projectcalico.docs.tigera.io/master/reference/installation/api#operator.tigera.io/v1.Installation
apiVersion: operator.tigera.io/v1
kind: Installation
metadata:
  name: default
spec:
  # Configures Calico networking.
  calicoNetwork:
    # Note: The ipPools section cannot be modified post-install.
    ipPools:
    - blockSize: 26
      cidr: 10.244.0.0/16		### 修改为--pod-network-cidr地址
      encapsulation: VXLANCrossSubnet
      natOutgoing: Enabled
      nodeSelector: all()

---

# This section configures the Calico API server.
# For more information, see: https://projectcalico.docs.tigera.io/master/reference/installation/api#operator.tigera.io/v1.APIServer
apiVersion: operator.tigera.io/v1
kind: APIServer 
metadata: 
  name: default 
spec: {}

（3）部署Calico网络

注意使用apply会有以下报错

[root@master ~]# kubectl apply -f tigera-operator.yaml 
...
The CustomResourceDefinition "installations.operator.tigera.io" is invalid: metadata.annotations: 
Too long: must have at most 262144 bytes

意思是 annotation 长度过长了，原因是 apply 和 create 的处理不同，这点 GitHub 上也有人在吐槽，这是 GitHub上的吐槽地址

使用以下命令运行部署

[root@master ~]# kubectl create -f tigera-operator.yaml
[root@master ~]# kubectl create -f custom-resources.yaml

查看集群Pod运行状态

[root@master ~]# kubectl get nodes 
NAME         STATUS   ROLES           AGE     VERSION
k8s-master   Ready    control-plane   4m11s   v1.28.0
k8s-node1    Ready    <none>          3m33s   v1.28.0

[root@k8s-master ~]# kubectl get nodes -owide
NAME         STATUS   ROLES           AGE    VERSION   INTERNAL-IP      EXTERNAL-IP   OS-IMAGE                      KERNEL-VERSION                CONTAINER-RUNTIME
k8s-master   Ready    control-plane   108s   v1.28.0   192.168.100.3   <none>        Rocky Linux 9.3 (Blue Onyx)   5.14.0-362.8.1.el9_3.x86_64   docker://27.3.1
k8s-node1    Ready    <none>          70s    v1.28.0   192.168.100.4   <none>        Rocky Linux 9.3 (Blue Onyx)   5.14.0-362.8.1.el9_3.x86_64   docker://27.3.1

[root@master ~]# kubectl get pod -A
NAMESPACE          NAME                                       READY   STATUS    RESTARTS   AGE
calico-apiserver   calico-apiserver-7b9c9fb95d-7w666          1/1     Running   0          12s
calico-apiserver   calico-apiserver-7b9c9fb95d-hfjjg          1/1     Running   0          12s
calico-system      calico-kube-controllers-685f7c9b88-rpmvl   1/1     Running   0          29s
calico-system      calico-node-d52d7                          1/1     Running   0          30s
calico-system      calico-node-t6qpr                          1/1     Running   0          30s
calico-system      calico-typha-589b7cd4b4-hvq7q              1/1     Running   0          30s
calico-system      csi-node-driver-crmm9                      2/2     Running   0          29s
calico-system      csi-node-driver-kjnlc                      2/2     Running   0          30s
kube-system        coredns-66f779496c-6vpq8                   1/1     Running   0          17m
kube-system        coredns-66f779496c-khqb4                   1/1     Running   0          17m
kube-system        etcd-master                                1/1     Running   0          18m
kube-system        kube-apiserver-master                      1/1     Running   0          18m
kube-system        kube-controller-manager-master             1/1     Running   0          18m
kube-system        kube-proxy-9ll4p                           1/1     Running   0          16m
kube-system        kube-proxy-wpgnh                           1/1     Running   0          18m
kube-system        kube-scheduler-master                      1/1     Running   0          18m
tigera-operator    tigera-operator-8547bd6cc6-vmjvq           1/1     Running   0          39s

（4）测试部署

[root@master ~]# kubectl create deployment --image nginx:1.20.2 nginx
[root@master ~]# kubectl get pod
NAME                     READY   STATUS    RESTARTS   AGE
nginx-6f974c44c8-xzvwg   1/1     Running   0          65s
[root@master ~]# kubectl describe pod nginx-6f974c44c8-xzvwg
...
Events:
  Type    Reason     Age   From               Message
  ----    ------     ----  ----               -------
  Normal  Scheduled  64s   default-scheduler  Successfully assigned default/nginx-6f974c44c8-xzvwg to node
  Normal  Pulling    63s   kubelet            Pulling image "nginx:1.20.2"
  Normal  Pulled     3s    kubelet            Successfully pulled image "nginx:1.20.2" in 1m0.406s (1m0.406s including waiting)
  Normal  Created    2s    kubelet            Created container nginx
  Normal  Started    2s    kubelet            Started container nginx