kubeadm安装高可用kubernetes1.29.14

架构说明

Kubernetes 主要由以下几个核心组件组成：
etcd：保存了整个集群的状态；
kube-apiserver：提供了资源操作的唯一入口，并提供认证、授权、访问控制、API 注册和发现等机制；
kube-controller-manager：负责维护集群的状态，比如故障检测、自动扩展、滚动更新等；
kube-scheduler：负责资源的调度，按照预定的调度策略将 Pod 调度到相应的机器上；
kubelet：负责维持容器的生命周期，同时也负责 Volume（CVI）和网络（CNI）的管理；
Container runtime：负责镜像管理以及 Pod 和容器的真正运行（CRI），默认的容器运行时为 Docker；
kube-proxy 负责为 Service 提供 cluster 内部的服务发现和负载均衡

系统环境说明(生产环境按需修改)

安装Nginx+keepalived高可用架构(HA节点部署)

1、下载nginx源码包
wget https://nginx.org/download/nginx-1.22.1.tar.gzwget https://nginx.org/download/nginx-1.22.1.tar.gz
2、安装依赖软件包
yum -y install gcc gcc-c++ pcre pcre-devel zlib zlib-devel openssl openssl-devel
3、解压缩nginx源码包
tar -zxvf nginx-1.22.1.tar.gz && cd tar -zxvf nginx-1.22.1
4、编译
./configure --prefix=/data/nginx --sbin-path=/data/nginx/sbin/nginx --conf-path=/data/nginx/conf/nginx.conf --error-log-path=/data/nginx/logs/error.log --http-log-path=/data/nginx/logs/access.log --pid-path=/data/nginx/nginx.pid --lock-path=/data/nginx/nginx.lock --http-client-body-temp-path=/var/cache/nginx/client_temp --http-proxy-temp-path=/var/cache/nginx/proxy_temp --http-fastcgi-temp-path=/var/cache/nginx/fastcgi_temp --http-uwsgi-temp-path=/var/cache/nginx/uwsgi_temp --http-scgi-temp-path=/var/cache/nginx/scgi_temp --user=nginx --group=nginx --with-file-aio --with-threads --with-http_addition_module --with-http_auth_request_module --with-http_dav_module --with-http_flv_module --with-http_gunzip_module --with-http_gzip_static_module --with-http_mp4_module --with-http_random_index_module --with-http_realip_module --with-http_secure_link_module --with-http_slice_module --with-http_ssl_module --with-http_stub_status_module --with-http_sub_module --with-http_v2_module --with-mail --with-mail_ssl_module --with-stream --with-stream_realip_module --with-stream_ssl_module --with-stream_ssl_preread_module
5、安装
make && make install
6、创建所需目录
mkdir /var/cache/nginx/ -p
7、创建nginx用户
useradd -M -s /sbin/nologin nginx
8、系统句柄调优
echo """
* soft nofile 1000000
* hard nofile 1000000
* soft nproc unlimited
* hard nproc unlimited
""" >> /etc/security/limits.conf
 ulimit -SHn 65535
9、启动nginx看有没有报错
/data/nginx/sbin/nginx -t
/data/nginx/sbin/nginx

10、安装keepalived
yum install keepalived -y
#修改配置
cd /etc/keepalived/
cp keepalived.conf.sample keepalived.conf
#keepalived主配置文件
[root@ha-1 ~]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived

global_defs {
     notification_email {
     acassen@firewall.loc
     failover@firewall.loc
     sysadmin@firewall.loc
   }
}

vrrp_script chk_apiserver {
    script "/etc/keepalived/check_apiserver.sh"
    interval 5 
    weight -5
    fall 2
    rise 1
}

vrrp_instance VI_1 {
    state MASTER
    interface ens33
    virtual_router_id 51
    priority 100
    advert_int 1

    authentication {
        auth_type PASS
        auth_pass 1111
    }

    virtual_ipaddress {
        10.1.20.52/24 dev ens33 label ens33:1
    }

    track_script {
      chk_apiserver
    }
}

#keepalived备配置文件
[root@ha-2 ~]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived

global_defs {
     notification_email {
     acassen@firewall.loc
     failover@firewall.loc
     sysadmin@firewall.loc
   }
}

vrrp_script chk_apiserver {
    script "/etc/keepalived/check_apiserver.sh"
    interval 5 
    weight -5
    fall 2
    rise 1
}

vrrp_instance VI_1 {
    state BACKUP
    interface ens33
    virtual_router_id 51
    priority 80
    advert_int 1

    authentication {
        auth_type PASS
        auth_pass 1111
    }

    virtual_ipaddress {
        10.1.20.52 dev ens33 label ens33:1
    }

    track_script {
      chk_apiserver
    }
}

#启动keepalived
systemctl enable --now keepalived

#nginx配置文件-配置请同步到ha-2节点
[root@ha-1 ~]# cat /data/nginx/conf/nginx.conf
worker_processes  auto;

events {
    worker_connections  1024;
}

stream {
    upstream kubernetes {
        server 10.1.20.200:6443 max_fails=3 fail_timeout=30s;
        server 10.1.20.201:6443 max_fails=3 fail_timeout=30s;
        server 10.1.20.202:6443 max_fails=3 fail_timeout=30s;
    }

    server {
        listen 8443;
        proxy_connect_timeout 1s;
        proxy_pass kubernetes;
    }
}

#重新加载nginx
[root@ha-1 ~]# /data/nginx/sbin/nginx -s reload

初始化参数(各个节点都执行、HA节点除外)

[root@openstack-master1 ~]# sh k8s.sh
[root@openstack-master1 ~]# cat k8s.sh
#! /bin/bash
yum install wget jq psmisc vim net-tools telnet device-mapper-persistent-data lvm2 git -y
echo "===================安装基础工具========================="
systemctl disable --now firewalld
echo "===================关闭防火墙========================="
systemctl disable --noe dnsmasq
echo "===================关闭dnsmasq========================="
system_type=$(uname -s)
echo "$system_type"

if [ $system_type = "Kylin" ]; then
    systemctl disable --now NetworkManager
    echo "===================麒麟系统关闭NetworkManager========================="
fi

setenforce 0
sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/sysconfig/selinux
sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/selinux/config
swapoff -a && sysctl -w vm.swappiness=0
sed -ri '/^[^#]*swap/s@^@#@' /etc/fstab
ulimit -SHn 65535
limit_src="/etc/security/limits.conf"
limit_txt=$(cat <<EOF
# 末尾添加如下内容
* soft nofile 65536
* hard nofile 131072
* soft nproc 65535
* hard nproc 655350
* soft memlock unlimited
* hard memlock unlimited　
EOF
)
echo "$limit_txt" >> "$limit_src"
echo "===================开始安装ipvs==============="
yum install ipvsadm ipset sysstat conntrack libseccomp -y
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack
modprobe br_netfilter

ipvs_txt=$(cat <<EOF
# 加入以下内容
ip_vs
ip_vs_lc
ip_vs_wlc
ip_vs_rr
ip_vs_wrr
ip_vs_lblc
ip_vs_lblcr
ip_vs_dh
ip_vs_sh
ip_vs_fo
ip_vs_nq
ip_vs_sed
ip_vs_ftp
ip_vs_sh
nf_conntrack
ip_tables
ip_set
xt_set
ipt_set
ipt_rpfilter
ipt_REJECT
ipip　
EOF
)
echo "$ipvs_txt" >> /etc/modules-load.d/ipvs.conf
systemctl enable --now systemd-modules-load.service
lsmod | grep -e ip_vs -e nf_conntrack
k8s_txt=$(cat <<EOF
# 贴入以下内容（大概就是开启转发，还有一些网络的内核参数优化）
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
vm.overcommit_memory=1
net.ipv4.conf.all.route_localnet = 1

vm.panic_on_oom=0
fs.inotify.max_user_watches=89100
fs.file-max=52706963
fs.nr_open=52706963
net.netfilter.nf_conntrack_max=2310720

net.ipv4.tcp_keepalive_time = 600
net.ipv4.tcp_keepalive_probes = 3
net.ipv4.tcp_keepalive_intvl =15
net.ipv4.tcp_max_tw_buckets = 36000
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_max_orphans = 327680
net.ipv4.tcp_orphan_retries = 3
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.tcp_timestamps = 0
net.core.somaxconn = 16384
EOF
)
echo "$k8s_txt" >> /etc/sysctl.d/k8s.conf
sysctl -p /etc/sysctl.d/k8s.conf

[root@openstack-master1 ~]# sh k8s.sh

安装docker 所有节点(HA两节点不需要安装)

wget https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo -O /etc/yum.repos.d/docker-ce.repo
yum clean all
yum makecache
#查看所有仓库中所有docker版本，并选择特定版本安装
yum list docker-ce --showduplicates | sort -r
yum install docker-ce-24.0.9-1.el9 -y

#docker配置文件
[root@openstack-master1 ~]# cat /etc/docker/daemon.json 
{
  "registry-mirrors": ["https://docker.mirrors.ustc.edu.cn","https://mirror.iscas.ac.cn","https://docker.m.daocloud.io"],
  "exec-opts": ["native.cgroupdriver=systemd"],
  "log-driver": "json-file",
  "log-opts": {
     "max-size": "100m",
     "max-file": "10"
  },
  "storage-driver": "overlay2",
  "live-restore": true,
  "default-shm-size": "128M",
  "max-concurrent-downloads": 10,
  "max-concurrent-uploads": 10,
  "debug": false
}

#启动docker
systemctl enable --now docker.service

安装cri-dockerd 所有节点(HA两节点不需要安装)

wget https://github.com/Mirantis/cri-dockerd/releases/download/v0.3.8/cri-dockerd-0.3.8.amd64.tgz
#不同版本cri
wget  https://github.com/Mirantis/cri-dockerd/releases/download/v0.3.10/cri-dockerd-0.3.10.amd64.tgz
tar -zxvf cri-dockerd-0.3.8.amd64.tgz && cd cri-dockerd
mv cri-dockerd /usr/bin/

#配置cri-dockerd启动文件
cat > /etc/systemd/system/cri-docker.service<<EOF
[Unit]
Description=CRI Interface for Docker Application Container Engine
Documentation=https://docs.mirantis.com
After=network-online.target firewalld.service docker.service
Wants=network-online.target
Requires=cri-docker.socket

[Service]
Type=notify
ExecStart=/usr/bin/cri-dockerd --network-plugin=cni --pod-infra-container-image=registry.aliyuncs.com/google_containers/pause:3.9
ExecReload=/bin/kill -s HUP 
TimeoutSec=0
RestartSec=2
Restart=always
StartLimitBurst=3
StartLimitInterval=60s
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
TasksMax=infinity
Delegate=yes
KillMode=process

[Install]
WantedBy=multi-user.target
EOF

cat > /etc/systemd/system/cri-docker.socket <<EOF
[Unit]
Description=CRI Docker Socket for the API
PartOf=cri-docker.service

[Socket]
ListenStream=%t/cri-dockerd.sock
SocketMode=0660
SocketUser=root
SocketGroup=docker

[Install]
WantedBy=sockets.target
EOF

#启动cri-dockerd
systemctl daemon-reload
systemctl enable --now cri-docker.service

安装kubernetes 所有节点(HA两节点不需要安装)

#配置kubernetes源
cat <<EOF | tee /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes-new/core/stable/v1.29/rpm/
enabled=1
gpgcheck=1
gpgkey=https://mirrors.aliyun.com/kubernetes-new/core/stable/v1.29/rpm/repodata/repomd.xml.key
EOF

#缓存
yum clean all
yum makecache
yum install -y kubelet-1.29.14 kubeadm-1.29.14 kubectl-1.29.14

# 配置 cgroup 驱动与docker一致
cp /etc/sysconfig/kubelet{,.bak}
cat > /etc/sysconfig/kubelet <<EOF
KUBELET_EXTRA_ARGS="--cgroup-driver=systemd"
EOF

#配置自启动kubelet
systemctl enable kubelet

集群初始化(在master1节点操作即可)

[root@openstack-master1 ~]# kubeadm config print init-defaults > kubeadm.yaml
#修改配置如下配置
[root@openstack-master1 ~]# cat kubeadm.yaml 
apiVersion: kubeadm.k8s.io/v1beta3
bootstrapTokens:
- groups:
  - system:bootstrappers:kubeadm:default-node-token
  token: abcdef.0123456789abcdef
  ttl: 24h0m0s
  usages:
  - signing
  - authentication
kind: InitConfiguration
localAPIEndpoint:
  advertiseAddress: 10.1.20.200   #本地ip地址
  bindPort: 6443
nodeRegistration:
  criSocket: unix:///var/run/cri-dockerd.sock
  imagePullPolicy: IfNotPresent
  name: rocky-k8s-master1-200
  taints: null
---
apiServer:
  timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta3
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controllerManager: {}
dns: {}
etcd:
  local:
    dataDir: /var/lib/etcd
imageRepository: registry.aliyuncs.com/google_containers
kind: ClusterConfiguration
kubernetesVersion: 1.29.14 #安装版本定义
controlPlaneEndpoint: 10.1.20.52:6443  #注意! 此处填写的是高可用地址
networking:
  dnsDomain: cluster.local
  podSubnet: 172.16.0.0/12
  serviceSubnet: 10.96.0.0/12
scheduler: {}
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
mode: ipvs
---
apiVersion: kubelet.config.k8s.io/v1beta1
authentication:
  anonymous:
    enabled: false
  webhook:
    cacheTTL: 0s
    enabled: true
  x509:
    clientCAFile: /etc/kubernetes/pki/ca.crt
authorization:
  mode: Webhook
  webhook:
    cacheAuthorizedTTL: 0s
    cacheUnauthorizedTTL: 0s
clusterDNS:
- 10.96.0.10
clusterDomain: cluster.local
cpuManagerReconcilePeriod: 0s
evictionPressureTransitionPeriod: 0s
fileCheckFrequency: 0s
healthzBindAddress: 127.0.0.1
healthzPort: 10248
httpCheckFrequency: 0s
imageMinimumGCAge: 0s
kind: KubeletConfiguration
cgroupDriver: systemd
logging: {}
memorySwap: {}
nodeStatusReportFrequency: 0s
nodeStatusUpdateFrequency: 0s
rotateCertificates: true
runtimeRequestTimeout: 0s
shutdownGracePeriod: 0s
shutdownGracePeriodCriticalPods: 0s
staticPodPath: /etc/kubernetes/manifests
streamingConnectionIdleTimeout: 0s
syncFrequency: 0s
volumeStatsAggPeriod: 0s
[root@openstack-master1 ~]# kubeadm init --config kubeadm.yaml --ignore-preflight-errors=all --cri-socket=unix:///var/run/cri-dockerd.sock --upload-certs

#初始化命令说明
#--cri-socket=unix:///var/run/cri-dockerd.sock docker垫片
#--ignore-preflight-errors=all 忽略检查的一些错误

#最终初始成功的后会输出以下信息
.......................................
[addons] Applied essential addon: kube-proxy
Your Kubernetes control-plane has initialized successfully!
To start using your cluster, you need to run the following as a regular user:

  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config

Alternatively, if you are the root user, you can run:

  export KUBECONFIG=/etc/kubernetes/admin.conf

You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  https://kubernetes.io/docs/concepts/cluster-administration/addons/

You can now join any number of control-plane nodes by copying certificate authorities
and service account keys on each node and then running the following as root:

  kubeadm join 10.1.20.52:8443 --token x1v36a.lqe5ul9zpzx55b10 \
	--discovery-token-ca-cert-hash sha256:869a5df85403ce519a47b6444dd120d88feccbf54356e510dc3c09f55a76f678 \
	--control-plane 

Then you can join any number of worker nodes by running the following on each as root:

kubeadm join 10.1.20.52:8443 --token x1v36a.lqe5ul9zpzx55b10 \
	--discovery-token-ca-cert-hash sha256:869a5df85403ce519a47b6444dd120d88feccbf54356e510dc3c09f55a76f678 

#按照上面的信息提示，对应的步骤即可
#上面初始化完成master01节点之后会提示你在master节点或node节点执行对应的命令来将master节点或node节点加入k8s集群
#注意：这段kubeamd join命令的token只有24h，24h就过期，需要执行kubeadm token create --print-join-command  重新生成token，但是
#要注意，重新生成的加入集群命令默认是node节点角色加入的，如果新节点是作为master角色加入集群，需要在打印出来的命令后面添加--control-plane 参数再执行。

#主节点添加配置
[root@openstack-master1 ~]# mkdir -p $HOME/.kube
[root@openstack-master1 ~]# sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
[root@openstack-master1 ~]# sudo chown $(id -u):$(id -g) $HOME/.kube/config

#master节点加入集群
[root@openstack-master1 ~]# kubeadm join 10.1.20.52:8443 --token x1v36a.lqe5ul9zpzx55b10 --discovery-token-ca-cert-hash sha256:869a5df85403ce519a47b6444dd120d88feccbf54356e510dc3c09f55a76f678 --control-plane --certificate-key 1a25a7ff664d170e865acfb296442a612c985292bfaebb772196cd001d6f2bec  --cri-socket=unix:///var/run/cri-dockerd.sock --ignore-preflight-errors=all

#node节点接入集群
kubeadm join 10.1.20.52:8443 --token x1v36a.lqe5ul9zpzx55b10 --discovery-token-ca-cert-hash sha256:869a5df85403ce519a47b6444dd120d88feccbf54356e510dc3c09f55a76f678 --cri-socket=unix:///var/run/cri-dockerd.sock --ignore-preflight-errors=all

#查看集群
[root@openstack-master1 ~]# kubectl get node
NAME       STATUS     ROLES                  AGE     VERSION
openstack-master1   NotReady   control-plane   50m     v1.29.1
openstack-master2   NotReady   control-plane   6m58s   v1.29.1
openstack-master3   NotReady   control-plane   6m10s   v1.29.1
openstack-work1     NotReady   <none>          39s     v1.29.1

重置集群操作

[root@openstack-master1 ~]# kubeadm reset --cri-socket unix:///var/run/cri-dockerd.sock
#需要加上配置指定cri-docker.sock文件

安装calico插件

wget https://raw.githubusercontent.com/projectcalico/calico/v3.27.0/manifests/calico.yaml