antony@notes:~/learning-notes$ cat "Build-HA-K8S-3M.md"
Build HA K8S - 3M
Build HA K8S - 3M
:::warning
:::spoiler 目錄
[TOC]
:::
1. 安裝 kubectl , kubeadm , kubelet
安裝 Kubernetes 要用的一些管理命令
$ sudo apk update; sudo apk add kubeadm kubelet kubectl --update-cache --repository http://dl-3.alpinelinux.org/alpine/edge/testing/ --allow-untrusted
$ ssh m2 'sudo apk update; sudo apk add kubeadm kubelet kubectl --update-cache --repository http://dl-3.alpinelinux.org/alpine/edge/testing/ --allow-untrusted'
$ ssh m3 'sudo apk update; sudo apk add kubeadm kubelet kubectl --update-cache --repository http://dl-3.alpinelinux.org/alpine/edge/testing/ --allow-untrusted'將 m1, m2, m3 的 kubelet 設定為系統自動啟動
$ sudo rc-update add kubelet default; ssh m2 'sudo rc-update add kubelet default'; ssh m3 'sudo rc-update add kubelet default':::spoiler 檢查 檢查有沒有安裝 $ sudo rc-service kubelet status *status: stopped (k8s還沒起來)
$ ssh m2 'sudo rc-service kubelet status';ssh m3 'sudo rc-service kubelet status'
:::
2. kube-vip (set up m1 node)
- 開機一定會自動產生的 pod
- static pods 的 yaml 檔案一定會放在 /etc/kubenetes/manifests
- 一定會在 node 上面 run
# Set configuration details
#IP 要改成叢集外的新 IP
$ export VIP=120.96.143.xxx (新的IP， 不是 cluster 的 IP)
($ export INTERFACE=eth0 這行可以不用做因為 環境變數 $env 有宣告 GWIF=eth0)
# Configure to use a container runtime
# 取得版本代號
$ export KVVERSION=$(curl -sL https://api.github.com/repos/kube-vip/kube-vip/releases | jq -r ".[0].name")
#檢查
$ echo $KVVERSION
v0.5.7
# podman
$ alias kube-vip="sudo podman run --network host --rm ghcr.io/kube-vip/kube-vip:$KVVERSION"
* image=ghcr.io/kube-vip/kube-vip:$KVVERSION
# ARP
# This configuration will create a manifest that starts kube-vip providing controlplane and services management, using leaderElection. When this instance is elected as the leader it will bind the vip to the specified interface, this is also the same for services of type:LoadBalancer.
# 手動建立 static pods 的目錄
$ sudo mkdir -p /etc/kubernetes/manifests/
$ kube-vip manifest pod \
--interface $GWIF \
--vip $VIP \
--controlplane \
--arp \
--leaderElection | sudo tee /etc/kubernetes/manifests/kube-vip.yaml:::spoiler 解釋
--vip : vip 是 kube vip 使用的
這些指令都會在 container 裡面執行 ,也就是把 kube-vip 做出來 ,且放到 static pods 的目錄區 ,讓他自動啟動
使用內建的環境變數 GWIF 代表 eth0 -> 也代表 ip 位置 .
把 yaml 放到 m1 的 static 目錄區
:::
3. 建立 k8s 系統
命令式
將整個 Clusert 重開，後初始化 K8S
$ sudo kubeadm init --service-cidr 10.98.0.0/24 --pod-network-cidr 10.244.0.0/16 --service-dns-domain=k8s.org --apiserver-advertise-address $IP --control-plane-endpoint "120.96.143.xxx:6443" --upload-certs:::spoiler 解釋
--control-plane-endpoint "LOAD_BALANCER_DNS:LOAD_BALANCER_PORT"：代入的DNS及PORT為 kube-apiserver的域名，port預設使用 6443。--upload-certs：自動將 control plane 會用到的 certificates 儲存到名為 kubeadm-certs 的 secret 中。 :::
設定檔方式
編輯 init-config.yaml
$ nano "${HOME}"/init-config.yaml檔案內容 :
apiVersion: kubeadm.k8s.io/v1beta3
kind: InitConfiguration
localAPIEndpoint:
advertiseAddress: 120.96.143.61 # change from Master node IP
bindPort: 6443
nodeRegistration:
criSocket: unix:///run/crio/crio.sock # change from CRI-O Unix Socket
imagePullPolicy: IfNotPresent
name: k2m1 # change from Master node hsotname
taints: []
---
apiServer:
timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta3
certificatesDir: /etc/kubernetes/pki
clusterName: topgun # set your clusterName
controllerManager: {}
dns: {}
etcd:
local:
dataDir: /var/lib/etcd
imageRepository: registry.k8s.io
kind: ClusterConfiguration
kubernetesVersion: 1.26.1
networking:
dnsDomain: k8s.org # DNS domain used by Kubernetes Services.
podSubnet: 10.244.0.0/16 # the subnet used by Pods.
serviceSubnet: 10.98.0.0/24 # subnet used by Kubernetes Services.
scheduler: {}
controlPlaneEndpoint: 120.96.143.60:6443初始化 K8S 命令
sudo kubeadm init --upload-certs --config="${HOME}"/init-config.yaml4. 將 bigred 設成 K8S 管理者
$ mkdir -p $HOME/.kube; sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config; sudo chown $(id -u):$(id -g) $HOME/.kube/config5. 設定 K8S Master 可以執行 Pod
$ kubectl taint node m1 node-role.kubernetes.io/control-plane:NoSchedule-
node/m1 untainted
6. 安裝 Flannel 網路套件
$ kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml7. join m2 and m3 control-plane node
$ ssh m2 'sudo kubeadm join 120.96.143.xxx:6443 --token 9t3pt8.j1smywokfw7qg5ez \
--discovery-token-ca-cert-hash sha256:606851b5dca00e0a375da14ba35bb2c91dbef7ce013757d4422fc25620ee9791 \
--control-plane --certificate-key 4a8d586ecb8706b01ec05547835587d5c7ffca2d18ccde8a824559f22b8e8432'
# ssh 到 m2 將 bigred 設成 K8S 管理者
$ ssh m2 'mkdir -p $HOME/.kube; sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config; sudo chown $(id -u):$(id -g) $HOME/.kube/config'
$ ssh m3 'sudo kubeadm join 120.96.143.xxx:6443 --token 9t3pt8.j1smywokfw7qg5ez \
--discovery-token-ca-cert-hash sha256:606851b5dca00e0a375da14ba35bb2c91dbef7ce013757d4422fc25620ee9791 \
--control-plane --certificate-key 4a8d586ecb8706b01ec05547835587d5c7ffca2d18ccde8a824559f22b8e8432'
# ssh 到 m3 將 bigred 設成 K8S 管理者
$ ssh m3 'mkdir -p $HOME/.kube; sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config; sudo chown $(id -u):$(id -g) $HOME/.kube/config'create control-plane install command
$ certs=$(sudo kubeadm init phase upload-certs --upload-certs | tail -n 1)
$ echo -e "\nPlease run this command to join master node:\n\n\
sudo $(sudo kubeadm token create --print-join-command) --control-plane --certificate-key ${certs}"- 注意 ! 這行命令只能在有 kubeVIP 的那台 master node 執行 !
8. set up m2 node
# Set configuration details
$ export VIP=120.96.143.xxx (新的IP， 不是 cluster 的 IP)
# Configure to use a container runtime
$ export KVVERSION=$(curl -sL https://api.github.com/repos/kube-vip/kube-vip/releases | jq -r ".[0].name")
$ echo $KVVERSION
v0.5.7
# podman
$ alias kube-vip="sudo podman run --network host --rm ghcr.io/kube-vip/kube-vip:$KVVERSION"
# ARP
# This configuration will create a manifest that starts kube-vip providing controlplane and services management, using leaderElection. When this instance is elected as the leader it will bind the vip to the specified interface, this is also the same for services of type:LoadBalancer.
$ kube-vip manifest pod \
--interface $GWIF \
--vip $VIP \
--controlplane \
--arp \
--leaderElection | sudo tee /etc/kubernetes/manifests/kube-vip.yaml設定 K8S Master 可以執行 Pod
$ kubectl taint node m2 node-role.kubernetes.io/control-plane:NoSchedule-
node/m1 untainted
9. set up m3 node
# Set configuration details
$ export VIP=120.96.143.xxx (新的IP， 不是 cluster 的 IP)
# Configure to use a container runtime
$ export KVVERSION=$(curl -sL https://api.github.com/repos/kube-vip/kube-vip/releases | jq -r ".[0].name")
$ echo $KVVERSION
v0.5.7
# podman
$ alias kube-vip="sudo podman run --network host --rm ghcr.io/kube-vip/kube-vip:$KVVERSION"
# ARP
# This configuration will create a manifest that starts kube-vip providing controlplane and services management, using leaderElection. When this instance is elected as the leader it will bind the vip to the specified interface, this is also the same for services of type:LoadBalancer.
$ kube-vip manifest pod \
--interface $GWIF \
--vip $VIP \
--controlplane \
--arp \
--leaderElection | sudo tee /etc/kubernetes/manifests/kube-vip.yaml設定 K8S Master 可以執行 Pod
$ kubectl taint node m3 node-role.kubernetes.io/control-plane:NoSchedule-
node/m1 untainted
安裝好後，將 m1 主機重新開機
$ sudo reboot10. 檢視 Kubernetes 系統狀態
檢查 node 的狀態
$ kubectl get nodes
NAME STATUS ROLES AGE VERSION
m1 Ready control-plane 18m v1.25.3
m2 Ready control-plane 11m v1.25.3
m3 Ready control-plane 10m v1.25.3檢查 kube-system 這個 Namespace 裡面 pod 的狀態
$ kubectl get pods -n kube-system
NAME READY STATUS RESTARTS AGE
coredns-565d847f94-mkj5s 1/1 Running 1 62m
coredns-565d847f94-wn6cp 1/1 Running 1 62m
etcd-d6m1 1/1 Running 2 62m
etcd-d6m2 1/1 Running 1 59m
etcd-d6m3 1/1 Running 1 58m
kube-apiserver-d6m1 1/1 Running 2 62m
kube-apiserver-d6m2 1/1 Running 2 59m
kube-apiserver-d6m3 1/1 Running 4 (16m ago) 57m
kube-controller-manager-d6m1 1/1 Running 2 62m
kube-controller-manager-d6m2 1/1 Running 1 58m
kube-controller-manager-d6m3 1/1 Running 1 56m
kube-proxy-2986z 1/1 Running 1 59m
kube-proxy-q7xt6 1/1 Running 2 46m
kube-proxy-v9kc2 1/1 Running 1 45m
kube-proxy-vfpwm 1/1 Running 2 62m
kube-proxy-vlswl 1/1 Running 1 58m
kube-scheduler-d6m1 1/1 Running 2 62m
kube-scheduler-d6m2 1/1 Running 1 59m
kube-scheduler-d6m3 1/1 Running 2 58m
kube-vip-d6m1 1/1 Running 3 62m
kube-vip-d6m2 1/1 Running 1 51m
kube-vip-d6m3 1/1 Running 1 52m開始建置 Kubernetes Worker
w1, w2 安裝 kubelet 及 kubeadm 命令
# ssh 到 w1 主機安裝 kubelet 及 kubeadm 命令
$ ssh w1 'sudo apk add kubeadm kubelet --update-cache --repository http://dl-3.alpinelinux.org/alpine/edge/testing/ --allow-untrusted'
# ssh 到 w2 主機安裝 kubelet 及 kubeadm 命令
$ ssh w2 'sudo apk add kubeadm kubelet --update-cache --repository http://dl-3.alpinelinux.org/alpine/edge/testing/ --allow-untrusted'
# ssh 到 w1, w2 主機，將 kubelet 設成 Deamon
$ ssh w1 'sudo rc-update add kubelet default'; ssh w2 'sudo rc-update add kubelet default'w1, w2 加入 K8S Cluster
在 m1 的終端機, 執行以下命令
$ export JOIN=$(echo " sudo `kubeadm token create --print-join-command 2>/dev/null`")
$ ssh w1 "$JOIN"; ssh w2 "$JOIN"
$ ssh w1 sudo reboot; ssh w2 sudo reboot設定 K8S Worker 標籤
$ kubectl label node w1 node-role.kubernetes.io/worker=; kubectl label node w2 node-role.kubernetes.io/worker=檢查是否符合預期
$ kubectl get nodes
NAME STATUS ROLES AGE VERSION
m1 Ready control-plane 61m v1.25.0
m2 Ready control-plane 58m v1.25.0
m3 Ready control-plane 57m v1.25.0
w1 Ready worker 45m v1.25.0
w2 Ready worker 44m v1.25.0測試
$ ks
NAME READY STATUS RESTARTS AGE
coredns-565d847f94-mkj5s 1/1 Running 1 3h5m
coredns-565d847f94-wn6cp 1/1 Running 1 3h5m
etcd-d6m1 1/1 Running 2 3h5m
etcd-d6m2 1/1 Running 1 3h2m
etcd-d6m3 1/1 Running 1 3h
kube-apiserver-d6m1 1/1 Running 2 3h5m
kube-apiserver-d6m2 1/1 Running 2 3h2m
kube-apiserver-d6m3 1/1 Running 4 (139m ago) 3h
kube-controller-manager-d6m1 1/1 Running 2 3h5m
kube-controller-manager-d6m2 1/1 Running 1 3h1m
kube-controller-manager-d6m3 1/1 Running 1 179m
kube-proxy-2986z 1/1 Running 1 3h2m
kube-proxy-q7xt6 1/1 Running 2 169m
kube-proxy-v9kc2 1/1 Running 1 168m
kube-proxy-vfpwm 1/1 Running 2 3h5m
kube-proxy-vlswl 1/1 Running 1 3h1m
kube-scheduler-d6m1 1/1 Running 2 3h5m
kube-scheduler-d6m2 1/1 Running 1 3h2m
kube-scheduler-d6m3 1/1 Running 2 3h1m
kube-vip-d6m1 1/1 Running 3 3h5m
kube-vip-d6m2 1/1 Running 1 174m
kube-vip-d6m3 1/1 Running 1 175m$ kubectl logs -n kube-system kube-vip-d6m1
time="2022-12-20T08:16:19Z" level=info msg="Starting kube-vip.io [v0.5.7]"
time="2022-12-20T08:16:19Z" level=info msg="namespace [kube-system], Mode: [ARP], Features(s): Control Plane:[true], Services:[false]"
time="2022-12-20T08:16:19Z" level=info msg="prometheus HTTP server started"
time="2022-12-20T08:16:19Z" level=info msg="Starting Kube-vip Manager with the ARP engine"
time="2022-12-20T08:16:19Z" level=info msg="Beginning cluster membership, namespace [kube-system], lock name [plndr-cp-lock], id [d6m1]"
I1220 08:16:19.293587 1 leaderelection.go:248] attempting to acquire leader lease kube-system/plndr-cp-lock...
time="2022-12-20T08:16:20Z" level=info msg="Node [d6m2] is assuming leadership of the cluster"
time="2022-12-20T08:16:26Z" level=info msg="Node [d6m3] is assuming leadership of the cluster"$ kubectl logs -n kube-system kube-vip-d6m2
time="2022-12-20T08:19:20Z" level=info msg="Starting kube-vip.io [v0.5.7]"
time="2022-12-20T08:19:20Z" level=info msg="namespace [kube-system], Mode: [ARP], Features(s): Control Plane:[true], Services:[false]"
time="2022-12-20T08:19:20Z" level=info msg="prometheus HTTP server started"
time="2022-12-20T08:19:20Z" level=info msg="Starting Kube-vip Manager with the ARP engine"
time="2022-12-20T08:19:20Z" level=info msg="Beginning cluster membership, namespace [kube-system], lock name [plndr-cp-lock], id [d6m2]"
I1220 08:19:20.510887 1 leaderelection.go:248] attempting to acquire leader lease kube-system/plndr-cp-lock...
time="2022-12-20T08:19:20Z" level=info msg="Node [d6m3] is assuming leadership of the cluster"$ kubectl logs -n kube-system kube-vip-d6m3
............
E1220 08:16:01.515714 1 leaderelection.go:330] error retrieving resource lock kube-system/plndr-cp-lock: Get "https://kubernetes:6443/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/plndr-cp-lock": dial tcp 127.0.0.1:6443: connect: connection refused
E1220 08:16:13.254844 1 leaderelection.go:330] error retrieving resource lock kube-system/plndr-cp-lock: Get "https://kubernetes:6443/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/plndr-cp-lock": net/http: TLS handshake timeout
time="2022-12-20T08:16:20Z" level=info msg="Node [d6m2] is assuming leadership of the cluster"
I1220 08:16:25.734487 1 leaderelection.go:258] successfully acquired lease kube-system/plndr-cp-lock
time="2022-12-20T08:16:25Z" level=info msg="Node [d6m3] is assuming leadership of the cluster"
time="2022-12-20T08:16:25Z" level=info msg="Gratuitous Arp broadcast will repeat every 3 seconds for [120.96.143.248]"bigred@d6m1:$ sudo poweroff
bigred@d6m2:$ kg all
NAME READY STATUS RESTARTS AGE
pod/presenation-bd9bb89cd-ptszx 1/1 Running 0 13s
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
service/kubernetes ClusterIP 10.98.0.1 <none> 443/TCP 3h9m
NAME READY UP-TO-DATE AVAILABLE AGE
deployment.apps/presenation 1/1 1 1 14s
NAME DESIRED CURRENT READY AGE
replicaset.apps/presenation-bd9bb89cd 1 1 1 14s一開始 m1 當 leadership of the cluster ，當我們把 m1 關機
- pod 還是正常 running
- leadership of the cluster 變成 m3
$ kubectl logs -n kube-system kube-vip-d6m2
time="2022-12-20T08:19:20Z" level=info msg="Starting kube-vip.io [v0.5.7]"
time="2022-12-20T08:19:20Z" level=info msg="namespace [kube-system], Mode: [ARP], Features(s): Control Plane:[true], Services:[false]"
time="2022-12-20T08:19:20Z" level=info msg="prometheus HTTP server started"
time="2022-12-20T08:19:20Z" level=info msg="Starting Kube-vip Manager with the ARP engine"
time="2022-12-20T08:19:20Z" level=info msg="Beginning cluster membership, namespace [kube-system], lock name [plndr-cp-lock], id [d6m2]"
I1220 08:19:20.510887 1 leaderelection.go:248] attempting to acquire leader lease kube-system/plndr-cp-lock...
time="2022-12-20T08:19:20Z" level=info msg="Node [d6m3] is assuming leadership of the cluster"$ kubectl logs -n kube-system kube-vip-d6m3
E1220 08:16:13.254844 1 leaderelection.go:330] error retrieving resource lock kube-system/plndr-cp-lock: Get "https://kubernetes:6443/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/plndr-cp-lock": net/http: TLS handshake timeout
time="2022-12-20T08:16:20Z" level=info msg="Node [d6m2] is assuming leadership of the cluster"
I1220 08:16:25.734487 1 leaderelection.go:258] successfully acquired lease kube-system/plndr-cp-lock
time="2022-12-20T08:16:25Z" level=info msg="Node [d6m3] is assuming leadership of the cluster"
time="2022-12-20T08:16:25Z" level=info msg="Gratuitous Arp broadcast will repeat every 3 seconds for [120.96.143.248]"重新建立損壞 master
參考文件
- https://stackoverflow.com/questions/67921552/re-installed-node-cannot-join-kubernetes-cluster
- https://facsiaginsa.com/kubernetes/join-existing-kubernetes-cluster
- https://stackoverflow.com/questions/35757620/how-to-gracefully-remove-a-node-from-kubernetes
- https://facsiaginsa.com/kubernetes/join-existing-kubernetes-cluster
- https://blog.scottlowe.org/2019/08/15/reconstructing-the-join-command-for-kubeadm/
- https://stackoverflow.com/questions/67921552/re-installed-node-cannot-join-kubernetes-cluster
- https://www.kancloud.cn/pshizhsysu/kubernetes/2055662
在 Alpine Linux 安裝 etcd-ctl 命令套件檔
$ sudo apk add etcd-ctl \
--no-cache \
--update-cache \
--allow-untrusted \
--repository http://dl-cdn.alpinelinux.org/alpine/edge/testing$ kubectl get pods -n kube-system | grep etcd
etcd-d4m1 1/1 Running 8 8d
etcd-d4m2 1/1 Running 8 8d
etcd-d4m3 0/1 CrashLoopBackOff 1020 (3m ago) 19h清除損壞的 master node
$ ks
NAME READY STATUS RESTARTS AGE
coredns-565d847f94-h6kwm 1/1 Running 7 15d
coredns-565d847f94-rjmkc 1/1 Running 7 15d
etcd-d3m1 1/1 Running 8 15d
etcd-d3m2 0/1 CrashLoopBackOff 885 (2m1s ago) 9h
etcd-d3m3 1/1 Running 8 15d
kube-apiserver-d3m1 1/1 Running 22 15d
kube-apiserver-d3m2 0/1 CrashLoopBackOff 866 (4m15s ago) 6d21h
kube-apiserver-d3m3 1/1 Running 23 (9h ago) 15d
kube-controller-manager-d3m1 1/1 Running 10 15d
kube-controller-manager-d3m2 1/1 Running 12 6d21h
kube-controller-manager-d3m3 1/1 Running 12 (9h ago) 15d
kube-proxy-8m84p 1/1 Running 7 15d
kube-proxy-9hwvx 1/1 Running 9 15d
kube-proxy-hvmvl 1/1 Running 9 15d
kube-proxy-j4l4m 1/1 Running 0 6d21h
kube-proxy-zn9d2 1/1 Running 8 15d
kube-scheduler-d3m1 1/1 Running 10 15d
kube-scheduler-d3m2 1/1 Running 11 6d21h
kube-scheduler-d3m3 1/1 Running 13 (9h ago) 15d
kube-vip-d3m1 1/1 Running 24 15d
kube-vip-d3m2 1/1 Running 11 6d21h
kube-vip-d3m3 1/1 Running 28 (9h ago) 15d$ kubectl get nodes
$ kubectl drain <node-name>
$ kubectl drain <node-name> --ignore-daemonsets --delete-local-data
$ kubectl delete node <node-name>找到那一台 壞掉 master node 的 ID
Find the id of the member to remove
sudo ETCDCTL_API=3 etcdctl --endpoints 127.0.0.1:2379 --cacert /etc/kubernetes/pki/etcd/ca.crt --cert /etc/kubernetes/pki/etcd/server.crt --key /etc/kubernetes/pki/etcd/server.key member list出現以下的結果
78c846a8a9465434, started, d3m3, https://120.96.143.183:2380, https://120.96.143.183:2379, false
92df15cb2da41804, started, d3m1, https://120.96.143.181:2380, https://120.96.143.181:2379, false
d3aa799c592c5386, started, d3m2, https://120.96.143.182:2380, https://120.96.143.182:2379, false把損壞刪除 ==I want to remove d3aa799c592c5386==
sudo ETCDCTL_API=3 etcdctl --endpoints 127.0.0.1:2379 --cacert /etc/kubernetes/pki/etcd/ca.crt --cert /etc/kubernetes/pki/etcd/server.crt --key /etc/kubernetes/pki/etcd/server.key member remove d3aa799c592c5386Checking, 檢查
sudo ETCDCTL_API=3 etcdctl --endpoints 127.0.0.1:2379 --cacert /etc/kubernetes/pki/etcd/ca.crt --cert /etc/kubernetes/pki/etcd/server.crt --key /etc/kubernetes/pki/etcd/server.key member listNow I can join my new master.
Join a Node to a Kubernetes Cluster
$ kubectl get cm kubeadm-config -n kube-system -o yaml
apiVersion: v1
data:
ClusterConfiguration: |
apiServer:
extraArgs:
authorization-mode: Node,RBAC
timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta3
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controlPlaneEndpoint: 120.96.143.194:6443
controllerManager: {}
dns: {}
etcd:
local:
dataDir: /var/lib/etcd
imageRepository: registry.k8s.io
kind: ClusterConfiguration
kubernetesVersion: v1.25.5
networking:
dnsDomain: k8s.org
podSubnet: 10.244.0.0/16
serviceSubnet: 10.98.0.0/24
scheduler: {}
kind: ConfigMap
metadata:
creationTimestamp: "2022-12-20T07:12:01Z"
name: kubeadm-config
namespace: kube-system
resourceVersion: "199"
uid: 75169c8c-9389-4f20-bacd-56b3f182f64d
複製以上那段貼到新的 nano 檔案
- Copy the marked text under ClusterConfiguration:, create a new file to store the configuration
$ nano kubeadm-config.yaml
apiServer:
extraArgs:
authorization-mode: Node,RBAC
timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta3
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controlPlaneEndpoint: 120.96.143.194:6443
controllerManager: {}
dns: {}
etcd:
local:
dataDir: /var/lib/etcd
imageRepository: registry.k8s.io
kind: ClusterConfiguration
kubernetesVersion: v1.25.5
networking:
dnsDomain: k8s.org
podSubnet: 10.244.0.0/16
serviceSubnet: 10.98.0.0/24
scheduler: {}Get Kubeadm Join Command for Master Node
$ sudo kubeadm init phase upload-certs --upload-certs --config kubeadm-config.yaml
[upload-certs] Storing the certificates in Secret “kubeadm-certs” in the “kube-system” Namespace [upload-certs] Using certificate key: ==ec52acebb6630efa584b6ff784db4a50ac59f8c22bc81baab6b82c1e70244787==
The marked part above is the ==certificate key== that we need to pass to the join command. 上面標記的部分是我們需要傳遞給加入命令的證書密鑰。
$ kubeadm token create --print-join-command
==kubeadm join 120.96.143.194:6443 –token nlzfjg.6e5x6p2hb36rcjd3 –discovery-token-ca-cert-hash sha256:2a3bd2c4c5a6de8332503d95dcdef141f45790504f5a2c11ea96c06bd86ca87c==
Joining Master Node to Kubernetes CLuster
Now, execute the join command with this format
<basic join command> --control-plane --certificate-key <ceritifcate-key>
In this case, I will execute this command to join my new master node:
==$ ssh <壞掉的 node>==
$ sudo kubeadm join 120.96.143.194:6443 --token nlzfjg.6e5x6p2hb36rcjd3 --discovery-token-ca-cert-hash sha256:2a3bd2c4c5a6de8332503d95dcdef141f45790504f5a2c11ea96c06bd86ca87c --control-plane --certificate-key ec52acebb6630efa584b6ff784db4a50ac59f8c22bc81baab6b82c1e70244787
如果不是全新的機器要把舊的檔案，目錄刪除
$ sudo rm -r /etc/kubernetes/kubelet.conf $ sudo rm -r /etc/kubernetes/manifests/ $ sudo rm -r /var/lib/etcd $ sudo reboot $ sudo rc-service kubelet stop
$ sudo kubeadm join 120.96.143.194:6443 --token nlzfjg.6e5x6p2hb36rcjd3 --discovery-token-ca-cert-hash sha256:2a3bd2c4c5a6de8332503d95dcdef141f45790504f5a2c11ea96c06bd86ca87c --control-plane --certificate-key ec52acebb6630efa584b6ff784db4a50ac59f8c22bc81baab6b82c1e70244787
加到 kube-vip
$ mkdir -p $HOME/.kube $ sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
cp: overwrite ‘/home/bigred/.kube/config’? ==y==
$ sudo chown $(id -u):$(id -g) $HOME/.kube/config
$ kubectl get nodes
NAME STATUS ROLES AGE VERSION
d4m1 Ready control-plane 9d v1.25.0
d4m2 Ready control-plane 9d v1.25.0
d4m3 Ready control-plane 18m v1.25.0
d4w1 Ready worker 8d v1.25.0
d4w2 Ready worker 8d v1.25.0#取得版本代號
$ export KVVERSION=$(curl -sL https://api.github.com/repos/kube-vip/kube-vip/releases | jq -r ".[0].name")
#檢查
$ echo $KVVERSION
v0.5.7
#podman
$ alias kube-vip="sudo podman run --network host --rm ghcr.io/kube-vip/kube-vip:$KVVERSION"
$ sudo mkdir -p /etc/kubernetes/manifests/
$ kube-vip manifest pod \
--interface eth0 \
--vip 120.96.143.194 \
--controlplane \
--arp \
--leaderElection | sudo tee /etc/kubernetes/manifests/kube-vip.yaml==$ kubectl taint node
$ ks
NAME READY STATUS RESTARTS AGE
coredns-565d847f94-69lpx 1/1 Running 9 9d
coredns-565d847f94-r626m 1/1 Running 9 9d
etcd-d4m1 1/1 Running 9 9d
etcd-d4m2 1/1 Running 8 9d
etcd-d4m3 1/1 Running 0 12m
kube-apiserver-d4m1 1/1 Running 14 9d
kube-apiserver-d4m2 1/1 Running 13 9d
kube-apiserver-d4m3 1/1 Running 0 12m
kube-controller-manager-d4m1 1/1 Running 13 9d
kube-controller-manager-d4m2 1/1 Running 15 (16m ago) 9d
kube-controller-manager-d4m3 1/1 Running 0 12m
kube-proxy-crwbg 1/1 Running 11 9d
kube-proxy-hbndh 1/1 Running 9 9d
kube-proxy-rnfxc 1/1 Running 8 9d
kube-proxy-t4bgz 1/1 Running 1 26m
kube-proxy-txfn6 1/1 Running 8 9d
kube-scheduler-d4m1 1/1 Running 13 9d
kube-scheduler-d4m2 1/1 Running 16 (16m ago) 9d
kube-scheduler-d4m3 1/1 Running 0 12m
kube-vip-d4m1 1/1 Running 26 9d
kube-vip-d4m2 1/1 Running 26 (21h ago) 9d
kube-vip-d4m3 1/1 Running 0 77s