antony@notes:~/rancher$ cat "Rancher-TroubleShooting.md"
Rancher TroubleShooting
Rancher TroubleShooting
問題 1: 無法刪除 DownStream Cluster
問題描述
在 Rancher 中無法刪除由 Harvester node provider 帶起來的 K3s 下游叢集，並顯示正在等待刪除節點。
解決辦法
# 1. find the cluster you want to delete
$ kubectl get clusters.management.cattle.io
NAME AGE
c-m-bzrsxlnb 62d
c-m-fb4btkbk 42d
local 62d
# 2. 宣告要刪除的 cluster 變數
$ export CLUSTERID="c-m-bzrsxlnb"
# 3. 將 finalizers 改成空值
$ kubectl patch clusters.management.cattle.io $CLUSTERID -p '{"metadata":{"finalizers":[]}}' --type=merge
cluster.management.cattle.io/c-m-bzrsxlnb patched
# 4. 刪除 Cluster
$ kubectl delete clusters.management.cattle.io $CLUSTERID
$ kubectl get clusters.management.cattle.io
NAME AGE
c-m-fb4btkbk 42d
local 62d
$ export CLUSTERID=c-m-fb4btkbk
$ kubectl patch clusters.management.cattle.io $CLUSTERID -p '{"metadata":{"finalizers":[]}}' --type=merge
cluster.management.cattle.io/c-m-fb4btkbk patched
$ kubectl delete clusters.management.cattle.io $CLUSTERID
Error from server (NotFound): clusters.management.cattle.io "c-m-fb4btkbk" not found
$ kubectl get clusters.management.cattle.io
NAME AGE
local 62d
$ kubectl -n cattle-system get pods
NAME READY STATUS RESTARTS AGE
rancher-7b4bdf6789-49sj4 1/1 Running 0 95m
rancher-webhook-644bcb4d99-mgfx9 1/1 Running 17 (107m ago) 62d
$ kubectl -n cattle-system get deploy
NAME READY UP-TO-DATE AVAILABLE AGE
rancher 1/1 1 1 62d
rancher-webhook 1/1 1 1 62d
$ kubectl -n cattle-system rollout restart rancher
$ kubectl -n cattle-system get pods
NAME READY STATUS RESTARTS AGE
rancher-7fbffb59cc-nn2bf 1/1 Running 0 80s
rancher-webhook-644bcb4d99-mgfx9 1/1 Running 17 (116m ago) 62d正常情況下應該到這裡就會在看到叢集會在 Web UI 裡面被移除了
但是做到這邊發現叢集並未被移除
Clean the orphaned cluster objects from the deleted cluster namespaces.
# 先找到孤兒叢集的 Namespace
$ kubectl get ns -A |grep "c-"
c-m-bzrsxlnb Active 62d
c-m-fb4btkbk Active 42d
# 列出該 Namespace 所有的物件
$ kubectl api-resources --verbs=list --namespaced -o name | xargs -n 1 kubectl get --show-kind --ignore-not-found -n c-m-fb4btkbk
NAME DATA AGE
configmap/kube-root-ca.crt 1 62d
configmap/provisioning-log 0 62d
NAME SECRETS AGE
serviceaccount/default 0 62d
NAME AGE
clusterregistrationtoken.management.cattle.io/default-token 62d
clusterregistrationtoken.management.cattle.io/system 62d
NAME AGE
clusterroletemplatebinding.management.cattle.io/c-m-bzrsxlnb-fleet-default-owner 62d
clusterroletemplatebinding.management.cattle.io/creator-cluster-owner 62d
clusterroletemplatebinding.management.cattle.io/u-hmec2q3jq3-admin 62d
NAME AGE
node.management.cattle.io/machine-2nttf 62d
NAME AGE
project.management.cattle.io/p-47xvp 62d
project.management.cattle.io/p-fhxkw 62d
NAME ROLE AGE
rolebinding.rbac.authorization.k8s.io/c-m-bzrsxlnb-fleet-default-owner-cluster-owner Role/cluster-owner 62d
rolebinding.rbac.authorization.k8s.io/creator-cluster-owner-cluster-owner Role/cluster-owner 62d
rolebinding.rbac.authorization.k8s.io/p-47xvp-creator-project-owner-project-owner Role/project-owner 62d
rolebinding.rbac.authorization.k8s.io/p-fhxkw-creator-project-owner-project-owner Role/project-owner 62d
rolebinding.rbac.authorization.k8s.io/rb-j23o5h2gdc Role/p-fhxkw-projectowner 62d
rolebinding.rbac.authorization.k8s.io/rb-nftlrlswfm Role/p-47xvp-projectowner 62d
rolebinding.rbac.authorization.k8s.io/u-hmec2q3jq3-admin-cluster-owner Role/cluster-owner 62d
NAME CREATED AT
role.rbac.authorization.k8s.io/cluster-owner 2024-04-03T03:50:44Z
role.rbac.authorization.k8s.io/p-47xvp-projectowner 2024-04-03T03:50:44Z
role.rbac.authorization.k8s.io/p-fhxkw-projectowner 2024-04-03T03:50:44Z
role.rbac.authorization.k8s.io/project-owner 2024-04-03T03:50:44Z
# 刪除在 Namespace 中的所有物件
$ kubectl -n c-m-fb4btkbk delete "$(kubectl api-resources --namespaced=true --verbs=delete -o name | tr "\n" "," | sed -e 's/,$//')" --all
configmap "kube-root-ca.crt" deleted
configmap "provisioning-log" deleted
serviceaccount "default" deleted
clusterregistrationtoken.management.cattle.io "default-token" deleted
clusterregistrationtoken.management.cattle.io "system" deleted
clusterroletemplatebinding.management.cattle.io "c-m-fb4btkbk-fleet-default-owner" deleted
clusterroletemplatebinding.management.cattle.io "creator-cluster-owner" deleted
node.management.cattle.io "machine-hj69z" deleted
project.management.cattle.io "p-lqsv2" deleted
rolebinding.rbac.authorization.k8s.io "p-42nc9-creator-project-owner-project-owner" deleted
rolebinding.rbac.authorization.k8s.io "p-lqsv2-creator-project-owner-project-owner" deleted
rolebinding.rbac.authorization.k8s.io "rb-utcrchdkew" deleted
rolebinding.rbac.authorization.k8s.io "rb-wxcfmlrysb" deleted
role.rbac.authorization.k8s.io "cluster-owner" deleted
role.rbac.authorization.k8s.io "p-42nc9-projectowner" deleted
role.rbac.authorization.k8s.io "project-owner" deleted
Error from server (BadRequest): admission webhook "rancher.cattle.io.projects.management.cattle.io" denied the request: System Project cannot be deleted
# 再次列出該 Namespace 所有的物件
$ kubectl api-resources --verbs=list --namespaced -o name | xargs -n 1 kubectl get --show-kind --ignore-not-found -n c-m-fb4btkbk
NAME DATA AGE
configmap/kube-root-ca.crt 1 16s
NAME SECRETS AGE
serviceaccount/default 0 16s
NAME AGE
project.management.cattle.io/p-42nc9 42d
# 刪除 namespace (刪不掉就按 Ctrl+C 終止)
$ kubectl delete ns c-m-fb4btkbk
namespace "c-m-fb4btkbk" deleted
# 修改 namespace finalizers 為空值
$ kubectl get namespace "c-m-fb4btkbk" -o json \
> | tr -d "\n" | sed "s/\"finalizers\": \[[^]]\+\]/\"finalizers\": []/" \
> | kubectl replace --raw /api/v1/namespaces/c-m-fb4btkbk/finalize -f -
...以上省略
{"type":"NamespaceFinalizersRemaining","status":"False","lastTransitionTime":"2024-06-04T10:32:48Z","reason":"ContentHasNoFinalizers","message":"All content-preserving finalizers finished"}]}}
$ kubectl get ns -A |grep "c-"
$ kubectl -n cattle-system rollout restart deploy rancher
deployment.apps/rancher restarted
$ kubectl -n cattle-system get pods -w
NAME READY STATUS RESTARTS AGE
rancher-7fbffb59cc-nn2bf 1/1 Running 0 33m
rancher-8b84f58fb-mw87h 0/1 Running 0 10s
rancher-webhook-644bcb4d99-mgfx9 1/1 Running 17 (148m ago) 62d
rancher-8b84f58fb-mw87h 1/1 Running 0 31s
rancher-7fbffb59cc-nn2bf 1/1 Terminating 0 33m
rancher-7fbffb59cc-nn2bf 1/1 Terminating 0 33m
rancher-7fbffb59cc-nn2bf 0/1 Terminating 0 33m
rancher-7fbffb59cc-nn2bf 0/1 Terminating 0 33m
rancher-7fbffb59cc-nn2bf 0/1 Terminating 0 33m
rancher-7fbffb59cc-nn2bf 0/1 Terminating 0 33m
$ kubectl -n cattle-system get pods
NAME READY STATUS RESTARTS AGE
rancher-8b84f58fb-mw87h 1/1 Running 0 86s
rancher-webhook-644bcb4d99-mgfx9 1/1 Running 17 (149m ago) 62d最後要在 UI 上把 machine 和 Cluster 的 finalizers 也改為空值，就能完全移除 Cluster 囉~