antony@notes:~/misc$ cat "20260629.md"
20260629
20260629
# us-tkcdc-01
$ kubectl get nodes -o wide
NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME
tkdt-control-plane Ready control-plane 19h v1.34.3 172.22.16.1 <none> Debian GNU/Linux 12 (bookworm) 6.8.0-124-generic containerd://2.2.0
tkdt-master1 Ready <none> 19h v1.34.3 172.22.16.2 <none> Debian GNU/Linux 12 (bookworm) 6.8.0-124-generic containerd://2.2.0
tkdt-worker1 Ready <none> 19h v1.34.3 172.22.16.3 <none> Debian GNU/Linux 12 (bookworm) 6.8.0-124-generic containerd://2.2.0
tkdt-worker2 Ready <none> 19h v1.34.3 172.22.16.4 <none> Debian GNU/Linux 12 (bookworm) 6.8.0-124-generic containerd://2.2.0
tkdt-worker3 Ready <none> 19h v1.34.3 172.22.16.5 <none> Debian GNU/Linux 12 (bookworm) 6.8.0-124-generic containerd://2.2.0
$ kubectl describe nodes | grep -i podcidr
PodCIDR: 10.244.16.0/25
PodCIDRs: 10.244.16.0/25
PodCIDR: 10.244.16.128/25
PodCIDRs: 10.244.16.128/25
PodCIDR: 10.244.17.0/25
PodCIDRs: 10.244.17.0/25
PodCIDR: 10.244.17.128/25
PodCIDRs: 10.244.17.128/25
PodCIDR: 10.244.18.0/25
PodCIDRs: 10.244.18.0/25
$ kubectl -n kube-system get cm kubeadm-config -o yaml | grep serviceSubnet
serviceSubnet: 10.98.16.0/24
sudo route add -net 172.22.26.0 netmask 255.255.255.128 gw 192.168.11.102
sudo route add -net 10.244.17.0 netmask 255.255.255.128 gw 172.22.16.3
sudo route add -net 10.244.17.128 netmask 255.255.255.128 gw 172.22.16.4
sudo route add -net 10.244.18.0 netmask 255.255.255.128 gw 172.22.16.5
sudo route add -net 10.244.25.0 netmask 255.255.255.128 gw 192.168.11.102
sudo route add -net 10.244.25.128 netmask 255.255.255.128 gw 192.168.11.102
sudo route add -net 10.244.26.0 netmask 255.255.255.128 gw 192.168.11.102
sudo route add -net 10.98.16.0 netmask 255.255.255.0 gw 172.22.16.5
sudo route add -net 10.98.24.0 netmask 255.255.255.0 gw 192.168.11.102
kubectl apply -f - <<'EOF'
apiVersion: v1
kind: Service
metadata:
name: nginx-tkdt
labels:
app: nginx-tkdt
spec:
clusterIP: None
selector:
app: nginx-tkdt
ports:
- port: 80
name: web
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: nginx-tkdt
spec:
serviceName: nginx-tkdt
replicas: 2
selector:
matchLabels:
app: nginx-tkdt
template:
metadata:
labels:
app: nginx-tkdt
spec:
containers:
- name: nginx-tkdt
image: nginx
ports:
- containerPort: 80
name: web
EOF
$ kubectl get pods -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
nginx-tkdt-0 1/1 Running 0 36s 10.244.17.5 tkdt-worker1 <none> <none>
nginx-tkdt-1 1/1 Running 0 33s 10.244.18.6 tkdt-worker3 <none> <none>
$ kubectl get svc nginx-tkdt
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
nginx-tkdt ClusterIP None <none> 80/TCP 44s
$ curl -sI http://10.244.17.5 | head -n 1
HTTP/1.1 200 OK
$ curl -sI http://10.244.26.5 | head -n 1
HTTP/1.1 200 OK
$ dig @10.98.16.10 nginx-tkdt-0.nginx-tkdt.default.svc.tkdt.k8s +short
10.244.17.5
$ dig @10.98.16.10 nginx-tkdt-1.nginx-tkdt.default.svc.tkdt.k8s +short
10.244.18.6
$ dig @10.98.24.10 nginx-tkndt-0.nginx-tkndt.default.svc.tkndt.k8s +short
10.244.26.5
$ dig @10.98.24.10 nginx-tkndt-1.nginx-tkndt.default.svc.tkndt.k8s +short
10.244.25.5
$ kubectl -n kube-system edit cm coredns
transfer tkdt.k8s { # 其他人可以跟我用 AXFR 同步
to * # 來源 IP 問題，先開放給所有人
}
secondary tkndt.k8s { # 我向 tkndt 同步的 domain
transfer from 10.98.24.10 # tkndt 的 coredns svc ip
}
$ kubectl -n kube-system rollout restart deployment coredns
$ kubectl -n kube-system get pods -l k8s-app=kube-dns
NAME READY STATUS RESTARTS AGE
coredns-7d597fb798-5zx2n 1/1 Running 0 16s
coredns-7d597fb798-7xmg7 1/1 Running 0 16s
$ dig @10.98.16.10 nginx-tkndt-0.nginx-tkndt.default.svc.tkndt.k8s +short
10.244.26.5
$ dig @10.98.16.10 nginx-tkndt-1.nginx-tkndt.default.svc.tkndt.k8s +short
10.244.25.5
$ kubectl exec nginx-tkdt-0 -- curl -sI nginx-tkndt-0.nginx-tkndt.default.svc.tkndt.k8s | head -n 1
HTTP/1.1 200 OK
$ kubectl exec nginx-tkdt-0 -- curl -sI nginx-tkndt-1.nginx-tkndt.default.svc.tkndt.k8s | head -n 1
HTTP/1.1 200 OK
==========================================================================
# us-tkcdc-02
$ kubectl get nodes -o wide
NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME
tkndt-control-plane Ready control-plane 18h v1.34.3 172.22.26.1 <none> Debian GNU/Linux 12 (bookworm) 6.8.0-124-generic containerd://2.2.0
tkndt-master1 Ready <none> 18h v1.34.3 172.22.26.2 <none> Debian GNU/Linux 12 (bookworm) 6.8.0-124-generic containerd://2.2.0
tkndt-worker1 Ready <none> 18h v1.34.3 172.22.26.3 <none> Debian GNU/Linux 12 (bookworm) 6.8.0-124-generic containerd://2.2.0
tkndt-worker2 Ready <none> 18h v1.34.3 172.22.26.4 <none> Debian GNU/Linux 12 (bookworm) 6.8.0-124-generic containerd://2.2.0
tkndt-worker3 Ready <none> 18h v1.34.3 172.22.26.5 <none> Debian GNU/Linux 12 (bookworm) 6.8.0-124-generic containerd://2.2.0
$ kubectl describe nodes | grep -i podcidr
PodCIDR: 10.244.24.0/25
PodCIDRs: 10.244.24.0/25
PodCIDR: 10.244.24.128/25
PodCIDRs: 10.244.24.128/25
PodCIDR: 10.244.25.0/25
PodCIDRs: 10.244.25.0/25
PodCIDR: 10.244.25.128/25
PodCIDRs: 10.244.25.128/25
PodCIDR: 10.244.26.0/25
PodCIDRs: 10.244.26.0/25
$ kubectl -n kube-system get cm kubeadm-config -o yaml | grep serviceSubnet
serviceSubnet: 10.98.24.0/24
sudo route add -net 172.22.16.0 netmask 255.255.255.128 gw 192.168.11.101
sudo route add -net 10.244.25.0 netmask 255.255.255.128 gw 172.22.26.3
sudo route add -net 10.244.25.128 netmask 255.255.255.128 gw 172.22.26.4
sudo route add -net 10.244.26.0 netmask 255.255.255.128 gw 172.22.26.5
sudo route add -net 10.244.17.0 netmask 255.255.255.128 gw 192.168.11.101
sudo route add -net 10.244.17.128 netmask 255.255.255.128 gw 192.168.11.101
sudo route add -net 10.244.18.0 netmask 255.255.255.128 gw 192.168.11.101
sudo route add -net 10.98.16.0 netmask 255.255.255.0 gw 192.168.11.101
sudo route add -net 10.98.24.0 netmask 255.255.255.0 gw 172.22.26.5
kubectl apply -f - <<'EOF'
apiVersion: v1
kind: Service
metadata:
name: nginx-tkndt
labels:
app: nginx-tkndt
spec:
clusterIP: None
selector:
app: nginx-tkndt
ports:
- port: 80
name: web
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: nginx-tkndt
spec:
serviceName: nginx-tkndt
replicas: 2
selector:
matchLabels:
app: nginx-tkndt
template:
metadata:
labels:
app: nginx-tkndt
spec:
containers:
- name: nginx-tkndt
image: nginx
ports:
- containerPort: 80
name: web
EOF
$ kubectl get pods -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
nginx-tkndt-0 1/1 Running 0 29s 10.244.26.5 tkndt-worker3 <none> <none>
nginx-tkndt-1 1/1 Running 0 23s 10.244.25.5 tkndt-worker1 <none> <none>
$ kubectl get svc nginx-tkndt
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
nginx-tkndt ClusterIP None <none> 80/TCP 90s
$ curl -sI http://10.244.17.5 | head -n 1
HTTP/1.1 200 OK
$ curl -sI http://10.244.26.5 | head -n 1
HTTP/1.1 200 OK
$ dig @10.98.24.10 nginx-tkndt-0.nginx-tkndt.default.svc.tkndt.k8s +short
10.244.26.5
$ dig @10.98.24.10 nginx-tkndt-1.nginx-tkndt.default.svc.tkndt.k8s +short
10.244.25.5
$ dig @10.98.16.10 nginx-tkdt-0.nginx-tkdt.default.svc.tkdt.k8s +short
10.244.17.5
$ dig @10.98.16.10 nginx-tkdt-1.nginx-tkdt.default.svc.tkdt.k8s +short
10.244.18.6
$ kubectl -n kube-system edit cm coredns
transfer tkndt.k8s { # 其他人可以跟我用 AXFR 同步
to * # 來源 IP 問題，先開放給所有人
}
secondary tkdt.k8s { # 我向 tkndt 同步的 domain
transfer from 10.98.16.10 # tkndt 的 coredns svc ip
}
$ kubectl -n kube-system rollout restart deployment coredns
$ kubectl -n kube-system get pods -l k8s-app=kube-dns
NAME READY STATUS RESTARTS AGE
coredns-7446d9745d-5pjh7 1/1 Running 0 11s
coredns-7446d9745d-5pxtj 1/1 Running 0 11s
$ $ dig @10.98.24.10 nginx-tkdt-0.nginx-tkdt.default.svc.tkdt.k8s +short
10.244.17.5
$ dig @10.98.24.10 nginx-tkdt-1.nginx-tkdt.default.svc.tkdt.k8s +short
10.244.18.6
$ kubectl exec nginx-tkndt-0 -- curl -sI nginx-tkdt-0.nginx-tkdt.default.svc.tkdt.k8s | head -n 1
HTTP/1.1 200 OK
$ kubectl exec nginx-tkndt-0 -- curl -sI nginx-tkdt-1.nginx-tkdt.default.svc.tkdt.k8s | head -n 1
HTTP/1.1 200 OK