antony@notes:~/kubernetes$ cat "Install-a-kubeadm-Kubernetes-cluster-with-Cilium-disabling-kube-proxy.md"
Install a kubeadm Kubernetes cluster with Cilium, disabling kube-proxy
Install a kubeadm Kubernetes cluster with Cilium, disabling kube-proxy
Ubuntu 24.04
1. 每一台節點都要做環境準備
$ sudo nano /etc/netplan/50-cloud-init.yaml
network:
version: 2
ethernets:
ens18:
dhcp4: false
addresses:
- 192.168.11.72/24
nameservers:
addresses:
- 192.168.11.11
routes:
- to: 0.0.0.0/0
via: 192.168.11.254
$ sudo nano /etc/systemd/timesyncd.conf
...
[Time]
NTP=192.168.11.11
...
$ sudo systemctl restart systemd-timesyncd
$ timedatectl show-timesync --all
$ sudo timedatectl set-timezone Asia/Taipei
$ timedatectl
Local time: Sun 2025-06-29 15:44:28 CST
Universal time: Sun 2025-06-29 07:44:28 UTC
RTC time: Sun 2025-06-29 07:44:28
Time zone: Asia/Taipei (CST, +0800)
System clock synchronized: yes
NTP service: active
RTC in local TZ: no
$ sudo swapoff -a
$ sudo sed -i '/swap/s/^/#/' /etc/fstab
$ cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
net.ipv4.ip_forward = 1
EOF
$ sudo sysctl --system
$ sudo mkdir -p /opt/cni/bin && \
curl -sL "$(curl -sL https://api.github.com/repos/containernetworking/plugins/releases/latest | \
jq -r '.assets[].browser_download_url' | grep 'linux-amd64.*.tgz$')" -o ~/cni-plugins.tgz && \
sudo tar xf ~/cni-plugins.tgz -C /opt/cni/bin
$ wget https://github.com/opencontainers/runc/releases/download/v1.3.0/runc.amd64 && \
sudo install -m 755 runc.amd64 /usr/local/sbin/runc
$ wget https://github.com/containerd/containerd/releases/download/v2.1.3/containerd-2.1.3-linux-amd64.tar.gz && \
sudo tar Cxzvf /usr/local containerd-2.1.3-linux-amd64.tar.gz && \
wget https://raw.githubusercontent.com/containerd/containerd/main/containerd.service && \
sudo mkdir -p /usr/local/lib/systemd/system && \
sudo mv containerd.service /usr/local/lib/systemd/system/containerd.service
$ sudo mkdir /etc/containerd
$ sudo containerd config default | sudo tee /etc/containerd/config.toml
$ sudo nano /etc/containerd/config.toml
......
[plugins.'io.containerd.cri.v1.runtime'.containerd.runtimes.runc.options]
SystemdCgroup = true #add
$ sudo systemctl daemon-reload
$ sudo systemctl enable --now containerd
$ sudo systemctl is-active containerd.service
active
$ sudo apt-get update; sudo apt-get upgrade -y
$ sudo apt-get install -y apt-transport-https ca-certificates curl gpg
$ K8S_VERSION='v1.33'
$ curl -fsSL https://pkgs.k8s.io/core:/stable:/${K8S_VERSION}/deb/Release.key | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg && \
echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/${K8S_VERSION}/deb/ /" | sudo tee /etc/apt/sources.list.d/kubernetes.list
# Worker node 可不用裝 kubectl
$ sudo apt-get update; sudo apt-get install -y kubelet kubeadm kubectl
$ sudo apt-mark hold kubelet kubeadm kubectl 裝 K8s
mkdir ~/k8s; cd ~/k8s
nano ~/k8s/init-config.yaml檔案內容 :
apiVersion: kubeadm.k8s.io/v1beta3
kind: InitConfiguration
localAPIEndpoint:
advertiseAddress: 192.168.11.71 # change from Master node IP
bindPort: 6443
nodeRegistration:
criSocket: unix:///run/containerd/containerd.sock # change from containerd Unix Socket
imagePullPolicy: IfNotPresent
name: c1 # change from Master node hsotname
taints: null
---
apiVersion: kubeadm.k8s.io/v1beta3
kind: ClusterConfiguration
kubernetesVersion: 1.33.2
controlPlaneEndpoint: 192.168.11.71:6443
apiServer:
timeoutForControlPlane: 4m0s
#certSANs:
# - 10.10.7.33 # 加入 kube-vip 的 VIP 作為 TLS SAN
certificatesDir: /etc/kubernetes/pki
clusterName: topgun # set your clusterName
controllerManager:
extraArgs:
bind-address: "0.0.0.0"
secure-port: "10257"
scheduler:
extraArgs:
bind-address: "0.0.0.0"
secure-port: "10259"
etcd:
local:
dataDir: /var/lib/etcd
extraArgs:
listen-metrics-urls: "http://0.0.0.0:2381"
dns: {}
imageRepository: registry.k8s.io
networking:
dnsDomain: cluster.local # DNS domain used by Kubernetes Services.
podSubnet: 10.244.0.0/16 # the subnet used by Pods.
serviceSubnet: 10.96.0.0/16 # subnet used by Kubernetes Services.
---
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
maxPods: 110
shutdownGracePeriod: 30s
shutdownGracePeriodCriticalPods: 10s
systemReserved:
memory: "1Gi"
kubeReserved:
memory: "2Gi"初始化 K8s
sudo kubeadm init --skip-phases=addon/kube-proxy --upload-certs --config=k8s/init-config.yamlJoin worker node
sudo kubeadm join 192.168.11.71:6443 --token 7pr0b4.79tdi8egs81l4i9g --discovery-token-ca-cert-hash sha256:c6bed66cc0c7f8616a4f920ccc62121b782c37475888a6cc9831ced22fde474f設定 kubeconfig
mkdir -p $HOME/.kube; sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config; sudo chown $(id -u):$(id -g) $HOME/.kube/config裝 Cilium
# Install the Cilium CLI
CILIUM_CLI_VERSION=$(curl -s https://raw.githubusercontent.com/cilium/cilium-cli/main/stable.txt)
CLI_ARCH=amd64
if [ "$(uname -m)" = "aarch64" ]; then CLI_ARCH=arm64; fi
curl -L --fail --remote-name-all https://github.com/cilium/cilium-cli/releases/download/${CILIUM_CLI_VERSION}/cilium-linux-${CLI_ARCH}.tar.gz{,.sha256sum}
sha256sum --check cilium-linux-${CLI_ARCH}.tar.gz.sha256sum
sudo tar xzvfC cilium-linux-${CLI_ARCH}.tar.gz /usr/local/bin
rm cilium-linux-${CLI_ARCH}.tar.gz{,.sha256sum}
API_SERVER_IP=192.168.11.71
API_SERVER_PORT=6443
curl -fsSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
helm repo add cilium https://helm.cilium.io/
helm install cilium cilium/cilium \
--namespace kube-system \
--set kubeProxyReplacement=true \
--set k8sServiceHost=${API_SERVER_IP} \
--set k8sServicePort=${API_SERVER_PORT} \
--set bpf.masquerade=true \
--set ipam.mode=kubernetes \
--set ipv4NativeRoutingCIDR=10.244.0.0/16 \
--set bpf.datapathMode=netkit
helm upgrade cilium cilium/cilium \
--namespace=kube-system \
--set kubeProxyReplacement=true \
--set k8sServiceHost=${API_SERVER_IP} \
--set k8sServicePort=${API_SERVER_PORT} \
--set hubble.ui.enabled=true \
--set hubble.relay.enabled=true \
--set hubble.ui.frontend.server.ipv6.enabled=false \
--set hubble.ui.service.type=NodePort \
--set bpf.datapathMode=netkit
helm upgrade cilium cilium/cilium --version 1.17.6 \
--namespace kube-system \
--reuse-values \
--set ingressController.enabled=true \
--set ingressController.loadbalancerMode=dedicated
kubectl -n kube-system rollout restart deployment/cilium-operator
kubectl -n kube-system rollout restart ds/ciliumValidate the Installation
$ kubectl -n kube-system get pods
NAME READY STATUS RESTARTS AGE
cilium-28kw8 1/1 Running 1 (117m ago) 124m
cilium-envoy-c5nxh 1/1 Running 1 (117m ago) 124m
cilium-envoy-phwpp 1/1 Running 0 117m
cilium-envoy-sb7c2 1/1 Running 0 117m
cilium-m5mvp 1/1 Running 0 116m
cilium-operator-c64699c6b-2kgr5 1/1 Running 0 117m
cilium-operator-c64699c6b-4sg5r 0/1 Completed 0 124m
cilium-operator-c64699c6b-ft9x8 0/1 Completed 0 124m
cilium-operator-c64699c6b-zflrv 1/1 Running 0 117m
cilium-r6x55 1/1 Running 0 117m
coredns-674b8bbfcf-bzk8x 1/1 Running 1 (117m ago) 131m
coredns-674b8bbfcf-tf5fh 1/1 Running 1 (117m ago) 131m
etcd-c1 1/1 Running 1 (117m ago) 131m
hubble-relay-67cfb6bcbd-hx9t8 1/1 Running 0 117m
hubble-ui-85bc4bfc56-57g8f 2/2 Running 0 96m
kube-apiserver-c1 1/1 Running 1 (117m ago) 131m
kube-controller-manager-c1 1/1 Running 1 (117m ago) 131m
kube-scheduler-c1 1/1 Running 1 (117m ago) 131m$ cilium status
/¯¯\
/¯¯\__/¯¯\ Cilium: OK
\__/¯¯\__/ Operator: OK
/¯¯\__/¯¯\ Envoy DaemonSet: OK
\__/¯¯\__/ Hubble Relay: OK
\__/ ClusterMesh: disabled
DaemonSet cilium Desired: 3, Ready: 3/3, Available: 3/3
DaemonSet cilium-envoy Desired: 3, Ready: 3/3, Available: 3/3
Deployment cilium-operator Desired: 2, Ready: 2/2, Available: 2/2
Deployment hubble-relay Desired: 1, Ready: 1/1, Available: 1/1
Deployment hubble-ui Desired: 1, Ready: 1/1, Available: 1/1
Containers: cilium Running: 3
cilium-envoy Running: 3
cilium-operator Running: 2, Succeeded: 2
clustermesh-apiserver
hubble-relay Running: 1
hubble-ui Running: 1
Cluster Pods: 8/8 managed by Cilium
Helm chart version: 1.18.0-dev
Image versions cilium quay.io/cilium/cilium-ci:latest: 3
cilium-envoy quay.io/cilium/cilium-envoy:v1.34.1-1750869463-42c7e8cf0f93ea19c9cb7e4d0ad2a339b3f81ad2@sha256:892cab92ffaf8499be90bd227bf07181e4b460ecd97750032ea7e91710a8acce: 3
cilium-operator quay.io/cilium/operator-generic-ci:latest: 4
hubble-relay quay.io/cilium/hubble-relay-ci:latest: 1
hubble-ui quay.io/cilium/hubble-ui-backend:v0.13.2@sha256:a034b7e98e6ea796ed26df8f4e71f83fc16465a19d166eff67a03b822c0bfa15: 1
hubble-ui quay.io/cilium/hubble-ui:v0.13.2@sha256:9e37c1296b802830834cc87342a9182ccbb71ffebb711971e849221bd9d59392: 1