antony@notes:~/kubernetes$ cat "Install-K3s-in-Containers-without-k3d.md"

Install K3s in Containers without k3d

2024-10-13· kubernetes

Install K3s in Containers without k3d

1. 環境準備

1.1. 準備 Podman Network

$ sudo podman network create --subnet 172.22.1.0/24 --gateway 172.22.1.254 k3s-network

$ sudo podman network ls
NETWORK ID    NAME         DRIVER
78b8668c5acd  k3s-network  bridge
2f259bab93aa  podman       bridge

1.2. 設定必要核心參數

$ cat <<EOF | sudo tee -a /etc/sysctl.conf
net.bridge.bridge-nf-call-iptables = 1
fs.inotify.max_user_instances=8192
fs.inotify.max_user_watches=524288
net.netfilter.nf_conntrack_max = 131072
kernel.dmesg_restrict = 0
EOF

$ sudo reboot

1.3. 建立工作目錄

$ mkdir -p "$HOME"/k3s/config/c31 &&
cd "$HOME"/k3s/config/c31

1.4. 產生設定 containerd 的基礎模板

$ cat <<EOF > "$HOME"/k3s/config/c31/config.toml.tmpl
{{ template "base" . }}

# RuntimeClass Handler
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runsc]
  runtime_type = "io.containerd.runsc.v1"
  [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runsc.options]
    BinaryName = "/usr/bin/runsc"
EOF

1.5. 產生設定私有 Container images Registry 設定檔

$ cat <<EOF > "$HOME"/k3s/config/c31/registries.yaml
mirrors:
  dkreg.kube-system:5000:
    endpoint:
      - "http://dkreg.kube-system:5000"
configs:
  "dkreg.kube-system:5000":
    auth:
      username: bigred
      password: bigred
EOF

1.6. 產生 runsc runtimeclass 設定檔

$ cat <<EOF > "$HOME"/k3s/config/c31/runtimeclass.yaml
apiVersion: node.k8s.io/v1
kind: RuntimeClass
metadata:
  name: gvisor
handler: runsc
EOF

1.7. 設計客製化 K3s image

$ nano "$HOME"/k3s/config/c31/Containerfile

內容如下 :

# 由於官方提供的 K3s Container images 中缺少套件管理命令，
# 因此我們需要重新製作 Container image，並添加 apk 套件管理工具。
# 同時，我們也將設定好 Containerd，以確保能夠下載來自 dkreg 的 Container images。
ARG K3S_TAG="v1.31.1-k3s1"
ARG ALPINE_TAG="3.20"

FROM docker.io/rancher/k3s:$K3S_TAG as k3s
FROM docker.io/library/alpine:$ALPINE_TAG as collect

# 安裝 OCI runtime spec (包含 crun 和 runsc)
RUN apk -U add crun bash nftables mount && \
    ARCH=$(uname -m); URL=https://storage.googleapis.com/gvisor/releases/release/latest/${ARCH} && \
    wget ${URL}/runsc ${URL}/runsc.sha512 ${URL}/containerd-shim-runsc-v1 ${URL}/containerd-shim-runsc-v1.sha512 && \
    rm -f *.sha512 && \
    chmod a+rx runsc containerd-shim-runsc-v1; mv runsc containerd-shim-runsc-v1 /bin

FROM scratch
COPY --from=k3s / /
RUN mkdir /usr/bin && \
    mkdir /usr/lib

# Setup musl libc
## musl is an implementation of the C standard library built on top of the Linux system call API
COPY --from=collect /lib/ld-musl-x86_64.so.1 /lib

# Setup Bash Shell
COPY --from=collect /bin/bash /bin
COPY --from=collect /usr/lib/libreadline.so.8 /usr/lib
COPY --from=collect /usr/lib/libncursesw.so.6 /usr/lib

# Setup runsc
COPY --from=collect /bin/runsc /bin
COPY --from=collect /bin/containerd-shim-runsc-v1 /bin

# Setup crun
COPY --from=collect /usr/bin/crun /usr/bin
COPY --from=collect /usr/lib/libseccomp.so.2 /usr/lib
COPY --from=collect /usr/lib/libcap.so.2 /usr/lib
COPY --from=collect /usr/lib/libyajl.so.2 /usr/lib

# Setup nft
COPY --from=collect /usr/sbin/nft /usr/bin
COPY --from=collect /usr/lib/libnftables.so.1 /usr/lib
COPY --from=collect /usr/lib/libreadline.so.8 /usr/lib
COPY --from=collect /usr/lib/libmnl.so.0 /usr/lib
COPY --from=collect /usr/lib/libnftnl.so.11 /usr/lib
COPY --from=collect /usr/lib/libjansson.so.4 /usr/lib
COPY --from=collect /usr/lib/libgmp.so.10 /usr/lib

# Setup mount
COPY --from=collect /bin/mount /bin
COPY --from=collect /lib/libmount.so.1 /lib
COPY --from=collect /lib/libblkid.so.1 /lib
COPY --from=collect /usr/lib/libeconf.so.0 /usr/lib

# 設定 containerd 的基礎模板
## 將 runSC OCI runtimes 的相關設定新增進 containerd
COPY config.toml.tmpl /var/lib/rancher/k3s/agent/etc/containerd/
## 設定私有 Container images Registry，確保可以拉取來自 dkreg 的 Container images
## 參考文章 : https://docs.k3s.io/installation/private-registry#configs
COPY registries.yaml /etc/rancher/k3s/

# 設定 runsc RuntimeClass
COPY runtimeclass.yaml /var/lib/rancher/k3s/server/manifests/

VOLUME /var/lib/kubelet
VOLUME /var/lib/rancher/k3s
VOLUME /var/lib/cni
VOLUME /var/log

ENV PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/bin/aux"
ENV CRI_CONFIG_FILE="/var/lib/rancher/k3s/agent/etc/crictl.yaml"

ENTRYPOINT ["/bin/k3s"]
CMD ["agent"]

1.8. 建立客製化 K3s image

$ K3S_TAG=v1.31.1-k3s1

$ sudo podman build --squash-all --no-cache \
  --build-arg K3S_TAG="$K3S_TAG" \
  -t k3s-add-runtime:"$K3S_TAG" \
  "$HOME"/k3s/config/c${K3S_TAG:3:2}/

2. 安裝 K3s

2.1. 建立 server node

# 2.1. 建立 podman volume
$ sudo podman volume create --label server=cni && \
sudo podman volume create --label server=kubelet && \
sudo podman volume create --label server=k3s && \
sudo podman volume create --label server=log

$ server_cni=$(sudo podman volume ls --format "{{.Name}}" --filter "label=server=cni" ) && \
server_kubelet=$(sudo podman volume ls --format "{{.Name}}" --filter "label=server=kubelet") && \
server_k3s=$(sudo podman volume ls --format "{{.Name}}" --filter "label=server=k3s") && \
server_log=$(sudo podman volume ls --format "{{.Name}}" --filter "label=server=log")

# 2.2. 建立 server node container
$ sudo podman run -d \
  --name k3s-c31-server-0 \
  --privileged \
  -e CRI_CONFIG_FILE=/var/lib/rancher/k3s/agent/etc/crictl.yaml \
  -e HOSTNAME=k3d-c31-server-0 \
  -e K3S_KUBECONFIG_OUTPUT=/output/kubeconfig.yaml \
  -e K3S_TOKEN=c31Token \
  -e container=podman \
  --network k3s-network \
  --ip=172.22.1.11 \
  --hostname k3s-c31-server-0 \
  --ulimit nproc=131072:131072 \
  --ulimit nofile=26677:26677 \
  --volume /home/bigred/k3d/wulin:/opt/kadm:rw,rprivate,rbind \
  --volume ${server_cni}:/var/lib/cni:rprivate,rw,nodev,exec,nosuid,rbind \
  --volume ${server_kubelet}:/var/lib/kubelet:rprivate,rw,nodev,exec,nosuid,rbind \
  --volume ${server_k3s}:/var/lib/rancher/k3s:rprivate,rw,nodev,exec,nosuid,rbind \
  --volume ${server_log}:/var/log:rprivate,rw,nodev,exec,nosuid,rbind \
  -p 80:80/tcp \
  -p 443:443/tcp \
  -p 5000:5000/tcp \
  -p 5900-5905:5900-5905/tcp \
  -p 6080:6080/tcp \
  -p 8080:8080/tcp \
  -p 9000:9000/tcp \
  -p 22001-22002:22001-22002/tcp \
  -p 22100-22102:22100-22102/tcp \
  -p 51820:51820/tcp \
  -p 6443:30001/tcp \
  localhost/k3s-add-runtime:v1.31.1-k3s1 \
  server --cluster-domain=k1.org \
  --bind-address=172.22.1.11 \
  --cluster-cidr=10.244.0.0/16 \
  --service-cidr=10.98.0.0/16 \
  --kubelet-arg=container-log-max-files=3 \
  --kubelet-arg=container-log-max-size=10Mi \
  --egress-selector-mode=disabled \
  --disable=traefik \
  --disable=servicelb \
  --node-label app=taroko \
  --tls-san k1.org \
  --tls-san 172.22.1.11

2.2. 檢查 K3s server node 狀態

$ sudo podman exec k3s-c31-server-0 kubectl get nodes

執行結果 :

NAME               STATUS   ROLES                  AGE   VERSION
k3s-c31-server-0   Ready    control-plane,master   7s    v1.31.1+k3s1

2.3. 檢查 K3s 所有 pods 狀態

$ sudo podman exec k3s-c31-server-0 kubectl get pods -A

執行結果 :

NAMESPACE     NAME                                      READY   STATUS    RESTARTS   AGE
kube-system   coredns-56f6fc8fd7-t9hlb                  1/1     Running   0          99s
kube-system   local-path-provisioner-846b9dcb6c-xddbs   1/1     Running   0          99s
kube-system   metrics-server-5985cbc9d7-889p8           1/1     Running   0          99s

2.4. 建立第一台 agent node

# 3.1. 建立 podman volume
$ sudo podman volume create --label agent0=cni && \
sudo podman volume create --label agent0=kubelet && \
sudo podman volume create --label agent0=k3s && \
sudo podman volume create --label agent0=log

$ agent_0_cni=$(sudo podman volume ls --format "{{.Name}}" --filter "label=agent0=cni") && \
agent_0_kubelet=$(sudo podman volume ls --format "{{.Name}}" --filter "label=agent0=kubelet") && \
agent_0_k3s=$(sudo podman volume ls --format "{{.Name}}" --filter "label=agent0=k3s") && \
agent_0_log=$(sudo podman volume ls --format "{{.Name}}" --filter "label=agent0=log") && \
server_ip=$(sudo podman exec k3s-c31-server-0 ip a s eth0 | grep "inet " | awk '{print $2}')

# 3.2. 建立 agent node container
$ sudo podman run -d\
  --name k3s-c31-agent-0 \
  --hostname k3s-c31-agent-0 \
  --privileged \
  --env K3S_TOKEN=c31Token \
  --env K3S_URL=https://${server_ip%/*}:6443 \
  --env K3S_KUBECONFIG_OUTPUT=/output/kubeconfig.yaml \
  --ulimit nproc=131072:131072 \
  --ulimit nofile=26677:26677 \
  --network k3s-network \
  -p 22011:22011 \
  -p 33891:33891 \
  -v ${agent_0_cni}:/var/lib/cni:rw,rprivate,nodev,exec,nosuid,rbind \
  -v ${agent_0_kubelet}:/var/lib/kubelet:rw,rprivate,nodev,exec,nosuid,rbind \
  -v ${agent_0_k3s}:/var/lib/rancher/k3s:rw,rprivate,nodev,exec,nosuid,rbind \
  -v ${agent_0_log}:/var/log:rw,rprivate,nodev,exec,nosuid,rbind \
  localhost/k3s-add-runtime:v1.31.1-k3s1 \
  agent

2.5. 檢查 K3s agent node 狀態

$ sudo podman exec k3s-c31-server-0 kubectl get nodes

執行結果 :

NAME               STATUS   ROLES                  AGE     VERSION
k3s-c31-agent-0    Ready    <none>                 3s      v1.31.1+k3s1
k3s-c31-server-0   Ready    control-plane,master   2m51s   v1.31.1+k3s1

2.6. 檢查 K3s 所有 pods 狀態

$ sudo podman exec k3s-c31-server-0 kubectl get pods -A

執行結果 :

NAMESPACE     NAME                                      READY   STATUS    RESTARTS   AGE
kube-system   coredns-56f6fc8fd7-t9hlb                  1/1     Running   0          3m39s
kube-system   local-path-provisioner-846b9dcb6c-xddbs   1/1     Running   0          3m39s
kube-system   metrics-server-5985cbc9d7-889p8           1/1     Running   0          3m39s

2.7. 建立第二台 agent node

# 3.1. 建立 podman volume
$ sudo podman volume create --label agent1=cni && \
sudo podman volume create --label agent1=kubelet && \
sudo podman volume create --label agent1=k3s && \
sudo podman volume create --label agent1=log

$ agent_2_cni=$(sudo podman volume ls --format "{{.Name}}" --filter "label=agent1=cni") && \
agent_2_kubelet=$(sudo podman volume ls --format "{{.Name}}" --filter "label=agent1=kubelet") && \
agent_2_k3s=$(sudo podman volume ls --format "{{.Name}}" --filter "label=agent1=k3s") && \
agent_2_log=$(sudo podman volume ls --format "{{.Name}}" --filter "label=agent1=log") && \
server_ip=$(sudo podman exec k3s-c31-server-0 ip a s eth0 | grep "inet " | awk '{print $2}')

# 3.2. 建立 server node container
$ sudo podman run -d\
  --name k3s-c31-agent-1 \
  --hostname k3s-c31-agent-1 \
  --privileged \
  --env K3S_TOKEN=c31Token \
  --env K3S_URL=https://${server_ip%/*}:6443 \
  --env K3S_KUBECONFIG_OUTPUT=/output/kubeconfig.yaml \
  --ulimit nproc=131072:131072 \
  --ulimit nofile=26677:26677 \
  --network k3s-network \
  -p 22012:22012 \
  -p 33892:33892 \
  -v ${agent_2_cni}:/var/lib/cni:rw,rprivate,nodev,exec,nosuid,rbind \
  -v ${agent_2_kubelet}:/var/lib/kubelet:rw,rprivate,nodev,exec,nosuid,rbind \
  -v ${agent_2_k3s}:/var/lib/rancher/k3s:rw,rprivate,nodev,exec,nosuid,rbind \
  -v ${agent_2_log}:/var/log:rw,rprivate,nodev,exec,nosuid,rbind \
  localhost/k3s-add-runtime:v1.31.1-k3s1 \
  agent

2.8. 檢查 K3s agent node 狀態

$ sudo podman exec k3s-c31-server-0 kubectl get nodes

執行結果 :

NAME               STATUS   ROLES                  AGE     VERSION
k3s-c31-agent-0    Ready    <none>                 65s     v1.31.1+k3s1
k3s-c31-agent-1    Ready    <none>                 18s     v1.31.1+k3s1
k3s-c31-server-0   Ready    control-plane,master   3m53s   v1.31.1+k3s1

2.9. 檢查 K3s 所有 pods 狀態

$ sudo podman exec k3s-c31-server-0 kubectl get pods -A

執行結果 :

NAMESPACE     NAME                                      READY   STATUS    RESTARTS   AGE
kube-system   coredns-56f6fc8fd7-t9hlb                  1/1     Running   0          4m3s
kube-system   local-path-provisioner-846b9dcb6c-xddbs   1/1     Running   0          4m3s
kube-system   metrics-server-5985cbc9d7-889p8           1/1     Running   0          4m3s

2.10. 設定 KubeConfig

$ mkdir -p "$HOME"/.kube && \
sudo podman cp k3s-c31-server-0:/output/kubeconfig.yaml "$HOME"/.kube/config && \
sudo chown -R $(id -u):$(id -g) "$HOME"/.kube

2.11. 確認可直接執行 kubectl

$ kubectl get nodes -o wide

安裝 kubectl 命令

$ curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" && \
chmod +x kubectl; sudo mv kubectl /usr/local/bin/

執行結果 :

NAME               STATUS   ROLES                  AGE     VERSION        INTERNAL-IP   EXTERNAL-IP   OS-IMAGE
  KERNEL-VERSION   CONTAINER-RUNTIME
k3s-c31-agent-0    Ready    <none>                 2m48s   v1.31.1+k3s1   172.22.1.4    <none>        K3s v1.31.1+k3s1   6.6.50-0-lts     containerd://1.7.21-k3s2
k3s-c31-agent-1    Ready    <none>                 2m30s   v1.31.1+k3s1   172.22.1.5    <none>        K3s v1.31.1+k3s1   6.6.50-0-lts     containerd://1.7.21-k3s2
k3s-c31-server-0   Ready    control-plane,master   4m24s   v1.31.1+k3s1   172.22.1.11   <none>        K3s v1.31.1+k3s1   6.6.50-0-lts     containerd://1.7.21-k3s2

3. 完全清除環境

$ sudo podman rm -f k3s-c31-server-0 k3s-c31-agent-0 k3s-c31-agent-1 \
k3s-working-container alpine-working-container && \
volume=$(for i in server agent0 agent1; do \
  sudo podman volume ls --format "{{.Name}}" --filter label=${i}; \
done) && \
sudo podman volume rm $volume && \
sudo podman rmi localhost/k3s-add-runtime:v1.31.1-k3s1 \
docker.io/rancher/k3s:v1.31.1-k3s1 && \
rm -rf "$HOME"/.kube "$HOME"/k3s

Enable kubectl bash completion on Alpine Linux Debugging with an ephemeral debug container