antony@notes:~/rancher$ cat "Enable-Rancher-vcluster-on-Airgap-Harvester.md"

Enable Rancher vcluster on Airgap Harvester

2024-07-05· rancher

Enable Rancher vcluster on Airgap Harvester

1. Prepare Container Images and Helm Charts

# 在可上網的機器進行環境準備
# 1. 建立工作目錄
$ mkdir -p ~/rancher_vcluster/charts && \
  cd ~/rancher_vcluster

# 2. 安裝 Helm
$ curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash

# 3. 準備 Vcluster 的 Helm Charts
$ helm repo add rancher-vcluster https://charts.loft.sh
$ helm fetch rancher-vcluster/vcluster --version 0.19.0

# 4. 準備 Cert-manager 的 Helm Charts
$ helm repo add jetstack https://charts.jetstack.io
$ helm fetch jetstack/cert-manager --version v1.5.1

# 5. 準備 Rancher 的 Helm Charts
$ helm repo add rancher https://releases.rancher.com/server-charts/stable/
$ helm fetch rancher/rancher --version v2.8.2

$ mv vcluster-0.19.0.tgz rancher-2.8.2.tgz cert-manager-v1.5.1.tgz ~/rancher_vcluster/charts

# 6. 撰寫 Container Images 清單
$ echo 'docker.io/coredns/coredns:1.10.1
ghcr.io/loft-sh/vcluster:0.19.0
quay.io/jetstack/cert-manager-cainjector:v1.5.1
quay.io/jetstack/cert-manager-controller:v1.5.1
quay.io/jetstack/cert-manager-webhook:v1.5.1
quay.io/jetstack/cert-manager-ctl:v1.5.1
docker.io/rancher/fleet:v0.9.0
docker.io/rancher/gitjob:v0.1.96
docker.io/rancher/k3s:v1.27.10-k3s2
docker.io/rancher/klipper-helm:v0.8.2-build20230815
docker.io/rancher/mirrored-cluster-api-controller:v1.4.4
docker.io/rancher/rancher-webhook:v0.4.2
docker.io/rancher/rancher:v2.8.2
docker.io/rancher/rancher-agent:v2.8.2
docker.io/rancher/shell:v0.1.22
docker.io/chartmuseum/chartmuseum:latest' > images-list.txt

## 7. 安裝 Podman
$ sudo zypper -n in podman

# 8. Pull Container images
$ for i in $(cat images-list.txt); do sudo podman pull $i; done

# 9. 將所有 Container images 存成 Tar 檔
$ sudo podman save -m $(cat images-list.txt | tr -s "\n" " ") > rancher-vcluster_v2.8.2.tar

# 10. 檢視當前目錄結構
$ tree --du -h .
.
├── [186K]  charts
│   ├── [153K]  cert-manager-v1.5.1.tgz
│   ├── [ 15K]  rancher-2.8.2.tgz
│   └── [ 18K]  vcluster-0.19.0.tgz
├── [ 601]  images-list.txt
└── [4.7G]  rancher-vcluster_v2.8.2.tar

 4.7G used in 1 directory, 5 files
 
# 11. 壓成一包壓縮檔
$ cd; tar -czvf rancher-vcluster-airgap.tar.gz ~/rancher_vcluster/

$ ls -lh ~/rancher-vcluster-airgap.tar.gz
-rw-r--r-- 1 rancher users 2.0G Jul  5 20:18 rancher-vcluster-airgap.tar.gz

2. Installing rancher-vcluster

# 1. 切換成 root
$ sudo su - root

# 2. 解壓縮 rancher-vcluster-airgap.tar.gz
$ tar -zxvf /PATH/TO/rancher-vcluster-airgap.tar.gz && \
  cd rancher_vcluster/

# 3. 匯入 Container images (每一台 Harvester Node 都要 Import)
$ ctr -n k8s.io images import rancher-vcluster_v2.8.2.tar

# 4. 為建立 chartmuseum Container 建立永存目錄
$ mkdir -m 755 -p /opt/charts && \
  cd /opt/charts

# 5. 將各 charts 放入
$ cp /root/rancher_vcluster/charts/* /opt/charts/

# 6. 修改權限讓其他人可讀取
$ chmod 644 /opt/charts/*

# 7. 建立並啟動 chartmuseum
$ nerdctl -n k8s.io run -d --name chartmuseum -p 8080:8080 \
  -e STORAGE=local \
  -e STORAGE_LOCAL_ROOTDIR=/charts \
  -v /opt/charts:/charts \
  chartmuseum/chartmuseum:latest

# 8. 檢查 chartmuseum 狀態
$ nerdctl -n k8s.io ps -a --filter name=chartmuseum
CONTAINER ID    IMAGE                                       COMMAND           CREATED           STATUS    PORTS                     NAMES
efee2f9023dd    docker.io/chartmuseum/chartmuseum:latest    "/chartmuseum"    20 seconds ago    Up        0.0.0.0:8080->8080/tcp    chartmuseum

# 9. TEST
$ curl localhost:8080/api/charts | jq
{
  "cert-manager": [
    {
      "name": "cert-manager",
      "home": "https://github.com/jetstack/cert-manager",
      "sources": [
        "https://github.com/jetstack/cert-manager"
      ],
      "version": "v1.5.1",
      "description": "A Helm chart for cert-manager",
      "keywords": [
        "cert-manager",
        "kube-lego",
        "letsencrypt",
        "tls"
      ],
      "maintainers": [
        {
          "name": "cert-manager-maintainers",
          "email": "cert-manager-maintainers@googlegroups.com"
        }
      ],
      "icon": "https://raw.githubusercontent.com/jetstack/cert-manager/master/logo/logo.png",
      "apiVersion": "v1",
      "appVersion": "v1.5.1",
      "annotations": {
        "artifacthub.io/prerelease": "false"
      },
      "urls": [
        "charts/cert-manager-v1.5.1.tgz"
      ],
      "created": "2024-07-05T10:50:03.202872073Z",
      "digest": "b152ccd3cf5bf03b602624110a661b62b188ab3cfee82fdaf269eb4a04770e1a"
    }
  ],
  "rancher": [
    {
      "name": "rancher",
      "home": "https://rancher.com",
      "sources": [
        "https://github.com/rancher/rancher",
        "https://github.com/rancher/server-chart"
      ],
      "version": "2.8.2",
      "description": "Install Rancher Server to manage Kubernetes clusters across providers.",
      "keywords": [
        "rancher"
      ],
      "maintainers": [
        {
          "name": "Rancher Labs",
          "email": "charts@rancher.com"
        }
      ],
      "icon": "https://github.com/rancher/ui/blob/master/public/assets/images/logos/welcome-cow.svg",
      "apiVersion": "v2",
      "appVersion": "v2.8.2",
      "kubeVersion": "< 1.28.0-0",
      "urls": [
        "charts/rancher-2.8.2.tgz"
      ],
      "created": "2024-07-05T10:50:03.202872073Z",
      "digest": "8c9da08c0e38c5e89dc3ae64296c2fdca0045cc6e4bc44919e196d8cfe748f45"
    }
  ],
  "vcluster": [
    {
      "name": "vcluster",
      "home": "https://vcluster.com",
      "sources": [
        "https://github.com/loft-sh/vcluster"
      ],
      "version": "0.19.0",
      "description": "vcluster - Virtual Kubernetes Clusters",
      "keywords": [
        "developer",
        "development",
        "sharing",
        "share",
        "multi-tenancy",
        "tenancy",
        "cluster",
        "space",
        "namespace",
        "vcluster",
        "vclusters"
      ],
      "maintainers": [
        {
          "name": "Loft Labs, Inc.",
          "email": "info@loft.sh",
          "url": "https://twitter.com/loft_sh"
        }
      ],
      "icon": "https://static.loft.sh/branding/logos/vcluster/vertical/vcluster_vertical.svg",
      "apiVersion": "v2",
      "appVersion": "0.19.0",
      "type": "application",
      "urls": [
        "charts/vcluster-0.19.0.tgz"
      ],
      "created": "2024-07-05T09:53:38.89886262Z",
      "digest": "37b1b43bbac942dd303cea991dcb6e992338329aef62df0a9cf4dbec870bee57"
    }
  ]
}

# 10. 編輯 rancher-vcluster 設定檔
$ echo 'apiVersion: v1
kind: Namespace
metadata:
  name: rancher-vcluster
---
apiVersion: harvesterhci.io/v1beta1
kind: Addon
metadata:
  name: rancher-vcluster
  namespace: rancher-vcluster
  labels:
    addon.harvesterhci.io/experimental: "true"
spec:
  enabled: false
  repo: https://charts.loft.sh
  version: "v0.19.0"
  chart: vcluster
  valuesContent: |-
    hostname: ""
    rancherVersion: "v2.8.2"
    bootstrapPassword: ""
    vcluster:
      image: rancher/k3s:v1.27.10-k3s2
    sync:
      ingresses:
        enabled: "true"
    init:
      manifestsTemplate: |-
        apiVersion: v1
        kind: Namespace
        metadata:
          name: cattle-system
        ---
        apiVersion: v1
        kind: Namespace
        metadata:
          name: cert-manager
          labels:
            certmanager.k8s.io/disable-validation: "true"
        ---
        apiVersion: helm.cattle.io/v1
        kind: HelmChart
        metadata:
          name: cert-manager
          namespace: kube-system
        spec:
          targetNamespace: cert-manager
          repo: https://charts.jetstack.io
          chart: cert-manager
          version: v1.5.1
          helmVersion: v3
          set:
            installCRDs: "true"
        ---
        apiVersion: helm.cattle.io/v1
        kind: HelmChart
        metadata:
          name: rancher
          namespace: kube-system
        spec:
          targetNamespace: cattle-system
          repo: https://releases.rancher.com/server-charts/stable/
          chart: rancher
          version: {{ .Values.rancherVersion }}
          set:
            ingress.tls.source: rancher
            hostname: {{ .Values.hostname }}
            replicas: 1
            global.cattle.psp.enabled: "false"
            bootstrapPassword: {{ .Values.bootstrapPassword | quote }}
          helmVersion: v3' > rancher-vcluster.yaml

# 11. Installing rancher-vcluster
$ kubectl apply -f rancher-vcluster.yaml
namespace/rancher-vcluster created
addon.harvesterhci.io/rancher-vcluster created

3. Configuring rancher-vcluster

點選 Advanced -> Addons -> ⋮ > Enable

點選 Enableled
Hostname 欄位, 輸入指向 Harvester VIP 的有效 DNS record 。這一點至關重要，因為 vcluster 入口會同步到 parent Harvester 叢集。有效的 hostname 用於過濾 vcluster workload 的入口流量。
Bootstrap Password 欄位, 為 vcluster 上部署的新 Rancher 輸入密碼。
點選右下角 Save 按鈕儲存。

進到 Harvester 內建 Racnher UI，點選 Workloads -> Pods -> rancher-vcluster namepsace ，一開始一定會看到 Job Failed

在 Rancher UI 上 clone helm-install-rancher-vcluster-xx 這個 pod 將 helm Container 的 Environment Variables 區塊底下的 REPO Variable Name 的 Value 改成 http://<啟動 chartmuseum Container 的 Host 主機的 IP Address>:8080

然後安裝 Cert-manager 和 Rancher 的 Jobs 也會 Failed

clone helm-install-cert-manager-xxx-x-kube-system-x-ranc-xxx 這個 pod 將 helm Container 的 Environment Variables 區塊底下的 REPO Variable Name 的 Value 改成 http://<啟動 chartmuseum Container 的 Host 主機的 IP Address>:8080
clone helm-install-rancher-xxx-x-kube-system-x-rancher-vcluster 這個 pod 將 helm Container 的 Environment Variables 區塊底下的 REPO Variable Name 的 Value 改成 http://<啟動 chartmuseum Container 的 Host 主機的 IP Address>:8080

確認 rancher-vcluster Namespace 中 Cert-manger 和 Rancher 所有的 Pods 都 Running，原來的 Job 還是 Failed 可以不理他。

再產生 Rancher-vcluster 的 KubeConfig，並移除其實已經成功的 Job

$ kubectl -n rancher-vcluster exec rancher-vcluster-0 -- cat /root/.kube/config > vcluster-kubeconfig

$ kubectl --kubeconfig=vcluster-kubeconfig get nodes
NAME    STATUS   ROLES    AGE    VERSION
hvx02   Ready    <none>   74m    v1.27.10+k3s2

$ kubectl --kubeconfig=vcluster-kubeconfig -n kube-system delete job helm-install-rancher helm-install-cert-manager

4. 將 Harvester 加入 Rancher

連線至 Rancher Web Console

其實該 Job 已成功

登入 Rancher 後，點選 Virtualization Management -> Import Exsting

Cluster Name: 輸入 hvx ，按 Create

之後會發現沒有註冊的訊息跳出來

手動將網址最後的 #events 改成 #registration

https://rancher.xxx.nip.io/dashboard/c/_/harvesterManager/harvesterhci.io.management.cluster/fleet-default/hvx#registration

然後按 Ctrl + Shift + R 清掉 Cache 並重新整理，就會看到註冊訊息

這時候就可以複製註冊 URL 貼到 Harvester 中

連線到 Harvester Web Console -> 左側選單 Advanced -> Settings -> ⋮ > Edit Setting

在 Value 欄位貼上 “註冊 URL”，然後按右下角 Save 按鈕

等一陣子後 Harvester Cluster 就 Active 囉~

Deploy Ollama & Open WebUI on RKE2 with Rancher Full Offline Installation - Harbor, RKE2, Rancher