antony@notes:~/rancher$ cat "Install-High-Availability-Rancher-and-Setting-up-an-NGINX-Load-Balancer.md"
Install High-Availability Rancher and Setting up an NGINX Load Balancer
Install High-Availability Rancher and Setting up an NGINX Load Balancer
環境介紹
- 三台 node 都會跑 K3S + Rancher
- 一台 node 當 Load Balancer
- OS 都是 SLES 15 SP5
Architecture
- DNS for Rancher should resolve to a layer 4 load balancer
- The Load Balancer should forward port TCP/80 and TCP/443 to all 3 nodes in the Kubernetes cluster.
- The Ingress controller will redirect HTTP to HTTPS and terminate SSL/TLS on port TCP/443.
- The Ingress controller will forward traffic to port TCP/80 on the pod in the Rancher deployment.
- Kubernetes Rancher install with layer 4 load balancer, depicting SSL termination at ingress controllers
1. 安裝 HA K3S v1.28.8
# 1. SSH 連線到 r1
$ ssh rancher@192.168.11.25
# 2. 安裝 k3s
$ curl -sfL https://get.k3s.io | \
INSTALL_K3S_VERSION=v1.28.8+k3s1 \
K3S_KUBECONFIG_MODE="644" \
K3S_TOKEN=SECRET sh -s - server \
--cluster-init
# 3. SSH 連線到 r2
$ ssh rancher@192.168.11.26
# 4. 安裝 k3s
$ curl -sfL https://get.k3s.io | \
INSTALL_K3S_VERSION=v1.28.8+k3s1 \
K3S_KUBECONFIG_MODE="644" \
K3S_TOKEN=SECRET sh -s - server \
--server https://192.168.11.25:6443
# 5. SSH 連線到 r3
$ ssh rancher@192.168.11.27
# 6. 安裝 k3s
$ curl -sfL https://get.k3s.io | \
INSTALL_K3S_VERSION=v1.28.8+k3s1 \
K3S_KUBECONFIG_MODE="644" \
K3S_TOKEN=SECRET sh -s - server \
--server https://192.168.11.25:6443
# 7. SSH 連線到 r1
$ ssh rancher@192.168.11.25
# 8. Check node Ready
$ kubectl get nodes
NAME STATUS ROLES AGE VERSION
r1 Ready control-plane,etcd,master 32m v1.28.8+k3s1
r2 Ready control-plane,etcd,master 22m v1.28.8+k3s1
r3 Ready control-plane,etcd,master 20m v1.28.8+k3s1
# 9. etcd 健康檢查
## 先安裝 etcdctl
$ curl -s https://raw.githubusercontent.com/braveantony/bash-script/main/install-etcdcheck.sh | sudo bash
...
etcd Version: 3.4.13
Git SHA: ae9734ed2
Go Version: go1.12.17
Go OS/Arch: linux/amd64
etcdctl version: 3.4.13
API version: 3.4
## 用 root 執行以下命令
$ ETCDCTL_ENDPOINTS='https://127.0.0.1:2379' \
ETCDCTL_CACERT='/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt' \
ETCDCTL_CERT='/var/lib/rancher/k3s/server/tls/etcd/server-client.crt' \
ETCDCTL_KEY='/var/lib/rancher/k3s/server/tls/etcd/server-client.key' \
ETCDCTL_API=3 etcdctl endpoint health --cluster --write-out=table
+----------------------------+--------+-------------+-------+
| ENDPOINT | HEALTH | TOOK | ERROR |
+----------------------------+--------+-------------+-------+
| https://192.168.11.27:2379 | true | 9.981773ms | |
| https://192.168.11.26:2379 | true | 10.882845ms | |
| https://192.168.11.25:2379 | true | 11.50637ms | |
+----------------------------+--------+-------------+-------+2. Install Rancher HA v2.8.3
# 1. 安裝 Helm
$ curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
# Setup kubeconfig
$ sudo cp /etc/rancher/k3s/k3s.yaml ~/.kube/config && \
sudo chmod 600 ~/.kube/config && \
sudo chown $(id -u):$(id -g) ~/.kube/config
# 2. Add the Helm Chart Repository
$ helm repo add rancher-prime https://charts.rancher.com/server-charts/prime
# 3. Create a Namespace for Rancher
$ kubectl create ns cattle-system
# 4. 使用 cert-manager 產生憑證
# 4.1. Add the Jetstack Helm repository
$ helm repo add jetstack https://charts.jetstack.io
# 4.2. Update your local Helm chart repository cache
$ helm repo update
# 4.3. Install the cert-manager Helm chart
$ helm install cert-manager jetstack/cert-manager \
--namespace cert-manager \
--create-namespace \
--set installCRDs=true \
--set prometheus.enabled=false \
--set webhook.timeoutSeconds=4 \
--version v1.14.4
# 4.4. Check cert-manager pod status
$ kubectl get pods -n cert-manager
NAME READY STATUS RESTARTS AGE
cert-manager-798bf649c9-4fsgg 1/1 Running 0 18m
cert-manager-cainjector-c7d4dbdd9-nzwqn 1/1 Running 0 18m
cert-manager-webhook-847d7676c9-7wvvz 1/1 Running 0 18m
# 5. Check DNS a record
## 要設為 Load Balancer 的 IP
$ dig @192.168.11.85 rancher.antony520.com +short
192.168.11.28
# 6. Install Rancher Prime with Helm
$ helm install rancher rancher-prime/rancher \
--namespace cattle-system \
--set hostname=rancher.antony520.com \
--set bootstrapPassword=rancheradmin \
--set global.cattle.psp.enabled=false \
--set replicas=3 \
--version 2.8.3
# 7. Verify that the Rancher Server is Successfully Deployed
$ kubectl -n cattle-system get pods
NAME READY STATUS RESTARTS AGE
rancher-648d4fbc6c-7rpwx 1/1 Running 0 4m1s
rancher-648d4fbc6c-bqznk 1/1 Running 1 (44s ago) 4m1s
rancher-648d4fbc6c-nvchh 1/1 Running 1 (44s ago) 4m1s
# 8. check web
$ curl -k -H "host: rancher.antony520.com" https://192.168.11.25/dashboard/3. Setting up an NGINX Load Balancer
Step 1: Create NGINX Configuration
# 1. SSH 連線到 lb 那台 node
$ ssh rancher@192.168.11.28
# 2. Example NGINX config
$ cat <<EOF | sudo tee /etc/nginx/nginx.conf
worker_processes 4;
worker_rlimit_nofile 40000;
events {
worker_connections 8192;
}
stream {
log_format proxy '\$remote_addr [\$time_local] '
'\$protocol \$status \$bytes_sent \$bytes_received '
'\$session_time "\$upstream_addr"';
error_log /var/log/nginx/error.log;
access_log /var/log/nginx/access.log proxy;
upstream rancher_servers_http {
least_conn;
server 192.168.11.25:80 max_fails=3 fail_timeout=5s;
server 192.168.11.26:80 max_fails=3 fail_timeout=5s;
server 192.168.11.27:80 max_fails=3 fail_timeout=5s;
}
server {
listen 80;
proxy_pass rancher_servers_http;
}
upstream rancher_servers_https {
least_conn;
server 192.168.11.25:443 max_fails=5 fail_timeout=8s;
server 192.168.11.26:443 max_fails=5 fail_timeout=8s;
server 192.168.11.27:443 max_fails=5 fail_timeout=8s;
}
server {
listen 443;
proxy_pass rancher_servers_https;
}
}
EOFStep 2: Run NGINX as Podman Pod
# 1. 寫 yaml 檔
$ cat <<EOF > nginx-lb.yaml
apiVersion: v1
kind: Pod
metadata:
labels:
app: nginx-lb
name: nginx-lb
spec:
containers:
- args:
- nginx
- -g
- daemon off;
image: docker.io/library/nginx:stable
name: nginx-lb
ports:
- containerPort: 80
hostPort: 80
- containerPort: 443
hostPort: 443
volumeMounts:
- mountPath: /etc/nginx/nginx.conf
name: etc-nginx-nginx.conf-host-0
- mountPath: /var/log/nginx
name: var-log-nginx-host-1
volumes:
- hostPath:
path: /etc/nginx/nginx.conf
type: File
name: etc-nginx-nginx.conf-host-0
- hostPath:
path: /var/log/nginx
type: Directory
name: var-log-nginx-host-1
EOF
# 2. Run Pod
$ sudo podman kube play nginx.yaml
# 3. Check Pod Status
$ sudo podman ps -a --pod
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES POD ID PODNAME
d974299ac86d localhost/podman-pause:4.4.4-1680004800 14 seconds ago Up 9 seconds 0.0.0.0:80->80/tcp, 0.0.0.0:443->443/tcp af284bde454b-infra af284bde454b nginx-lb
9437a4d9d554 docker.io/library/nginx:stable nginx -g daemon o... 9 seconds ago Up 9 seconds 0.0.0.0:80->80/tcp, 0.0.0.0:443->443/tcp nginx-lb-nginx-lb af284bde454b nginx-lb
# 4. Access nginx lb on other node
$ curl -k https://rancher.antony520.com/
# 5. Check access log
$ sudo cat /var/log/nginx/access.log
192.168.11.85 [24/Apr/2024:09:42:16 +0000] TCP 200 10052 0.027 "192.168.11.26:443"
# 6. Start on Boot
$ cd ~/nginx/
$ sudo podman generate systemd --name nginx-lb --files
$ sudo cp container-nginx-lb-nginx-lb.service pod-nginx-lb.service /usr/lib/systemd/system/
$ sudo systemctl daemon-reload
$ sudo systemctl enable --now container-nginx-lb-nginx-lb.service pod-nginx-lb.service