antony@notes:~/kubernetes$ cat "K8S-雲端企業永續應用系統平台建置.md"

K8S 雲端企業永續應用系統平台建置

2022-09-10· kubernetes ·系統工程

K8S 雲端企業永續應用系統平台建置

:::warning

:::spoiler 目錄

[TOC]

:::

如何讓網站應用系統達到永續 (Run 的長長久久)

如何讓資料庫達到永續 (Run 的長長久久)

環境準備

$ wget http://www.oc99.org/zip/bobowas.zip

$ unzip bobowas.zip

$ cd k8simg/goweb/src/

# 編譯 main.go
$ ./go-compile.sh

$ ls -l ..
total 6556
-rw-r--r-- 1 bigred bigred     160 Sep  5 23:40 Dockerfile
-rwxr-xr-x 1 bigred bigred     200 Sep  5 23:48 cgiTEST.sh
-rwxr-xr-x 1 bigred bigred 6700289 Sep  6 13:21 goweb
drwxr-sr-x 2 bigred bigred    4096 Sep  5 20:52 src

$ cd ../..

# 手動部屬 image 到 w1 w2 node
$ ./go-k8s-images.sh
[images build]
alp.myfbs image ok
alp.mysql image ok
alp.goweb image ok

[images deploy]
w1: localhost/alp.myfbs image ok
w2: localhost/alp.myfbs image ok

w1: localhost/alp.mysql image ok
w2: localhost/alp.mysql image ok

w1: localhost/alp.goweb image ok
w2: localhost/alp.goweb image ok

# 檢查
$ ssh w1 'sudo podman images | grep alp'
localhost/alp.goweb                                       latest      acc7309b79f5  About a minute ago  67.6 MB
localhost/alp.mysql                                       latest      922af3b4f804  About a minute ago  439 MB
localhost/alp.myfbs                                       latest      317eb22331fb  About a minute ago  74.5 MB
quay.io/ict39/alpine                                      latest      6044a98f30fd  11 months ago       10.4 MB
quay.io/ict39/alpine.derby                                latest      175f1c9e20f6  3 years ago         188 MB

$ ssh w2 'sudo podman images | grep alp'
localhost/alp.goweb                                       latest      acc7309b79f5  About a minute ago  67.6 MB
localhost/alp.mysql                                       latest      922af3b4f804  2 minutes ago       439 MB
localhost/alp.myfbs                                       latest      317eb22331fb  2 minutes ago       74.5 MB
quay.io/ict39/alpine                                      latest      6044a98f30fd  11 months ago       10.4 MB

# 切換目錄
$ cd ../k8sobj/

# 將 pod 全部建立出來
$ ./go-k8s-object.sh
bobo namespace created
bobo.fbs pod(w1) created
bobo.mysql(w2) pod created
bobo.goweb(w2) pod created
bobo.gosvc service created

# 檢查，注意 svc 的 EXTERNAL-IP
$ kubectl get all -n bobo -o wide
NAME             READY   STATUS    RESTARTS   AGE   IP            NODE   NOMINATED NODE   READINESS GATES
pod/bobo.fbs     1/1     Running   0          55s   10.244.1.15   w1     <none>           <none>
pod/bobo.goweb   1/1     Running   0          55s   10.244.1.16   w1     <none>           <none>
pod/bobo.mysql   1/1     Running   0          55s   10.244.2.10   w2     <none>           <none>

NAME                TYPE        CLUSTER-IP   EXTERNAL-IP     PORT(S)   AGE   SELECTOR
service/svc-gosrv   ClusterIP   10.98.0.22   192.168.179.4   80/TCP    54s   run=bobo.goweb

環境架構圖

我們產生的 Deployment 事實上是一種 Contoller manager ，他會產生 ReplicaSet ( 也是 Controller )

目前 goweb 的 pod 由 Deployment object 做出來的，pod 會共用 w1 主機的 /opt/www 目錄
還有用 HPA ，所以當 pod 使用的 CPU 超過設定的平均時，pod 會擴充出來，最多 5 個。

但是我們 pod 當初在建立的時候，CPU 限制是 400m，所在的 node CPU 最多只能用 2000m ，還要扣掉系統運作要用的 CPU (假設 200m)，再扣掉 fbs 那台 pod (假設用掉 300m)，實際上只剩 1500m 的 CPU 可以用，但是 pod 的上限卻設定 5 個，只要擴充到 4 個，實際上 CPU 就會不夠用了，在狀態上才會顯示 panding
deployment 的 yaml 檔有指定 POD 都建在 w1 上，這樣雖然可以共用目錄來提供相同的服務，但無法達到容錯，只要 w1 機器出問題，裡面所有的 POD 也都無法正常運作。且所有 POD 都共用 w1 的資源，難達到分流的目的。

改善 goweb 底層架構

Deployment 的 pod 在擴充時，應平均分配到每一台 node 上
每個 pod 會掛載到同一個目錄區，這個目錄區通過網路來讓每個 pod 共享
PV = PersistentVolume
PVC = PersistentVolumeClaim
NFS = Network File System

攥寫 YAML 檔

Goweb YAML

depgw.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: bobo.goweb
  labels:
    app: bobo.goweb
  namespace: bobo
spec:
  replicas: 2
  selector:
    matchLabels:
      app: bobo.goweb
  template:
    metadata:
      labels:
        app: bobo.goweb
    spec:
      containers:
      - image: localhost/alp.goweb
        name: myweb
        ports:
        - containerPort: 8080
        volumeMounts:
        - mountPath: /www
          name: pv-goweb-volume
        resources:
          limits:
            cpu: 400m
      volumes:
      - name: pv-goweb-volume
        persistentVolumeClaim:
         claimName: pvc-goweb

pod-goweb.yaml (hostPath)

apiVersion: v1
kind: Pod
metadata:
  labels:
    run: bobo.goweb
  namespace: bobo
  name: bobo.goweb
spec:
  containers:
  - image: localhost/alp.goweb
    name: myweb
    ports:
     - containerPort: 8080
    volumeMounts:
    - mountPath: /www
      name: goweb-volume
  volumes:
  - name: goweb-volume
    hostPath:
      # directory location on host
      path: /opt/www
  nodeSelector:
    kubernetes.io/hostname: w1

pv-goweb.yaml

apiVersion: v1
kind: PersistentVolume
metadata:
  name: pv-goweb
spec:
  claimRef:
    name: pvc-goweb
    namespace: bobo
  capacity:
    storage: 10Gi
  accessModes:
    - ReadWriteOnce
  local:
    path: "/opt/www/"
  nodeAffinity:
    required:
      nodeSelectorTerms:
      - matchExpressions:
        - key: kubernetes.io/hostname
          operator: In
          values:
          - w1

pvc-goweb.yaml

kind: PersistentVolumeClaim
apiVersion: v1
metadata:
  name: pvc-goweb
  namespace: bobo
spec:
  accessModes:
    - ReadWriteOnce
  storageClassName: ""
  resources:
    requests:
      storage: 5Gi

hpa-bobo_goweb.yml

apiVersion: autoscaling/v1
kind: HorizontalPodAutoscaler
metadata:
  name: hpa-bobo.goweb
  namespace: bobo
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: bobo.goweb
  minReplicas: 2
  maxReplicas: 5
  targetCPUUtilizationPercentage: 20

File Browser Server yaml

pod-fbs.yaml

apiVersion: v1
kind: Pod
metadata:
  name: bobo.fbs
  namespace: bobo
  labels:
    name: bobo.fbs
spec:
  volumes:
  - name: pv-fbs-storage
    persistentVolumeClaim:
     claimName: pvc-fbs
  containers:
  - name: myfbs
    image: localhost/alp.myfbs
    imagePullPolicy: IfNotPresent
    ports:
     - containerPort: 4000
       hostPort: 8080
    volumeMounts:
    - mountPath: /srv
      name: pv-fbs-storage

pv-fbs.yaml

apiVersion: v1
kind: PersistentVolume
metadata:
  name: pv-fbs
spec:
  claimRef:
    name: pvc-fbs
    namespace: bobo
  capacity:
    storage: 10Gi
  accessModes:
    - ReadWriteOnce
  local:
    path: "/opt/www/"
  nodeAffinity:
    required:
      nodeSelectorTerms:
      - matchExpressions:
        - key: kubernetes.io/hostname
          operator: In
          values:
          - w1

pvc-fbs.yaml

kind: PersistentVolumeClaim
apiVersion: v1
metadata:
  name: pvc-fbs
  namespace: bobo
spec:
  accessModes:
    - ReadWriteOnce
  storageClassName: ""
  resources:
    requests:
      storage: 5Gi

MySQL Server

pod-mysql.yaml

apiVersion: v1
kind: Pod
metadata:
  labels:
    run: bobo.mysql
  namespace: bobo
  name: bobo.mysql
spec:
  volumes:
  - name: pv-storage
    persistentVolumeClaim:
     claimName: pvc-mysql
  containers:
  - image: localhost/alp.mysql
    name: mydb
    ports:
     - containerPort: 3306
       hostPort: 3306
    volumeMounts:
    - mountPath: /var/lib/mysql
      name: pv-storage

pv-mysql.yaml

apiVersion: v1
kind: PersistentVolume
metadata:
  name: pv-mysql
spec:
  claimRef:
    name: pvc-mysql
    namespace: bobo
  capacity:
    storage: 10Gi
  accessModes:
    - ReadWriteOnce
  local:
    path: "/home/bigred/mysql/"
  nodeAffinity:
    required:
      nodeSelectorTerms:
      - matchExpressions:
        - key: kubernetes.io/hostname
          operator: In
          values:
          - w2

pvc-mysql.yaml

kind: PersistentVolumeClaim
apiVersion: v1
metadata:
  name: pvc-mysql
  namespace: bobo
spec:
  accessModes:
    - ReadWriteOnce
  storageClassName: ""
  resources:
    requests:
      storage: 10Gi

Kubernetes-Cluster-build Kubernetes Init with Config.yaml