antony@notes:~/kubernetes$ cat "Kubernetes-Multi-Tenancy.md"

Kubernetes Multi Tenancy

2022-06-13· kubernetes ·系統工程

Kubernetes Multi Tenancy

[TOC]

認識 Kubernetes AA

AuthN abbreviates AuthenticatioN: this is the process of verifying the user’s identity and decide whether she is legitimated to access your system.

認證你是誰

AuthZ abbreviates AuthoriZation: this is the process of verifying what an authenticated user can do within your system.

認證你可以做什麼事情

確認環境乾淨

$ kubectl get all
NAME                 TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)   AGE
service/kubernetes   ClusterIP   10.98.0.1    <none>        443/TCP   13d
bigred@m1:~/0613$ kubectl get pvc
No resources found in default namespace.

建立使用者憑證

$ mkdir kuser; cd kuser
$ mkdir user01; cd user01

## 產生私鑰
$ openssl genrsa -out private.key 2048
Generating RSA private key, 2048 bit long modulus (2 primes)
..............................+++++
.......................+++++
e is 65537 (0x010001)

## 檢視 CSR 內容
## 產生請求檔
## CN 使用者帳號
## O 使用者群組
$ openssl req -new -key private.key -out CSR.csr -subj "/CN=user01/O=user01"

$ ls -l
total 8
-rw-r--r-- 1 bigred bigred  911 Jun 13 13:22 CSR.csr
-rw------- 1 bigred bigred 1675 Jun 13 13:22 private.key

$ openssl req -noout -subject -in CSR.csr
subject=CN = user01, O = user01

## Base64 格式的 CSR
$ cat CSR.csr | base64 | tr -d '\n'; echo ""
LS0tLS1CRUdJTiBDRVJUSUZJQ0FURSBSRVFVRVNULS0tLS0KTUlJQ1p6Q0NBVThDQVFBd0lqRVBNQTBHQTFVRUF3d0dkWE5sY2pBeE1ROHdEUVlEVlFRS0RBWjFjMlZ5TURFdwpnZ0VpTUEwR0NTcUdTSWIzRFFFQkFRVUFBNElCRHdBd2dnRUtBb0lCQVFEQ0c2ZEd0RDE2VDNIWGh4d0pqQUcxCkFuUkVjYVNXWm1Zc3kvSStBZmNSVWdFM2hMRzd3SjhjWGhIWURFcWZqcHhGWGw0bFJCaVYvNWJLSUloVVo3RW8KS2dSMEZTVHQ3WWU3cDFSVzhNTVA2QkdUUXR4d0Fub1M3T2dIOGp4WWRmV2tNK3ZHVVgrbjd2Zjg4UTlsZ0Frago2NHJGNTlJSUFRVzdWMHdXTVlpbW1VUHhqK085Z2o5WEVtYTU0U1dHZ0thb0xvWDJrVkl4V2d3b1dzWnBCUUdSCkdieEZhVnFGSUQyNEh2aFVVbFIrL0RBTEFJb1VkN3htNzJ0ejk2cWVseUFKelpYU0RjN0pDSHRMMWN4SndGZFkKYm55V2pubUx4RW5JYUdqL0lDL0E3c3pMazlNUHZVbVVtQkZJdlFuMm9OeXU2WDdNU3ZwZGJOdlhtdkFBaHdaSApBZ01CQUFHZ0FEQU5CZ2txaGtpRzl3MEJBUXNGQUFPQ0FRRUFRTEhqNktQVzNIbjdiQ240Wm9BYmVzdnVWbDR1CkRvMURsNGpWY2Y0OUs2L20zb1p1RU5yZDlBeTFoYmR0eFI5MGpJb2IyWFhjcUFKV25EaFJtenpoQkdsczU5QW0KbStmNjBVN3FKS1I4V2RadnNqWkR3a0RoSGxDQjNJSjRNTVpQS3hVWU5tM1JKL1RoNVg3KzJnMytUQnpoUWI3LwplK3Q5MDhtM1lBeTN0Z2VzZDZFcDg1NVBPT3dFSCt2THZRY3k4djhwdDhwVWxkeXBYVk9hN2lzUHJQdzczT2t2CkhocStNTXFDeGEvaE12MFVFUVdNaWZ4THFJMUNrSnZFZVF3U09iM3gzdDR2Wm1vcDhZUVRSYTQyclJpemJYWTUKczRiWVNWdUh5ZitsYVFNQkdZNEdkVDE5Tml2Q1VOaWFZOGZ4VnlidDdEdER0ODUxY0hmTnVML2pYUT09Ci0tLS0tRU5EIENFUlRJRklDQVRFIFJFUVVFU1QtLS0tLQo=

$ export BASE64_CSR=$(base64 < CSR.csr | tr -d "\n")

## 上傳 CSR 至 K8S
$ echo 'apiVersion: certificates.k8s.io/v1
kind: CertificateSigningRequest
metadata:
  name: user01-csr
spec:
  groups:
    - system:authenticated
  request: ${BASE64_CSR}
  signerName: kubernetes.io/kube-apiserver-client
  expirationSeconds: 1800 # 30 Min
  usages:
    - client auth' > csr.yaml

$ cat csr.yaml | envsubst | kubectl apply -f -
certificatesigningrequest.certificates.k8s.io/user01-csr created

## 許可憑證
$ kubectl get csr
NAME         AGE   SIGNERNAME                            REQUESTOR          REQUESTEDDURATION   CONDITION
user01-csr   41s   kubernetes.io/kube-apiserver-client   kubernetes-admin   30m                 Pending

$ kubectl certificate approve user01-csr
certificatesigningrequest.certificates.k8s.io/user01-csr approved

$ kubectl get csr
NAME         AGE   SIGNERNAME                            REQUESTOR          REQUESTEDDURATION   CONDITION
user01-csr   55s   kubernetes.io/kube-apiserver-client   kubernetes-admin   30m                 Approved,Issued

## 下載許可憑證
$ kubectl get csr user01-csr -o jsonpath='{.status.certificate}' | base64 -d > k8s-signed.crt

$ cat k8s-signed.crt
-----BEGIN CERTIFICATE-----
MIIDCDCCAfCgAwIBAgIRAKEA0ts5MHzGleKU1MOrPAswDQYJKoZIhvcNAQELBQAw
FTETMBEGA1UEAxMKa3ViZXJuZXRlczAeFw0yMjA2MTMwNTIwMTBaFw0yMjA2MTMw
NTU1MTBaMCIxDzANBgNVBAoTBnVzZXIwMTEPMA0GA1UEAxMGdXNlcjAxMIIBIjAN
BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwhunRrQ9ek9x14ccCYwBtQJ0RHGk
lmZmLMvyPgH3EVIBN4Sxu8CfHF4R2AxKn46cRV5eJUQYlf+WyiCIVGexKCoEdBUk
7e2Hu6dUVvDDD+gRk0LccAJ6EuzoB/I8WHX1pDPrxlF/p+73/PEPZYAJI+uKxefS
CAEFu1dMFjGIpplD8Y/jvYI/VxJmueElhoCmqC6F9pFSMVoMKFrGaQUBkRm8RWla
hSA9uB74VFJUfvwwCwCKFHe8Zu9rc/eqnpcgCc2V0g3OyQh7S9XMScBXWG58lo55
i8RJyGho/yAvwO7My5PTD71JlJgRSL0J9qDcrul+zEr6XWzb15rwAIcGRwIDAQAB
o0YwRDATBgNVHSUEDDAKBggrBgEFBQcDAjAMBgNVHRMBAf8EAjAAMB8GA1UdIwQY
MBaAFLVzbcvg3zKcW7sqsazkDmMpdrRgMA0GCSqGSIb3DQEBCwUAA4IBAQA/dTMI
xV8K7WmN93nVCrhdhO49PB+E8/IV/Sl0sR8/9ZH/ue1Z8+mYM+lK3eGncNnCBulm
Uj5mFkoGxSgq+HIMlANTLz29wvZptU+GlnjFwqgejVF2srZOX1SZqGTj6GQEFMiG
QWKSixRq8VA9umacx9e/9WJ2pEANO8eqsSnWqS8i6mcW98tuE1MSuSxe0Ld/3GK7
5ijM9AyrLkOMZPs4qTHbIZnkpMKhAGLggwl/cWUQ8iRrAZUI+ywvMatVGch9X41x
60upWoReRY8ofvfE+PbNeaZRY8bDMq6fdw91rZHvniSyRTXOVcFuGMQbzlLDJ54j
JhVHdp7rI37uqYpK
-----END CERTIFICATE-----

$ openssl x509 -noout -subject -in k8s-signed.crt
subject=O = user01, CN = user01

注意，不管請求檔狀態如何，過了一定的時間以後， K8s 會自動刪除

產生使用者的 kubeconfig

$ kubectl config view --flatten=true得到的結果與$ cat ~/.kube/config 相同

$ kubectl config view --flatten=true | head -n 6 > config.tmp

$ echo 'contexts:
>   - context:
>       cluster: kubernetes
>       user: user01
>     name: user01-context
> current-context: user01-context
> kind: Config
> preferences: {}
> users:
>   - name: user01
>     user:
>       client-certificate-data: ${K8S_CRT}
>       client-key-data: ${K8S_KEY}' >> config.tmp

$ export K8S_CRT=$(base64 < k8s-signed.crt | tr -d "\n")

$ export K8S_KEY=$(base64 < private.key | tr -d "\n")

$ cat config.tmp | envsubst > config

$ sudo podman run -it --rm --name k1 -d quay.io/flysangel/image:landlord.kubectl bash
Trying to pull quay.io/flysangel/image:landlord.kubectl...
Getting image source signatures
Copying blob ce67b7059232 done
Copying blob df9b9388f04a skipped: already exists
Copying blob 6a897ff1da73 done
Copying blob 4e828f1a37ea done
Copying config 47ef1553bd done
Writing manifest to image destination
Storing signatures
a153d0c3fcb2b1ccb77ed403661a512d4f6a007c8a8bd57f67c620cdff302674

bigred@m1:~/0613/kuser/user01$ sudo podman cp config k1:/home/bigred/.kube/config

bigred@m1:~/0613/kuser/user01$ sudo podman exec -it k1 bash

## 認證失效，時間超過 30 分鐘
bash-5.1$ kubectl get pod
error: You must be logged in to the server (Unauthorized)

## 如未超過時間限制，錯誤訊息應如下
Bash-5.1$ kubectl get pod
Error from server (Forbidden): pods is forbidden: User "user01" cannot list resource "pods" in API group "" in the namespace "default"

練習

請問 K8S etcd 是否存有使用者資訊。請重新建立 K8S user01 帳號使用效期為 1 年將設定檔複製到 podman 測試

[注意] 請確認 csr 是否存留舊資料

$ kubectl get csr
NAME         AGE   SIGNERNAME                            REQUESTOR          REQUESTEDDURATION   CONDITION
user01-csr   48m   kubernetes.io/kube-apiserver-client   kubernetes-admin   365d                Approved,Issued

$ kubectl delete csr user01-csr
certificatesigningrequest.certificates.k8s.io "user01-csr" deleted

$ openssl genrsa -out private.key 2048
Generating RSA private key, 2048 bit long modulus (2 primes)
.......+++++
....+++++
e is 65537 (0x010001)

$ openssl req -new -key private.key -out CSR.csr -subj "/CN=user01/O=user01"

1$ openssl req -noout -subject -in CSR.csr
subject=CN = user01, O = user01

$ export BASE64_CSR=$(base64 < CSR.csr | tr -d "\n")

$ echo 'apiVersion: certificates.k8s.io/v1
> kind: CertificateSigningRequest
> metadata:
>   name: user01-csr
> spec:
>   groups:
>     - system:authenticated
>   request: ${BASE64_CSR}
>   signerName: kubernetes.io/kube-apiserver-client
>   expirationSeconds: 31536000 # 365 day
>   usages:
>     - client auth' > csr.yaml

$ ls -l
total 12
-rw-r--r-- 1 bigred bigred  911 Jun 13 14:28 CSR.csr
-rw-r--r-- 1 bigred bigred  282 Jun 13 14:29 csr.yaml
-rw------- 1 bigred bigred 1675 Jun 13 14:28 private.key

$ cat csr.yaml | envsubst | kubectl apply -f -
certificatesigningrequest.certificates.k8s.io/user01-csr created

$ kubectl certificate approve user01-csr
certificatesigningrequest.certificates.k8s.io/user01-csr approved

$ kg csr
NAME         AGE   SIGNERNAME                            REQUESTOR          REQUESTEDDURATION   CONDITION
user01-csr   18s   kubernetes.io/kube-apiserver-client   kubernetes-admin   365d                Approved,Issued

$ kubectl get csr user01-csr -o jsonpath='{.status.certificate}' | base64 -d > k8s-signed.crt

$ openssl x509 -noout -subject -in k8s-signed.crt
subject=O = user01, CN = user01

$ kubectl config view --flatten=true | head -n 6 > config.tmp

$ echo 'contexts:
>   - context:
>       cluster: kubernetes
>       user: user01
>     name: user01-context
> current-context: user01-context
> kind: Config
> preferences: {}
> users:
>   - name: user01
>     user:
>       client-certificate-data: ${K8S_CRT}
>       client-key-data: ${K8S_KEY}' >> config.tmp

$ export K8S_CRT=$(base64 < k8s-signed.crt | tr -d "\n")

$ export K8S_KEY=$(base64 < private.key | tr -d "\n")

$ cat config.tmp | envsubst > config

$ sudo podman ps -a
CONTAINER ID  IMAGE                                     COMMAND     CREATED         STATUS                   PORTS                                                                 NAMES
b18077d221fd  localhost/podman-pause:4.1.0-1652313655               12 days ago     Exited (0) 4 days ago    0.0.0.0:80->8080/tcp, 0.0.0.0:5432->5432/tcp, 0.0.0.0:6379->6379/tcp  943dc009d7bb-infra
b78deed44826  quay.io/flysangel/quay-rhel8:v3.6.6       registry    12 days ago     Exited (137) 4 days ago  0.0.0.0:80->8080/tcp, 0.0.0.0:5432->5432/tcp, 0.0.0.0:6379->6379/tcp  quay-quay-app
d45e257168cc  quay.io/flysangel/postgres:9.6.24-alpine  postgres    12 days ago     Exited (0) 4 days ago    0.0.0.0:80->8080/tcp, 0.0.0.0:5432->5432/tcp, 0.0.0.0:6379->6379/tcp  quay-postgresql
2992e6a01ba6  quay.io/flysangel/redis-5:1-169           run-redis   12 days ago     Exited (0) 4 days ago    0.0.0.0:80->8080/tcp, 0.0.0.0:5432->5432/tcp, 0.0.0.0:6379->6379/tcp  quay-redis
a153d0c3fcb2  quay.io/flysangel/image:landlord.kubectl  bash        33 minutes ago  Up 33 minutes ago                                                                              k1
$ sudo podman run -it --rm --name k1 -d quay.io/flysangel/image:landlord.kubectl bash
Error: error creating container storage: the container name "k1" is already in use by "a153d0c3fcb2b1ccb77ed403661a512d4f6a007c8a8bd57f67c620cdff302674". You have to remove that container to be able to reuse that name.: that name is already in use

$ sudo podman rm k1
Error: cannot remove container a153d0c3fcb2b1ccb77ed403661a512d4f6a007c8a8bd57f67c620cdff302674 as it is running - running or paused containers cannot be removed without force: container state improper

$ sudo podman rm -f k1
WARN[0010] StopSignal SIGTERM failed to stop container k1 in 10 seconds, resorting to SIGKILL
a153d0c3fcb2b1ccb77ed403661a512d4f6a007c8a8bd57f67c620cdff302674

$ sudo podman run -it --rm --name k1 -d quay.io/flysangel/image:landlord.kubectl bash
f6d7b99e4e39059d62113c4d641ff766a378d1976d9dc1c3b49e90f99db9d494

$ sudo podman cp config k1:/home/bigred/.kube/config

$ sudo podman exec -it k1 bash
bash-5.1$ kubectl get pod
Error from server (Forbidden): pods is forbidden: User "user01" cannot list resource "pods" in API group "" in the namespace "default"

檢查憑證指令

$ openssl x509 -noout -text -in k8s-signed.crt

認識 Role Based Access Control

Subjects: The set of users and processes that want to access the Kubernetes API.

Resources: The set of Kubernetes API Objects available in the cluster. Examples include Pods, Deployments, Services, Nodes, and PersistentVolumes, among others.

Verbs: The set of operations that can be executed to the resources above. Different verbs are available (examples: get, watch, create, delete, etc.), but ultimately all of them are Create, Read, Update or Delete (CRUD) operations.

Role Based Access Control (RBAC)

$ kubectl create ns user01

$ echo 'apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: user01-role
  namespace: user01
rules:
- apiGroups: [""] # "" indicates the core API group
  resources: ["pods", "services"]
  verbs: ["get", "watch", "list"]' > role.yaml

$ kubectl apply -f role.yaml
role.rbac.authorization.k8s.io/user01-role created

$ echo 'apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: user01-rbind
  namespace: user01
subjects:
- kind: User
  name: user01
  apiGroup: rbac.authorization.k8s.io
roleRef:
  kind: Role #this must be Role or ClusterRole
  name: user01-role 
  apiGroup: rbac.authorization.k8s.io' > rolebind.yaml

$ kubectl apply -f rolebind.yaml
rolebinding.rbac.authorization.k8s.io/user01-rbind created

$ kubectl get role -n user01
NAME          CREATED AT
user01-role   2022-06-13T06:49:41Z

$ kubectl get rolebinding -n user01
NAME           ROLE               AGE
user01-rbind   Role/user01-role   50s

練習

參考投影片 10、14 兩頁。請重新建立 K8S user02 帳號使用效期為 1 年預設使用 user02 namespace 權限具備完全控制將設定檔複製到 podman 測試

$ mkdir user02
$ cd user02

$ openssl genrsa -out private.key 2048
$ openssl req -new -key private.key -out CSR2.csr -subj "/CN=user02/O=user02"

$ openssl req -noout -subject -in CSR2.csr

$ export BASE64_CSR2=$(base64 < CSR2.csr | tr -d "\n")

$ echo 'apiVersion: certificates.k8s.io/v1
kind: CertificateSigningRequest
metadata:
  name: user02-csr2
spec:
  groups:
    - system:authenticated
  request: ${BASE64_CSR2}
  signerName: kubernetes.io/kube-apiserver-client
  expirationSeconds: 31536000 # 365 day
  usages:
    - client auth' > csr2.yaml
	
$ cat csr2.yaml | envsubst | kubectl apply -f -

$ kubectl get csr
$ kubectl certificate approve user02-csr2
$ kubectl get csr
$ kubectl get csr user02-csr2 -o jsonpath='{.status.certificate}' | base64 -d > k8s-signed.crt
$ openssl x509 -noout -subject -in k8s-signed.crt
$ kubectl config view --flatten=true | head -n 6 > config.tmp
$ echo 'contexts:
  - context:
      cluster: kubernetes
      namespace: user02
      user: user02
    name: user02-context
current-context: user02-context
kind: Config
preferences: {}
users:
  - name: user02
    user:
      client-certificate-data: ${K8S_CRT}
      client-key-data: ${K8S_KEY}' >> config.tmp

$ cat config.tmp
$ export K8S_CRT=$(base64 < k8s-signed.crt | tr -d "\n")
$ export K8S_KEY=$(base64 < private.key | tr -d "\n")
$ cat config.tmp | envsubst > config
$ sudo podman run -it --rm --name k2 -d quay.io/flysangel/image:landlord.kubectl bash
$ sudo podman cp config k2:/home/bigred/.kube/config
$ sudo podman exec -it k2 bash
kubectl create ns user02

$ echo 'apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: user02-role
  namespace: user02
rules:
- apiGroups: [""] # "" indicates the core API group
  resources: ["*"]
  verbs: ["*"]' > role.yaml
  
$ kubectl apply -f role.yaml
$ echo 'apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: user02-rbind
  namespace: user02
subjects:
- kind: User
  name: user02
  apiGroup: rbac.authorization.k8s.io
roleRef:
  kind: Role #this must be Role or ClusterRole
  name: user02-role
  apiGroup: rbac.authorization.k8s.io' > rolebind.yaml

$ kubectl apply -f rolebind.yaml
$ kubectl get roles -n user02
$ kubectl get rolebinding -n user01
$ kubectl get rolebinding -n user02
$ sudo podman exec -it k2 bash

tags: `系統工程`

Kubernetes StatefulSet Kubernetes Grafana

Kubernetes Multi Tenancy

Kubernetes Multi Tenancy

認識 Kubernetes AA

確認環境乾淨

建立使用者憑證

產生使用者的 kubeconfig

練習

認識 Role Based Access Control

Role Based Access Control (RBAC)

練習

tags: 系統工程

tags: `系統工程`