面向 AI/ML 工作负载的 Stateful KES

kind create cluster --config ~/operator/testing/kind-config.yaml

kubectl apply -k github.com/minio/operator/

kubectl apply -k github.com/minio/operator/examples/kustomization/tenant-lite

cat <<EOF | kubectl apply -f -

apiVersion: v1

kind: Pod

metadata

  name: ubuntu

  namespace: tenant-lite

  labels

app: ubuntu

spec

  containers

  - image: ubuntu

command:

   - "sleep"

   - "604800"

imagePullPolicy: IfNotPresent

name: ubuntu

  restartPolicy: Always

EOF

apt update

apt install wget

wget https://github.com/minio/kes/releases/latest/download/kes-linux-amd64

mv kes-linux-amd64 kes

chmod +x kes

mv kes /usr/local/bin/kes

wget https://dl.min.io/client/mc/release/linux-amd64/mc

chmod +x mc

mv mc /usr/local/bin/mc

rm -rf ~/kes

mkdir ~/kes

cd ~/kes

touch init.yml

cd ~/kes

kes identity new --key sys-admin.key --cert sys-admin.crt kes-sys-admin

kes identity new --key minio-admin.key --cert minio-admin.crt minio-admin

kes identity new --key minio.key --cert minio.crt minio

kes identity new --ip "10.244.2.7" localhost # 将生成 private.key 和 public.crt

                        |

                        |____ Ubuntu Pod 的 IP 地址。

root@ubuntu:/# cd ~/kes

root@ubuntu:~/kes# ls

data  init.yml  minio-admin.crt  minio-admin.key  minio.crt  minio.key  private.key  public.crt  sys-admin.crt  sys-admin.key

                   |           |          |      |                                |            |

                   |           |          |__________|___ minio                       |________________|___ kes-sys-admin

                   |           |

                   |_______________|___ minio-admin

root@ubuntu:~/kes#

cat /dev/urandom | head -c 32 | base64 # 将结果放入 .bashrc 文件中

vi ~/.bashrc

export KES_UNSEAL_KEY=<上面命令的结果>

source ~/.bashrc

echo $KES_UNSEAL_KEY # 应该打印出密钥值

cd ~/kes

echo "version: v1" > ~/kes/init.yml

echo "address: 0.0.0.0:7373" >> ~/kes/init.yml

echo "" >> ~/kes/init.yml

echo "tls:" >> ~/kes/init.yml

echo " key: private.key" >> ~/kes/init.yml

echo " cert: public.crt" >> ~/kes/init.yml

echo " client:" >> ~/kes/init.yml

echo " verify_cert: false" >> ~/kes/init.yml

echo "" >> ~/kes/init.yml

echo "system:" >> ~/kes/init.yml

echo " admin:" >> ~/kes/init.yml

echo " identity: $(kes identity of sys-admin.crt)" >> ~/kes/init.yml

echo "" >> ~/kes/init.yml

echo "unseal:" >> ~/kes/init.yml

echo " environment:" >> ~/kes/init.yml

echo " name: KES_UNSEAL_KEY" >> ~/kes/init.yml

echo "" >> ~/kes/init.yml

echo "enclave:" >> ~/kes/init.yml

echo " default:" >> ~/kes/init.yml

echo " admin:" >> ~/kes/init.yml

echo " identity: $(kes identity of minio-admin.crt)" >> ~/kes/init.yml

echo " policy:" >> ~/kes/init.yml

echo " minio:" >> ~/kes/init.yml

echo " allow:" >> ~/kes/init.yml

echo " - /v1/api" >> ~/kes/init.yml

echo " - /v1/log/audit" >> ~/kes/init.yml

echo " - /v1/log/error" >> ~/kes/init.yml

echo " - /v1/key/create/*" >> ~/kes/init.yml

echo " - /v1/key/generate/*" >> ~/kes/init.yml

echo " - /v1/key/decrypt/*" >> ~/kes/init.yml

echo " - /v1/key/bulk/decrypt/*" >> ~/kes/init.yml

cd ~/kes # init.yml 文件保存的路径

kes init --config init.yml ~/kes/data

root@ubuntu:~/kes# cd ~/kes # init.yml 文件保存的路径

kes init --config init.yml ~/kes/data

TLS

· 私钥: private.key

· 证书: public.crt

系统

· 标识: 1a65f6f86c3268b30528fe4aab88fc6994730346e1c1863052fa3fa192d77d3e

解封

· 环境变量: KES_UNSEAL_KEY

╭─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮

│ 在 /root/kes/data 中初始化 KES v0.22.3 │

╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯

root@ubuntu:~/kes#

kes server ~/kes/data

root@ubuntu:~/kes# kes server ~/kes/data

版权所有 MinIO, Inc. https://min-io.cn

许可证 GNU AGPLv3 https://gnu.ac.cn/licenses/agpl-3.0.html

版本 v0.22.3 linux/amd64

端点 https://127.0.0.1:7373

https://10.244.4.2:7373

mTLS   跳过验证  客户端证书未经验证

内存锁定   关闭      无法锁定RAM页面。请考虑授予CAP_IPC_LOCK权限

cd ~/kes

export KES_SERVER=https://127.0.0.1:7373

export KES_CLIENT_KEY=minio-admin.key

export KES_CLIENT_CERT=minio-admin.crt

kes policy assign -k minio $(kes identity of minio.crt)

root@ubuntu:/# cd ~/kes

export KES_SERVER=https://127.0.0.1:7373

export KES_CLIENT_KEY=minio-admin.key

export KES_CLIENT_CERT=minio-admin.crt

kes policy assign -k minio $(kes identity of minio.crt)

root@ubuntu:~/kes#

root@ubuntu:~/kes#

root@ubuntu:~/kes#

root@ubuntu:~/kes#

root@ubuntu:~/kes#

root@ubuntu:~/kes# printenv | grep KES_SERVER

KES_SERVER=https://127.0.0.1:7373

root@ubuntu:~/kes#

# /Users/aj/minio/private.key 是 ~/kes/minio.key

# /Users/aj/minio/public.crt 是 ~/kes/minio.crt

kubectl create secret generic kes-minio -n tenant-lite --from-file=/Users/aj/minio/private.key --from-file=/Users/aj/minio/public.crt

# /Users/aj/minio/private.key 是 ~/kes/private.key

# /Users/aj/minio/public.crt 是 ~/kes/public.crt

kubectl create secret generic kes-minio-public -n tenant-lite --from-file=/Users/aj/minio/private.key --from-file=/Users/aj/minio/public.crt

k edit tenant -n tenant-lite

apiVersion: minio.min.io/v2

kind: Tenant

metadata

  name: storage

  namespace: minio-tenant

spec

  # externalClientCertSecrets 用于与MinIO Pod共享密钥

  # 在：/tmp/certs/client-0 您将找到

  # client.crt 和 client.key

  # 我们可以使用这些文件在k8s中设置KES

  externalClientCertSecrets

  - name: kes-minio

type: Opaque

  - name: kes-minio-public

type: Opaque

  env

# 设置 MINIO_KMS_KES_ENDPOINT

# 它是Ubuntu Pod的IP地址。

- name: MINIO_KMS_KES_ENDPOINT

   value: "https://<UBUNTU POD 的 IP 地址>:7373"

# 设置MinIO客户端凭据，它来自kes-minio密钥

- name: MINIO_KMS_KES_CERT_FILE

   value: "/tmp/certs/client-0/client.crt"

# 设置MinIO客户端凭据，它来自kes-minio密钥

- name: MINIO_KMS_KES_KEY_FILE

   value: "/tmp/certs/client-0/client.key"

# 设置MinIO默认密钥

- name: MINIO_KMS_KES_KEY_NAME

   value: "minio-default-key"

# 信任KES服务器证书，它来自kes-minio-public密钥

- name: MINIO_KMS_KES_CAPATH

   value: "/tmp/certs/client-1/client.crt"

# 根用户

- name: MINIO_ROOT_USER

   value: minio

# 根密码：

- name: MINIO_ROOT_PASSWORD

   value: minio123

mc alias set myminio https://minio.tenant-lite.svc.cluster.local:443 minio minio123

mc rb myminio/my-bucket --force # 删除之前的存储桶以全新开始

mc mb myminio/my-bucket # 创建新的存储桶

mc admin kms key create myminio minio-my-bucket # 创建密钥

mc encrypt set sse-kms minio-my-bucket myminio/my-bucket # 加密存储桶

root@ubuntu:/# mc alias set myminio https://minio.tenant-lite.svc.cluster.local:443 minio minio123

已成功添加 `myminio`。

root@ubuntu:/# mc rb myminio/my-bucket --force # 删除之前的存储桶以全新开始

mc: <ERROR> 无法验证目标 `myminio/my-bucket`。存储桶 `my-bucket` 不存在。

root@ubuntu:/# mc mb myminio/my-bucket # 创建新的存储桶

存储桶 `myminio/my-bucket` 创建成功。

root@ubuntu:/# mc admin kms key create myminio minio-my-bucket # 创建密钥

已成功创建主密钥 `minio-my-bucket`

root@ubuntu:/# mc encrypt set sse-kms minio-my-bucket myminio/my-bucket # 加密存储桶

已成功为 myminio/my-bucket 设置自动加密配置

root@ubuntu:/#