ceph osd pool create ceph_k8s
ceph osd pool create metadata_ceph_k8s
ceph fs new cephfs_k8s metadata_ceph_k8s ceph_k8s #用创建好的资源池组成cephfs
ceph fs subvolume  create cephfs_k8s csi #创建子卷

USER=csi-cephfs
FS_NAME=cephfs
SUB_VOL=csi
ceph auth get-or-create client.$USER \
  mgr "allow rw" \
  osd "allow rw tag cephfs metadata=$FS_NAME, allow rw tag cephfs data=$FS_NAME" \
  mds "allow r fsname=$FS_NAME path=/volumes, allow rws fsname=$FS_NAME path=/volumes/$SUB_VOL" \
  mon "allow r fsname=$FS_NAME"

#ceph mon dump
epoch 3
fsid ec3f3a04-ea05-11ef-a686-bc2411ba81e9 #FSID --required
last_changed 2025-02-13T12:34:06.030850+0000
created 2025-02-13T12:28:28.092545+0000
min_mon_release 16 (pacific)
election_strategy: 1 #monIP below --required
0: [v2:10.0.0.13:3300/0,v1:10.0.0.13:6789/0] mon.ceph01
1: [v2:10.0.0.15:3300/0,v1:10.0.0.15:6789/0] mon.ceph03
2: [v2:10.0.0.14:3300/0,v1:10.0.0.14:6789/0] mon.ceph02
dumped monmap epoch 3

# ceph fs ls 
#需要FSname（cephfs_k8s)以及data pools名字（ceph_k8s)，不要混淆！
name: cephfs_k8s, metadata pool: metadata_ceph_k8s, data pools: [ceph_k8s ]
​
# ceph fs subvolume ls cephfs_k8s #确认子卷已经创建
[
    {
        "name": "csi"
    }
]

#ceph auth get client.k8s 需要key。
[client.k8s]
        key = AQA5m7lntJ60BxAAKbT5dbeltplkiU9xZETyvQ==
        caps mds = "allow r fsname=cephfs_k8s path=/volumes, allow rws fsname=cephfs_k8s path=/volumes"
        caps mgr = "allow rw"
        caps mon = "allow r fsname=cephfs_k8s"
        caps osd = "allow rw tag cephfs metadata=cephfs_k8s, allow rw tag cephfs data=cephfs_k8s"
exported keyring for client.k8s

git clone https://github.com/ceph/ceph-csi.git
cd ceph-csi/deploy/cephfs/kubernetes/

kubeadm create ns csi
sed -i 's/namespace: default/namespace: csi/g'

---
apiVersion: v1
kind: ConfigMap
metadata:
  name: "ceph-csi-config"
data:
  config.json: |-
    [
      {
        "clusterID": "ec3f3a04-ea05-11ef-a686-bc2411ba81e9",
        "monitors": [
          "10.0.0.13:6789",
          "10.0.0.14:6789",
          "10.0.0.15:6789"
        ]
      }
    ]

---
apiVersion: storage.k8s.io/v1
kind: CSIDriver
metadata:
  name: "cephfs.csi.ceph.com"
spec:
  attachRequired: false
  podInfoOnMount: false
  fsGroupPolicy: File
  seLinuxMount: true
[root@k8s-master kubernetes]# cat csi-secret.yaml
---
apiVersion: v1
kind: Secret
metadata:
  name: csi-secret
  namespace: csi
stringData:
  # Required for statically and dynamically provisioned volumes
  # The userID must not include the "client." prefix!
  userID: k8s
  userKey: AQA5m7lntJ60BxAAKbT5dbeltplkiU9xZETyvQ==
​
  # Encryption passphrase
  encryptionPassphrase: test_passphrase

---
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: csi-cephfs-sc
  namespace: default
  annotations:
        storageclass.kubernetes.io/is-default-class: "true"
provisioner: cephfs.csi.ceph.com
parameters:
  clusterID: ec3f3a04-ea05-11ef-a686-bc2411ba81e9 #fsID
  fsName: cephfs_k8s #fsName
  pool: ceph_k8s #poolname
  #认证密钥名
  csi.storage.k8s.io/provisioner-secret-name: csi-secret
  csi.storage.k8s.io/provisioner-secret-namespace: csi
  csi.storage.k8s.io/controller-expand-secret-name: csi-secret
  csi.storage.k8s.io/controller-expand-secret-namespace: csi
  csi.storage.k8s.io/node-stage-secret-name: csi-secret
  csi.storage.k8s.io/node-stage-secret-namespace: csi
  mounter: kernel
reclaimPolicy: Delete
allowVolumeExpansion: true

ls *.yaml | xargs -I {} kubectl apply -n csi -f {} --validate=false