1
root@Ceph01 ~# ceph osd pool create opsfs_data
2
--autoscale_mode  #   PG自动伸缩默认是开启的~
3
# 一个是数据池，另一个是元数据池
4
root@Ceph01 ~# ceph osd pool create opsfs_metadata
5
root@Ceph01 ~# ceph osd pool set opsfs_data bulk true        # 标记为大容量池（PG自动伸缩）
6
root@Ceph01 ~# ceph osd pool ls detail | grep opsfs
7
`需要等一会才会出结果！`
8
pool 7 'opsfs_data' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 256 pgp_num 129 pgp_num_target 256 autoscale_mode on last_change 264 lfor 0/0/264 flags hashpspool,bulk stripe_width 0 read_balance_score 2.04
9
# pg_num 256 pgp_num 129 (延迟30s)
10
pool 8 'opsfs_metadata' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode on last_change 262 lfor 0/0/260 flags hashpspool stripe_width 0 read_balance_score 2.81
11

12
# 创建一个文件系统
13
ceph fs new <文件系统名> <元数据池> <数据池>
14
         ↓        ↓       ↓          ↓
15
       新建    起个名    存文件索引  存文件内容
16
root@Ceph01 ~# ceph fs new ops-cephfs opsfs_metadata opsfs_data
17

18
`ceph orch apply mds <fs-name>`
19
# "让编排器给 ops-cephfs 这个文件系统拉起 MDS 进程"
20
root@Ceph01 ~# ceph orch apply mds ops-cephfs
21
  1. 会启动两个 MDS 守护进程（分别在两台主机上）
22
  2. 一个设为 active，一个设为 standby；与 MGR 的高可用结构一致
23
  3. MDS 连到 opsfs_metadata 池，开始维护 inode 索引

1
# 看 MDS 进程整体概况（所有）
2
root@Ceph01 ~# ceph mds stat
3
ops-cephfs:1 {0=ops-cephfs.Ceph02.xlaleh=up:active} 1 up:standby
4

5
# 查看某个文件系统的详细信息
6
root@Ceph01 ~# ceph fs status ops-cephfs
7
ops-cephfs - 0 clients
8
==========
9
RANK  STATE             MDS                ACTIVITY     DNS    INOS   DIRS   CAPS
10
 0    active  ops-cephfs.Ceph02.xlaleh  Reqs:    0 /s    10     13     12      0
11
     POOL         TYPE     USED  AVAIL
12
opsfs_metadata  metadata   157k  56.5G
13
  opsfs_data      data       0   56.5G
14
      STANDBY MDS
15
ops-cephfs.Ceph01.rltckv
16
MDS version: ceph version 20.2.2 (0f...c) tentacle (stable)
17
# Ceph02活跃，Ceph01备用
18

19
root@Ceph01 ~# ceph fs ls
20
name: ops-cephfs, metadata pool: opsfs_metadata, data pools: [opsfs_data ]
21

22
root@Ceph01 ~# ceph -s | grep mds
23
    mds: 1/1 daemons up, 1 standby

1
# Ceph03 同样挂载
2
root@Ceph03 ~# mkdir -p /mnt/cephfs
3
root@Ceph03 ~# mount -t ceph 10.0.0.3:6789,10.0.0.4:6789,10.0.0.5:6789:/ /mnt/cephfs   -o name=cephfs,secret=AQA5wGRqh4c+IRAAw4a9Nh3Ukek0A4ziYVgREg==
4
# 这个也是可读写的用户
5
root@Ceph03 ~# df -h | grep cephfs
6
10.0.0.3:6789,10.0.0.4:6789,10.0.0.5:6789:/   57G     0   57G   0% /mnt/cephfs
7

8
# Ceph03 写入
9
root@Ceph03 ~# cp /etc/os-release /mnt/cephfs/
10
root@Ceph03 ~# ls /mnt/cephfs/
11
fstab  hosts  os-release
12

13
# Ceph02 & Ceph03 数据一致（内容共享）
14
root@Ceph02 ~# ls /mnt/cephfs/
15
fstab  hosts  os-release

1
root@Ceph03 ~# mkdir /mnt/cephfs-ro
2

3
# Ceph03 上用只读用户挂载
4
root@Ceph03 ~# mount -t ceph 10.0.0.3:6789,10.0.0.4:6789,10.0.0.5:6789:/ /mnt/cephfs-ro   -o name=cephfs-ro,secret=AQBBwGRqWp0HBRAALDOXr+Zvj7TnVoQCOUBWwA==
5

6
root@Ceph03 ~# df -h | grep cephfs
7
10.0.0.3:6789,10.0.0.4:6789,10.0.0.5:6789:/   57G     0   57G   0% /mnt/cephfs
8
10.0.0.3:6789,10.0.0.4:6789,10.0.0.5:6789:/   57G     0   57G   0% /mnt/cephfs-ro
9
root@Ceph03 ~# ls /mnt/cephfs
10
fstab  hosts  os-release  # 读写
11
root@Ceph03 ~# ls /mnt/cephfs-ro
12
fstab  hosts  os-release  #⚠️ 只读
13
root@Ceph03 ~# cp /etc/group /mnt/cephfs-ro/  # ❌ Permission denied
14
cp: cannot create regular file '/mnt/cephfs-ro/group': Permission denied
15
root@Ceph03 ~# cp /etc/group /mnt/cephfs  # ✅ 读写

1
# 以Ceph02为例
2
root@Ceph02 ~# tree /mnt/cephfs/
3
/mnt/cephfs/
4
├── fstab
5
├── group
6
├── hosts
7
└── os-release
8
root@Ceph02 ~# mkdir -p /mnt/cephfs/data
9
root@Ceph02 ~# mkdir -p /mnt/cephfs/resources
10
root@Ceph02 ~# touch /mnt/cephfs/data/index{1..2}.html
11
root@Ceph02 ~# touch /mnt/cephfs/resources/test{1..2}.php
12
root@Ceph02 ~# tree /mnt/cephfs/
13
/mnt/cephfs/
14
├── data
15
│   ├── index1.html
16
│   └── index2.html
17
├── fstab
18
├── group
19
├── hosts
20
├── os-release
21
└── resources
22
    ├── test1.php
23
    └── test2.php
24

25
# 我们只去挂载根下的/data
26
root@Ceph02 ~# mkdir /mnt/path
27
root@Ceph02 ~# mount -t ceph 10.0.0.3:6789,10.0.0.4:6789,10.0.0.5:6789:/data /mnt/path   -o name=cephfs,secret=AQA5wGRqh4c+IRAAw4a9Nh3Ukek0A4ziYVgREg==
28
root@Ceph02 ~# ls /mnt/path/
29
index1.html  index2.html
30
# data目录下只有html页面

1
# 查看linux内核是否有这个ceph模块
2
root@Ceph02 ~# modinfo ceph
3
filename:       /lib/modules/6.8.0-136-generic/kernel/fs/ceph/ceph.ko.zst
4
license:        GPL
5
description:    Ceph filesystem for Linux
6
author:         Patience Warnick <patience@newdream.net>
7
author:         Yehuda Sadeh <yehuda@hq.newdream.net>
8
.................
9
# 查看正在启用的内核模块（已经挂载上CephFS）
10
root@Ceph02 ~# lsmod | grep ceph
11
ceph                  626688  1
12
libceph               548864  1 ceph
13
netfs                 512000  1 ceph
14
libcrc32c              12288  6 nf_conntrack,nf_nat,btrfs,nf_tables,raid456,libceph
15

16
# 1. 安装 fuse 客户端工具
17
apt -y install ceph-fuse
18

19
# 2. 服务端导出用户 keyring
20
ceph auth export client.cephfs -o ceph.client.cephfs.keyring
21
scp ceph.client.cephfs.keyring root@10.0.0.4:/tmp/
22

23
# 3. 客户端挂载
24
mkdir -p /mnt/cephfs
25
ceph-fuse -n client.cephfs -m 10.0.0.3:6789,10.0.0.4:6789,10.0.0.5:6789/ /mnt/cephfs \
26
  -c /tmp/ceph.client.cephfs.keyring
27
# -n 用户名    -m Monitor地址    -c 指定keyring文件位置

1
1)查看集群现状
2
root@Ceph01 ~# ceph -s
3
  cluster:
4
    id:     a9b741cd-856a-11f1-b216-525400c6b9fa
5
    health: HEALTH_OK
6
  services:
7
    mon: 3 daemons, quorum Ceph01,Ceph02,Ceph03 (age 70s) [leader: Ceph01]
8
    mgr: Ceph01.stlhbi(active, since 59s), standbys: Ceph02.lbqijv
9
    osd: 9 osds: 9 up (since 59s), 9 in (since 4w)
10
'确认集群健康，9 个 OSD 已就位'
11

12
2)部署 rgw 服务(服务名用 kpyun，落 Ceph01/Ceph02 两台，端口 80)
13
root@Ceph01 ~# ceph orch apply rgw kpyun --placement="2 Ceph01 Ceph02" --port 80
14
Scheduled rgw.kpyun update...
15
'cephadm 会自动拉起容器、建池、注册服务'
16

17
3)查看 rgw 组件是否起来
18
root@Ceph01 ~# ceph -s
19
  services:
20
    mon: 3 daemons, quorum Ceph01,Ceph02,Ceph03 (age 70s) [leader: Ceph01]
21
    mgr: Ceph01.stlhbi(active, since 59s), standbys: Ceph02.lbqijv
22
    osd: 9 osds: 9 up (since 59s), 9 in (since 4w)
23
    rgw: 2 daemons active (2 hosts, 1 zones)   # 👈 出现 rgw 了
24

25
4)看 rgw 守护进程详情
26
root@Ceph01 ~# ceph orch ps --service_name rgw.kpyun
27
NAME                     HOST    PORTS  STATUS         REFRESHED  AGE  MEM USE  VERSION
28
rgw.kpyun.Ceph01.saocsj  Ceph01  *:80   running (33s)    23s ago  33s   121M     20.2.2
29
rgw.kpyun.Ceph02.zjvuxs  Ceph02  *:80   running (30s)    24s ago  30s   120M     20.2.2
30
'两台各一个 rgw 容器，都监听 80'
31

32
5)看 rgw 自动建的池
33
root@Ceph01 ~# ceph osd pool ls | grep -i rgw
34
.rgw.root
35
default.rgw.log
36
default.rgw.control
37
default.rgw.meta
38
'就这 4 个，object 数据池(default.rgw.buckets.data)等用到时才出现'
39

40
6)访问 WebUI(HTTP 接口)
41
root@Ceph01 ~# curl -s -I  http://10.0.0.3/ | head -1
42
HTTP/1.1 200 OK
43
✅️ 返回 200
44

45
http://10.0.0.3/
46
http://10.0.0.4/

1
1)安装 s3cmd 工具包(Ubuntu 用 apt)
2
root@Ceph01 ~# apt -y install s3cmd
3
root@Ceph01 ~# s3cmd --version
4
s3cmd version 2.4.0
5
'装好，版本 2.4.0'
6

7
2)创建 rgw 账号(在 Ceph01 上用 radosgw-admin)
8
root@Ceph01 ~# radosgw-admin user create --uid "jiuzhao" --display-name "jzops"
9
--uid      # 用户ID（系统唯一标识）  ❌ 全局唯一，不可重复
10
--display-name  # 显示名称（昵称/别名）  ✅ 可以重复，建议唯一
11
{
12
    "user_id": "jiuzhao",
13
    "display_name": "jzops",
14
    "email": "",
15
    "suspended": 0,
16
    "max_buckets": 1000,
17
    "subusers": [],
18
    "keys": [
19
        {
20
            "user": "jiuzhao",
21
            "access_key": "42GA8G0I7JGTVGX43NPK",
22
            "secret_key": "8pJrGe843L6C8xemXAi3RhA2OE28WIguFcjKg247",
23
            "active": true,
24
            "create_date": "2026-08-25T10:29:15.951471Z"
25
        }
26
    ],
27
    ...
28
}
29
`拿到 access_key 和 secret_key，后面 s3cmd 要用`
30

31
3)写 s3cmd 配置文件 /root/.s3cfg
32
root@Ceph01 ~# cat > /root/.s3cfg <<'EOF'
33
[default]
34
access_key = 42GA8G0I7JGTVGX43NPK
35
secret_key = 8pJrGe843L6C8xemXAi3RhA2OE28WIguFcjKg247
36
host_base = 10.0.0.3
37
host_bucket = 10.0.0.3/%(bucket)
38
use_https = False
39
EOF
40
'host_base 直接填 rgw 的 IP(10.0.0.3)，不配域名也能用'
41
'host_bucket 用 路径风格: 10.0.0.3/%(bucket)，因为没配 rgw dns name'
42

43
4)测试连通性
44
root@Ceph01 ~# s3cmd ls
45
'现在还没有 bucket，所以没输出，但只要不报错就说明 key 对了'
46
✅️ 配置成功

1
root@Ceph01 ~# s3cmd --configure
2

3
Enter new values or accept defaults in brackets with Enter.
4
Refer to user manual for detailed description of all options.
5

6
Access key and Secret key are your identifiers for Amazon S3. Leave them empty for using the env variables.
7
Access Key [42GA8G0I7JGTVGX43NPK]: # rgw账号的access_key
8
Secret Key [8pJrGe843L6C8xemXAi3RhA2OE28WIguFcjKg247]: # rgw账号的secret_key
9
Default Region [US]: # 直接回车即可
10

11
Use "s3.amazonaws.com" for S3 Endpoint and not modify it to the target Amazon S3.
12
S3 Endpoint [10.0.0.3]: # 用于访问rgw的地址
13

14
Use "%(bucket)s.s3.amazonaws.com" to the target Amazon S3. "%(bucket)s" and "%(location)s" vars can be used
15
if the target S3 system supports dns based buckets.
16
DNS-style bucket+hostname:port template for accessing a bucket [10.0.0.3/%(bucket)]: # 设置DNS解析风格
17

18
Encryption password is used to protect your files from reading
19
by unauthorized persons while in transfer to S3
20
Encryption password: # 文件不加密，直接回车即可
21
Path to GPG program [/usr/bin/gpg]: # 指定自定义的gpg程序路径，直接回车即可
22

23
When using secure HTTPS protocol all communication with Amazon S3
24
servers is protected from 3rd party eavesdropping. This method is
25
slower than plain HTTP, and can only be proxied with Python 2.7 or newer
26
Use HTTPS protocol [No]: # 你的rgw是否是https，如果不是设置为No
27

28
On some networks all internet access must go through a HTTP proxy.
29
Try setting it here if you can't connect to S3 directly
30
HTTP Proxy server name: # 代理服务器的地址，我并没有配置代理服务器，因此直接回车即可
31

32
New settings:  # 注意，下面的信息是上面咱们填写时一个总的预览信息
33
  Access Key: 42GA8G0I7JGTVGX43NPK
34
  Secret Key: 8pJrGe843L6C8xemXAi3RhA2OE28WIguFcjKg247
35
  Default Region: US
36
  S3 Endpoint: 10.0.0.3
37
  DNS-style bucket+hostname:port template for accessing a bucket: 10.0.0.3/%(bucket)
38
  Encryption password:
39
  Path to GPG program: /usr/bin/gpg
40
  Use HTTPS protocol: False
41
  HTTP Proxy server name:
42
  HTTP Proxy server port: 0
43

44
Test access with supplied credentials? [Y/n] Y  # 如果确认上述信息没问题的话，则输入字母Y即可
45
Please wait, attempting to list all buckets...
46
Success. Your access key and secret key worked fine :-)
47

48
Now verifying that encryption works...
49
Not configured. Never mind.
50

51
Save settings? [y/N] y  # 保存配置
52
Configuration saved to '/root/.s3cfg'

1
1)创建一个新的存储桶 bucket
2
root@Ceph01 ~# s3cmd mb s3://kpyun-bucket
3
# mb: 这是 s3cmd 的一个子命令，是 "make bucket" 的缩写，专门用来创建新的存储桶
4
Bucket 's3://kpyun-bucket/' created
5
root@Ceph01 ~# s3cmd ls
6
2026-08-29 02:21  s3://kpyun-bucket
7
✅️ bucket 建好了
8

9
2)准备测试文件并上传
10
root@Ceph01 ~# ls /tmp | egrep 'bili|png'
11
bilibili.webm  `视频`
12
s.png      `图片`
13
root@Ceph01 ~# echo "hello kpyun ceph storage" > /tmp/s3test.txt
14
root@Ceph01 ~# s3cmd put /tmp/s3test.txt /tmp/{bilibili.webm,s.png} s3://kpyun-bucket
15
pload: '/tmp/s3test.txt' -> 's3://kpyun-bucket/s3test.txt'
16
upload: '/tmp/bilibili.webm' -> 's3://kpyun-bucket/bilibili.webm'
17
upload: '/tmp/s.png' -> 's3://kpyun-bucket/s.png'
18
===================================
19
`s3cmd 使用教程`
20
root@Ceph01 ~# s3cmd --help
21
Commands:
22
  Make bucket
23
      s3cmd mb s3://BUCKET
24
  Remove bucket ✅ '删除整个存储桶及其所有元数据'
25
      s3cmd rb s3://BUCKET
26
      # 先递归删除桶里所有文件，再删除桶本身：s3cmd rb -r s3://kpyun-backup
27
      `桶被彻底删除，无法恢复`
28
  List objects or buckets ✅ '列出所有存储桶（不带路径时）'
29
      s3cmd ls [s3://BUCKET[/PREFIX]]
30
  List all object in all buckets ✅ '列出所有存储桶中的所有对象（文件）'
31
      s3cmd la
32
      # s3cmd la 命令的后面不应该再跟具体的存储桶路径；正确的用法应该是直接执行 s3cmd la
33
      # 要准确检查 某个存储通 是否已被清空，你应该使用专门针对单个存储桶的 ls 命令，而不是 la
34
  Put file into bucket ✅ '也可以是多个文件'
35
      s3cmd put FILE [FILE...] s3://BUCKET[/PREFIX]
36
  Get file from bucket ✅ '从 S3 存储桶下载文件到本地'
37
      s3cmd get s3://BUCKET/OBJECT LOCAL_FILE
38
  Disk usage by buckets ✅ '查看 存储桶/对象 磁盘使用量'
39
      s3cmd du [s3://BUCKET[/PREFIX]]
40
  Delete file from bucket ✅ `从 S3 存储桶中删除指定文件`
41
      s3cmd del s3://BUCKET/OBJECT
42
  Delete file from bucket (alias for del) ✅ `从 S3 存储桶中删除指定文件`
43
      s3cmd rm s3://BUCKET/OBJECT  '两者等价，rm 是 del 的别名（alias），执行效果一模一样'
44
    # 删除所有文件：s3cmd rm s3://kpyun-backup/*
45
    `桶变为空桶，但桶本身还在，可以继续往这个桶里上传新文件`
46

47
3)下载回来验证内容一致
48
root@Ceph01 ~# s3cmd get s3://kpyun-bucket/s3test.txt ./
49
download: 's3://kpyun-bucket/s3test.txt' -> './s3test.txt'
50
root@Ceph01 ~# cat ./s3test.txt
51
hello kpyun ceph storage
52
'内容一致!'
53

54
4)查看 bucket 内容 / 对象信息 / 大小
55
root@Ceph01 ~# s3cmd la
56
2026-08-29 02:37      2053779  s3://kpyun-bucket/bilibili.webm
57
2026-08-29 02:37       582512  s3://kpyun-bucket/s.png
58
2026-08-29 02:34           25  s3://kpyun-bucket/s3test.txt
59

60
root@Ceph01 ~# s3cmd info s3://kpyun-bucket/s3test.txt
61
s3://kpyun-bucket/s3test.txt (object):
62
   File size: 25
63
   Last mod:  Sat, 29 Aug 2026 02:34:17 GMT
64
   MIME type: text/plain
65
   Storage:   STANDARD
66
   MD5 sum:   8ddbf928658ad38b5ffdcf5618ae8d28
67
   SSE:       none
68
   Policy:    none
69
   CORS:      none
70
   ACL:       jzops: FULL_CONTROL
71

72
root@Ceph01 ~# s3cmd du --human-readable s3://kpyun-bucket
73
--human-readable = -H  # 以人类可读格式显示
74
   2M       3 objects s3://kpyun-bucket/
75
`这里是查看整个存储桶大小，可以查看 单个文件/某个路径下所有文件 的大小`

1
5)大文件分片上传(默认 15MB 一片)
2
root@Ceph01 ~# dd if=/dev/zero of=/tmp/big.bin bs=1M count=40
3
root@Ceph01 ~# ls -lh /tmp/big.bin
4
-rw-r--r-- 1 root root 40M Aug 29 11:05 /tmp/big.bin
5
root@Ceph01 ~# s3cmd put /tmp/big.bin s3://kpyun-bucket
6
upload: '/tmp/big.bin' -> 's3://kpyun-bucket/big.bin'  [part 1 of 3, 15MB🔥] [1 of 1]
7
upload: '/tmp/big.bin' -> 's3://kpyun-bucket/big.bin'  [part 2 of 3, 15MB🔥] [1 of 1]
8
upload: '/tmp/big.bin' -> 's3://kpyun-bucket/big.bin'  [part 3 of 3, 10MB🔥] [1 of 1]
9
'40M 文件被自动按 15MB 切片上传'
10

11
6)桶间拷贝 / 移动 / 目录同步
12
root@Ceph01 ~# s3cmd mb s3://kpyun-backup
13
Bucket 's3://kpyun-backup/' created
14
root@Ceph01 ~# s3cmd ls | grep kpyun-backup
15
2026-08-29 03:08  s3://kpyun-backup
16

17
root@Ceph01 ~# s3cmd cp s3://kpyun-bucket/big.bin s3://kpyun-backup
18
remote copy: 's3://kpyun-bucket/big.bin' -> 's3://kpyun-backup/big.bin'  [1 of 1]
19

20
root@Ceph01 ~# s3cmd mv s3://kpyun-bucket/s3test.txt s3://kpyun-backup
21
move: 's3://kpyun-bucket/s3test.txt' -> 's3://kpyun-backup/s3test.txt'  [1 of 1]
22
root@Ceph01 ~# s3cmd ls -H s3://kpyun-backup
23
-H = --human-readable  # 用短选项！
24
2026-08-29 03:08    40M  s3://kpyun-backup/big.bin
25
2026-08-29 03:13    25   s3://kpyun-backup/s3test.txt
26

27
root@Ceph01 ~# s3cmd sync /var/log/ --exclude '*' --include 'auth.*' --dry-run s3://kpyun-backup
28
# 默认情况下，所有文件都会被包含（相当于隐含了一个 --include '*'）
29
-n = --dry-run  # 模拟运行（只显示会做什么，不实际执行）
30
✅ 正确的做法：先用 --exclude 排除所有，再用 --include 挑选， --dry-run 干跑一遍
31
exclude: .........  `排除了很多文件`
32
upload: '/var/log/auth.log' -> 's3://kpyun-backup/auth.log'
33
upload: '/var/log/auth.log.1' -> 's3://kpyun-backup/auth.log.1'
34
upload: '/var/log/auth.log.2.gz' -> 's3://kpyun-backup/auth.log.2.gz'
35
WARNING: Exiting now because of --dry-run
36
root@Ceph01 ~# s3cmd sync /var/log/ --exclude '*' --include 'auth.*' s3://kpyun-backup
37
upload: '/var/log/auth.log' -> 's3://kpyun-backup/auth.log'  [1 of 3]
38
upload: '/var/log/auth.log.1' -> 's3://kpyun-backup/auth.log.1'  [2 of 3]
39
upload: '/var/log/auth.log.2.gz' -> 's3://kpyun-backup/auth.log.2.gz'  [3 of 3]
40

41
root@Ceph01 ~# s3cmd ls -H s3://kpyun-bucket
42
2026-08-29 03:05    40M  s3://kpyun-bucket/big.bin
43
2026-08-29 02:37  2005K  s3://kpyun-bucket/bilibili.webm
44
2026-08-29 02:37   568K  s3://kpyun-bucket/s.png
45
root@Ceph01 ~# s3cmd ls -H s3://kpyun-backup
46
2026-08-29 03:45    64K  s3://kpyun-backup/auth.log
47
2026-08-29 03:45     3K  s3://kpyun-backup/auth.log.1
48
2026-08-29 03:45    26K  s3://kpyun-backup/auth.log.2.gz
49
2026-08-29 03:38    40M  s3://kpyun-backup/big.bin
50
2026-08-29 03:38    25   s3://kpyun-backup/s3test.txt
51
'cp/mv/sync 都正常，跨桶复制秒级完成'

1
1)写策略文件(允许任何人 GetObject)
2
root@Ceph01 ~# cat > /root/anon-policy.json <<'EOF'
3
{
4
  // 版本号：策略语言的版本，固定用 "2012-10-17"，这是AWS IAM策略的最终版本，不用改
5
  "Version": "2012-10-17",
6

7
  // 策略语句数组：可以包含多条规则
8
  "Statement": [
9
    {
10
      // Effect: 效果，Allow 表示"允许"，Deny 表示"拒绝"
11
      "Effect": "Allow",
12

13
      // Principal: 授权主体，{"AWS": ["*"]} 表示"允许所有人（匿名用户）"
14
      // 这里的 "*" 是通配符，代表所有AWS用户（包括未认证的）
15
      "Principal": {"AWS": ["*"]},
16

17
      // Action: 允许执行的操作，"s3:GetObject" 表示"下载/读取对象"
18
      // 其他常见操作：s3:PutObject（上传）、s3:DeleteObject（删除）、s3:ListBucket（列出）
19
      "Action": "s3:GetObject",
20

21
      // Resource: 资源范围，指定这个策略应用到哪些对象上
22
      // "arn:aws:s3:::kpyun-bucket/*" 表示 kpyun-bucket 这个桶里的所有对象
23
      // 注意：arn:aws:s3::: 是AWS的资源命名规范，固定写法
24
      "Resource": ["arn:aws:s3:::kpyun-bucket/*"]
25
    }
26
  ]
27
}
28
EOF

1
2)应用策略
2
root@Ceph01 ~# s3cmd info s3://kpyun-bucket/ | grep -i policy
3
   Policy:    none
4
root@Ceph01 ~# s3cmd setpolicy /root/anon-policy.json s3://kpyun-bucket
5
# 把这个策略文件应用（设置）到 kpyun-bucket 这个存储桶上
6
s3://kpyun-bucket/: Policy updated
7
root@Ceph01 ~# s3cmd info s3://kpyun-bucket/ | grep -A3 -i policy
8
   Policy:    {
9
  // 版本号：策略语言的版本，固定用 "2012-10-17"，这是AWS IAM策略的最终版本，不用改
10
  "Version": "2012-10-17",
11
  ......  # 现在就有自己的策略了
12

13
3)匿名访问验证(这次不带 key)
14
http://10.0.0.3/kpyun-bucket/bilibili.webm    # 视频
15
http://10.0.0.3/kpyun-bucket/s.png        # 图片

1
4)删掉策略
2
root@Ceph01 ~# s3cmd delpolicy s3://kpyun-bucket
3
s3://kpyun-bucket/: Policy deleted
4
root@Ceph01 ~# curl -s http://10.0.0.3/kpyun-bucket/s.png
5
<Code>AccessDenied</Code><Message>missing s3:GetObject permission</Message>
6
'访问被拒绝，因为缺少 s3:GetObject 权限'
7
root@Ceph01 ~# s3cmd info s3://kpyun-bucket | grep Policy
8
   Policy:    none
9

10
5)应用到 kpyun-backup 桶
11
root@Ceph01 ~# sed -i s#kpyun-bucket#kpyun-backup# /root/anon-policy.json
12
root@Ceph01 ~# s3cmd setpolicy /root/anon-policy.json s3://kpyun-backup
13
s3://kpyun-backup/: Policy updated
14
root@Ceph01 ~# curl -s http://10.0.0.3/kpyun-backup/s3test.txt
15
hello kpyun ceph storage
16
✅️ 不用 secret_key 也能读到
17

18
6)删除桶 + 桶中的文件
19
root@Ceph01 ~# s3cmd ls s3://kpyun-backup | wc -l
20
5
21
root@Ceph01 ~# s3cmd rb -r s3://kpyun-backup
22
-r = --recursive  # 递归操作（处理子目录）
23
WARNING: Bucket is not empty. Removing all the objects from it first. This may take some time...
24
delete: 's3://kpyun-backup/auth.log'
25
delete: 's3://kpyun-backup/auth.log.1'
26
delete: 's3://kpyun-backup/auth.log.2.gz'
27
delete: 's3://kpyun-backup/big.bin'
28
delete: 's3://kpyun-backup/s3test.txt'
29
Bucket 's3://kpyun-backup/' removed
30
`彻底删除整个桶（包括里面的所有文件）`

1
1)准备隔离环境
2
jiuzhao@Ubuntu ~$ pipx --version
3
1.8.0
4
jiuzhao@Ubuntu ~$ uv --version
5
uv 0.11.29 (x86_64-unknown-linux-gnu)
6
jiuzhao@Ubuntu ~$ uv venv test --python 3.14
7
Using CPython 3.14.4 interpreter at: /usr/bin/python3.14
8
Creating virtual environment at: test
9
Activate with: source test/bin/activate
10
jiuzhao@Ubuntu ~$ source ./test/bin/activate
11
(test) jiuzhao@Ubuntu ~$ uv pip install -i https://pypi.tuna.tsinghua.edu.cn/simple boto3
12
# 用清华镜像
13

14
2)写操作脚本
15
(test) jiuzhao@Ubuntu ~$ cat > ./test/rgw-kpyun.py <<'EOF'
16
# 导入boto3库，这是AWS官方Python SDK，用于操作S3对象存储
17
import boto3
18
# 从botocore导入Config类，用于配置客户端参数
19
from botocore.client import Config
20

21
# 设置访问密钥（Access Key和Secret Key）
22
# 这些相当于用户名和密码，用于身份认证
23
access_key = "42GA8G0I7JGTVGX43NPK"
24
secret_key = "8pJrGe843L6C8xemXAi3RhA2OE28WIguFcjKg247"
25

26
# 创建S3客户端对象，用于后续操作
27
# 这个客户端相当于一个"连接器"，连接到S3服务
28
s3 = boto3.client("s3",
29
    # endpoint_url指定S3服务的访问地址（这里是本地的RGW服务）
30
    endpoint_url="http://10.0.0.3",
31
    # 传入访问密钥
32
    aws_access_key_id=access_key,
33
    aws_secret_access_key=secret_key,
34
    # 指定使用签名版本4（S3v4），这是较新的认证方式
35
    config=Config(signature_version="s3v4"),
36
)
37

38
# 创建一个名为"kpyun-rgw"的存储桶（Bucket）
39
# 存储桶类似于文件系统中的根目录，用来存放对象
40
s3.create_bucket(Bucket="kpyun-rgw")
41

42
# 列出所有存储桶并打印出来
43
print("=== buckets ===")
44
# list_buckets()返回所有存储桶列表，["Buckets"]取出这个列表
45
for b in s3.list_buckets()["Buckets"]:
46
    # 打印每个存储桶的名称和创建时间
47
    print("{name}\t{created}".format(name=b["Name"], created=b["CreationDate"]))
48

49
# 在"kpyun-rgw"存储桶中上传一个对象（文件）
50
# put_object相当于上传文件，Key是文件名，Body是文件内容
51
s3.put_object(Bucket="kpyun-rgw", Key="test.txt", Body=b"Hello World")
52

53
# 列出"kpyun-rgw"存储桶中的所有对象并打印
54
print("=== objects in kpyun-rgw ===")
55
# list_objects_v2列出指定存储桶中的所有对象，["Contents"]取出对象列表
56
for o in s3.list_objects_v2(Bucket="kpyun-rgw")["Contents"]:
57
    # 打印每个对象的文件名和大小
58
    print("{name}\t{size}".format(name=o["Key"], size=o["Size"]))
59

60
# 从存储桶中下载"test.txt"对象
61
# get_object相当于下载文件
62
obj = s3.get_object(Bucket="kpyun-rgw", Key="test.txt")
63
print("=== get test.txt ===")
64
# 读取对象内容（Body是文件内容），然后decode()将字节转为字符串打印
65
print(obj["Body"].read().decode())
66
EOF
67

68
3)运行
69
(test) jiuzhao@Ubuntu ~$ python ./test/rgw-kpyun.py
70
=== buckets ===
71
kpyun-bucket  2026-08-29 02:21:44.625000+00:00
72
kpyun-rgw  2026-08-29 10:38:04.612000+00:00
73
=== objects in kpyun-rgw ===
74
test.txt  11
75
=== get test.txt ===
76
Hello World
77
✅️ 建桶、写对象、读对象一条龙通了
78

79
脚本执行条件：
80
  1)只需要 Python环境 + boto3库
81
  2)网络能连通 Ceph RGW 服务地址（http://10.0.0.3）
82
  3)有正确的访问密钥（代码里已写好）

1
1)创建子用户(挂在 jiuzhao 这个 user 下)
2
root@Ceph01 ~# radosgw-admin subuser create --uid=jiuzhao --subuser=jiuzhao:swift --access=full
3
`子用户不指定 --access 默认是「无权限」(empty)，必须显式指定`
4
# --access=full 是最高档，子用户能建容器、上传、删除全覆盖（读 + 写 + 管理）
5
{
6
    ...
7
    "swift_keys": [
8
        {
9
            "user": "jiuzhao:swift",
10
            "secret_key": "cBVi4iEZ8SG6XdVpKm6uQ9GljgsXMxFxhxj7e1qH",
11
            "active": true,
12
            "create_date": "2026-08-29T11:21:38.256534Z"
13
        }
14
    ]
15
}
16
'拿到 swift 子用户的 secret_key'
17

18
2)装 swift 客户端
19
(test) jiuzhao@Ubuntu ~$ uv pip install -i https://pypi.tuna.tsinghua.edu.cn/simple python-swiftclient==4.6.0
20
(test) jiuzhao@Ubuntu ~$ swift --version
21
python-swiftclient 4.6.0
22

23
3)建容器 + 上传文件(认证端点用 rgw 的 /auth)
24
(test) jiuzhao@Ubuntu ~$ swift -A http://10.0.0.3/auth -U jiuzhao:swift -K cBVi4iEZ8SG6XdVpKm6uQ9GljgsXMxFxhxj7e1qH post kpyun-swift
25
-A http://10.0.0.3/auth  # Auth URL 认证服务的地址，告诉swift去哪里验证身份
26
-U jiuzhao:swift    # User 用户名，格式是 账号:用户名，这里是 jiuzhao 账号下的 swift 用户
27
-K cBVi4iEZ8SG6Xd...  # Key 用户的认证密钥（相当于密码）
28
post          # 操作命令，创建或更新容器（这里是创建）
29
kpyun-swift        # 容器名称，要创建的容器名字
30
(test) jiuzhao@Ubuntu ~$ swift -A http://10.0.0.3/auth -U jiuzhao:swift -K cBVi4iEZ8SG6XdVpKm6uQ9GljgsXMxFxhxj7e1qH upload kpyun-swift /etc/os-release /etc/hosts /etc/fstab
31
# 用同样的认证信息登录，然后上传三个文件到 kpyun-swift 容器里
32
etc/os-release
33
etc/hosts
34
etc/fstab
35

36
4)查看容器内容
37
(test) jiuzhao@Ubuntu ~$ swift -A http://10.0.0.3/auth -U jiuzhao:swift -K cBVi4iEZ8SG6XdVpKm6uQ9GljgsXMxFxhxj7e1qH list kpyun-swift
38
etc/fstab
39
etc/hosts
40
etc/os-release
41
✅️ 三个文件进容器了
42

43
5)用环境变量简化(避免每次敲 -A/-U/-K)
44
(test) jiuzhao@Ubuntu ~$ cat > ./test/.swift <<'EOF'
45
export ST_AUTH=http://10.0.0.3/auth
46
export ST_USER=jiuzhao:swift
47
export ST_KEY=cBVi4iEZ8SG6XdVpKm6uQ9GljgsXMxFxhxj7e1qH
48
EOF
49
(test) jiuzhao@Ubuntu ~$ source ./test/.swift
50
(test) jiuzhao@Ubuntu ~$ swift list
51
kpyun-bucket
52
kpyun-rgw
53
kpyun-swift
54
🌿 重点看：swift list 把前面 s3cmd 建的 bucket 也列出来了!
55
'证明 S3 和 Swift 在 RGW 里共用同一套后端，并不是两套隔离的桶'
56
(test) jiuzhao@Ubuntu ~$ swift list kpyun-bucket --lh
57
--lh  # 这个参数就是专门用来显示人类可读格式的（显示详细信息：大小、时间、类型）
58
 40M 2026-08-29 03:05:33 application/octet-stream big.bin
59
2.0M 2026-08-29 02:37:28               video/webm bilibili.webm
60
568K 2026-08-29 02:37:28                image/png s.png
61
 42M