（1）安裝ceph-radosgw (安裝集群時，已經安裝過了，所以略過)
yum -y install ceph-radosgw
（2）部署
ceph-deploy rgw create c720181 c720182 c720183
（3）配置80端口（僅做參考，因為在生產環境會在rgw前面部署負載均衡，所以一般不會在這配置）
vim /etc/ceph/ceph.conf
……
[client.rgw.c720181]
rgw frontentds = "civetweb port=80"
sudo systemctl restart ceph-radosgw@rgw.c720181.service
（4）創建池
wget https://raw.githubusercontent.com/aishangwei/ceph-demo/master/ceph-deploy/rgw/pool
wget https://raw.githubusercontent.com/aishangwei/ceph-demo/master/ceph-deploy/rgw/create_pool.sh
[ceph-admin@c720181 my-cluster]$ cat pool
.rgw
.rgw.root
.rgw.control
.rgw.gc
.rgw.buckets
.rgw.buckets.index
.rgw.buckets.extra
.log
.intent-log
.usage
.users
.users.email
.users.swift
.users.uid
[ceph-admin@c720181 my-cluster]$ cat pool |wc -l
14
[ceph-admin@c720181 my-cluster]$ cat create_pool.sh
#!/bin/bash
PG_NUM=30
PGP_NUM=30
SIZE=3
for i in `cat ./pool`
do
ceph osd pool create $i $PG_NUM
ceph osd pool set $i size $SIZE
done
for i in `cat ./pool`
do
ceph osd pool set $i pgp_num $PGP_NUM
done
[ceph-admin@c720181 my-cluster]$ chmod +x create_pool.sh
[ceph-admin@c720181 my-cluster]$ ./create_pool.sh
pool '.rgw' created
set pool 6 size to 3
pool '.rgw.root' already exists
set pool 2 size to 3
pool '.rgw.control' created
set pool 7 size to 3
pool '.rgw.gc' created
set pool 8 size to 3
Error ERANGE: pg_num 30 size 3 would mean 840 total pgs, which exceeds max 750 (mon_max_pg_per_osd 250 * num_in_osds 3)
Error ENOENT: unrecognized pool '.rgw.buckets'
Error ERANGE: pg_num 30 size 3 would mean 840 total pgs, which exceeds max 750 (mon_max_pg_per_osd 250 * num_in_osds 3)
Error ENOENT: unrecognized pool '.rgw.buckets.index'
Error ERANGE: pg_num 30 size 3 would mean 840 total pgs, which exceeds max 750 (mon_max_pg_per_osd 250 * num_in_osds 3)
Error ENOENT: unrecognized pool '.rgw.buckets.extra'
Error ERANGE: pg_num 30 size 3 would mean 840 total pgs, which exceeds max 750 (mon_max_pg_per_osd 250 * num_in_osds 3)
Error ENOENT: unrecognized pool '.log'
Error ERANGE: pg_num 30 size 3 would mean 840 total pgs, which exceeds max 750 (mon_max_pg_per_osd 250 * num_in_osds 3)
Error ENOENT: unrecognized pool '.intent-log'
Error ERANGE: pg_num 30 size 3 would mean 840 total pgs, which exceeds max 750 (mon_max_pg_per_osd 250 * num_in_osds 3)
Error ENOENT: unrecognized pool '.usage'
Error ERANGE: pg_num 30 size 3 would mean 840 total pgs, which exceeds max 750 (mon_max_pg_per_osd 250 * num_in_osds 3)
Error ENOENT: unrecognized pool '.users'
Error ERANGE: pg_num 30 size 3 would mean 840 total pgs, which exceeds max 750 (mon_max_pg_per_osd 250 * num_in_osds 3)
Error ENOENT: unrecognized pool '.users.email'
Error ERANGE: pg_num 30 size 3 would mean 840 total pgs, which exceeds max 750 (mon_max_pg_per_osd 250 * num_in_osds 3)
Error ENOENT: unrecognized pool '.users.swift'
Error ERANGE: pg_num 30 size 3 would mean 840 total pgs, which exceeds max 750 (mon_max_pg_per_osd 250 * num_in_osds 3)
Error ENOENT: unrecognized pool '.users.uid'
set pool 6 pgp_num to 30
Error EINVAL: specified pgp_num 30 > pg_num 8
set pool 7 pgp_num to 30
set pool 8 pgp_num to 30
Error ENOENT: unrecognized pool '.rgw.buckets'
Error ENOENT: unrecognized pool '.rgw.buckets.index'
Error ENOENT: unrecognized pool '.rgw.buckets.extra'
Error ENOENT: unrecognized pool '.log'
Error ENOENT: unrecognized pool '.intent-log'
Error ENOENT: unrecognized pool '.usage'
Error ENOENT: unrecognized pool '.users'
Error ENOENT: unrecognized pool '.users.email'
Error ENOENT: unrecognized pool '.users.swift'
Error ENOENT: unrecognized pool '.users.uid'
報錯原因：每個osd最多只支持250個pg，之前創建了一個rbd池，pg總數是128。平均分配到3個osd，那已經占了43個。現在又新建了14個池，每個池占30個pg.總共需要420個pg，平均分到3個osd，每個osd占140.（這樣計算也不對）
Error ERANGE: pg_num 30 size 3 would mean 840 total pgs, which exceeds max 750 (mon_max_pg_per_osd 250 * num_in_osds 3)
處理辦法：刪除之前的池，然后修改腳本把pg數目設置小一點，再創建對象池。
這是由於沒有配置mon節點的 mon_allow_pool_delete 字段所致，解決辦法就是到mon節點進行相應的設置。
解決方案：
注：1-3步的操作必須在mon節點上執行
===================在管理端192.168.20.181操作===========================
打開mon節點的配置文件：
[root@c720181 ~]# vi /etc/ceph/ceph.conf
在配置文件中添加如下內容：
[mon]
mon allow pool delete = true
重啟ceph-mon服務：
[root@c720181 ~]# systemctl restart ceph-mon.target
執行刪除pool命令：
[root@c720181 ~]# ceph osd pool delete rbd rbd –yes-i-really-really-mean-it
pool ‘ecpool’ removed
[ceph-admin@c720181 my-cluster]$ vim delete_pool.sh
#!/bin/bash
PG_NUM=30
PGP_NUM=30
SIZE=3
for i in `cat ./pool`
do
ceph osd pool delete $i $i --yes-i-really-really-mean-it
[ceph-admin@c720181 my-cluster]$ ./delete_pool.sh
pool '.rgw' removed
pool '.rgw.root' removed
pool '.rgw.control' removed
pool '.rgw.gc' removed
pool '.rgw.buckets' does not exist
pool '.rgw.buckets.index' does not exist
pool '.rgw.buckets.extra' does not exist
pool '.log' does not exist
pool '.intent-log' does not exist
pool '.usage' does not exist
pool '.users' does not exist
pool '.users.email' does not exist
pool '.users.swift' does not exist
pool '.users.uid' does not exist
================================================
[ceph-admin@c720181 my-cluster]$ ceph osd pool create rbd 20
pool 'rbd' created
[ceph-admin@c720181 my-cluster]$ vim create_pool.sh
#!/bin/bash
PG_NUM=8
PGP_NUM=8
SIZE=3
for i in `cat ./pool`
do
ceph osd pool create $i $PG_NUM
ceph osd pool set $i size $SIZE
done
for i in `cat ./pool`
do
ceph osd pool set $i pgp_num $PGP_NUM
done
[ceph-admin@c720181 my-cluster]$ ./create_pool.sh
pool '.rgw' created
set pool 20 size to 3
pool '.rgw.root' created
set pool 21 size to 3
pool '.rgw.control' created
set pool 22 size to 3
pool '.rgw.gc' created
set pool 23 size to 3
pool '.rgw.buckets' created
set pool 24 size to 3
pool '.rgw.buckets.index' created
set pool 25 size to 3
pool '.rgw.buckets.extra' created
set pool 26 size to 3
pool '.log' created
set pool 27 size to 3
pool '.intent-log' created
set pool 28 size to 3
pool '.usage' created
set pool 29 size to 3
pool '.users' created
set pool 30 size to 3
pool '.users.email' created
set pool 31 size to 3
pool '.users.swift' created
set pool 32 size to 3
pool '.users.uid' created
set pool 33 size to 3
set pool 20 pgp_num to 8
set pool 21 pgp_num to 8
set pool 22 pgp_num to 8
set pool 23 pgp_num to 8
set pool 24 pgp_num to 8
set pool 25 pgp_num to 8
set pool 26 pgp_num to 8
set pool 27 pgp_num to 8
set pool 28 pgp_num to 8
set pool 29 pgp_num to 8
set pool 30 pgp_num to 8
set pool 31 pgp_num to 8
set pool 32 pgp_num to 8
set pool 33 pgp_num to 8
（5）測試是否能夠訪問ceph集群
[ceph-admin@c720181 my-cluster]$ ceph -s -k /var/lib/ceph/radosgw/ceph-rgw.c720181/keying --name client.rgw.c720181
2019-08-18 21:19:22.908292 7f3bd2d43700 -1 auth: unable to find a keyring on /var/lib/ceph/radosgw/ceph-rgw.c720181/keying: (13) Permission denied
2019-08-18 21:19:22.908414 7f3bd2d43700 0 librados: client.rgw.c720181 initialization error (13) Permission denied
[errno 13] error connecting to the cluster
提示沒有權限，可以通過下面的方式先拷貝到當前目錄，然后再測試訪問：
[ceph-admin@c720181 my-cluster]$ sudo ls -l /var/lib/ceph/
total 0
drwxr-x--- 2 ceph ceph 6 Apr 11 21:15 bootstrap-mds
drwxr-x--- 2 ceph ceph 26 Aug 18 16:00 bootstrap-mgr
drwxr-x--- 2 ceph ceph 26 Aug 18 15:56 bootstrap-osd
drwxr-x--- 2 ceph ceph 6 Apr 11 21:15 bootstrap-rbd
drwxr-x--- 2 ceph ceph 26 Aug 18 19:54 bootstrap-rgw
drwxr-x--- 2 ceph ceph 6 Apr 11 21:15 mds
drwxr-x--- 3 ceph ceph 26 Aug 18 16:00 mgr
drwxr-x--- 3 ceph ceph 26 Aug 18 15:39 mon
drwxr-x--- 3 ceph ceph 20 Aug 18 15:56 osd
drwxr-xr-x 3 root root 30 Aug 18 19:54 radosgw
drwxr-x--- 2 ceph ceph 6 Aug 18 15:39 tmp
[ceph-admin@c720181 my-cluster]$ sudo cp /var/lib/ceph/radosgw/ceph-rgw.c720181/keyring ./
[ceph-admin@c720181 my-cluster]$ ceph -s -k keyring --name client.rgw.c720181
cluster:
id: a4088ae8-c818-40d6-ab40-8f40c5bedeee
health: HEALTH_OK
services:
mon: 3 daemons, quorum c720181,c720182,c720183
mgr: c720181(active), standbys: c720183, c720182
osd: 3 osds: 3 up, 3 in
rgw: 3 daemons active
data:
pools: 18 pools, 156 pgs
objects: 183 objects, 0B
usage: 3.06GiB used, 56.9GiB / 60.0GiB avail
pgs: 156 active+clean
io:
client: 70.1KiB/s rd, 0B/s wr, 70op/s rd, 46op/s wr
=====================================================================================
使用S3 API訪問Ceph對象存儲
===================在管理端192.168.20.181操作===========================
（1）創建radosgw用戶
[ceph-admin@c720181 my-cluster]$ radosgw-admin user create --uid=radosgw --display-name="Radosgw"
注意：請把access_key secret_key保存下來，如果沒記住可使用命令查看：
radosgw-admin user info --uid=radosgw --display-name="Radosgw"
{
"user_id": "radosgw",
"display_name": "Radosgw",
"email": "",
"suspended": 0,
"max_buckets": 1000,
"auid": 0,
"subusers": [],
"keys": [
{
"user": "radosgw",
"access_key": "XBKOS3SHEB0KZ86U8QN6",
"secret_key": "kK9Rq7o1Wm8oBe02I35uqt3yIB0YOQOb4xHCVBnV"
}
],
"swift_keys": [],
"caps": [],
"op_mask": "read, write, delete",
"default_placement": "",
"placement_tags": [],
"bucket_quota": {
"enabled": false,
"check_on_raw": false,
"max_size": -1,
"max_size_kb": 0,
"max_objects": -1
},
"user_quota": {
"enabled": false,
"check_on_raw": false,
"max_size": -1,
"max_size_kb": 0,
"max_objects": -1
},
"temp_url_keys": [],
"type": "rgw"
}
===================在客戶端192.168.20.184操作===========================
（2）安裝s3cmd客戶端
yum install -y s3cmd
安裝完成之后，將會自動在家目錄下生成配置文件.s3cfg
[root@c720184 ~]# s3cmd --configure
Enter new values or accept defaults in brackets with Enter.
Refer to user manual for detailed description of all options.
Access key and Secret key are your identifiers for Amazon S3. Leave them empty for using the env variables.
Access Key: XBKOS3SHEB0KZ86U8QN6
Secret Key: kK9Rq7o1Wm8oBe02I35uqt3yIB0YOQOb4xHCVBnV
Default Region [US]: ZH
Use "s3.amazonaws.com" for S3 Endpoint and not modify it to the target Amazon S3.
S3 Endpoint [s3.amazonaws.com]:
Use "%(bucket)s.s3.amazonaws.com" to the target Amazon S3. "%(bucket)s" and "%(location)s" vars can be used
if the target S3 system supports dns based buckets.
DNS-style bucket+hostname:port template for accessing a bucket [%(bucket)s.s3.amazonaws.com]:
Encryption password is used to protect your files from reading
by unauthorized persons while in transfer to S3
Encryption password:
Path to GPG program [/usr/bin/gpg]:
When using secure HTTPS protocol all communication with Amazon S3
servers is protected from 3rd party eavesdropping. This method is
slower than plain HTTP, and can only be proxied with Python 2.7 or newer
Use HTTPS protocol [Yes]: no
On some networks all internet access must go through a HTTP proxy.
Try setting it here if you can't connect to S3 directly
HTTP Proxy server name:
New settings:
Access Key: XBKOS3SHEB0KZ86U8QN6
Secret Key: kK9Rq7o1Wm8oBe02I35uqt3yIB0YOQOb4xHCVBnV
Default Region: ZH
S3 Endpoint: s3.amazonaws.com
DNS-style bucket+hostname:port template for accessing a bucket: %(bucket)s.s3.amazonaws.com
Encryption password:
Path to GPG program: /usr/bin/gpg
Use HTTPS protocol: False
HTTP Proxy server name:
HTTP Proxy server port: 0
Test access with supplied credentials? [Y/n] n
Save settings? [y/N] y
Configuration saved to '/root/.s3cfg'
（3）編輯.s3cfg文件，修改host_base和host_bucket
vim .s3cfg
修改如下兩行:
……
host_base = s3.amazonaws.com 改成host_base = c720183.kevin.cn:7480 #必須要有域名
host_bucket = %(bucket)s.s3.amazonaws.com 改成host_bucket = %(bucket). c720183.kevin.cn:7480 #必須要有域名
……
vim /etc/hosts #添加如下行：
172.22.69.65 c720183.kevin.cn
（4）創建桶並放入文件
[root@c720184 ~]# s3cmd mb s3://first-bucket #創建一個桶
ERROR: [Errno -2] Name or service not known
ERROR: Connection Error: Error resolving a server hostname.
Please check the servers address specified in 'host_base', 'host_bucket', 'cloudfront_host', 'website_endpoint'
解決辦法：
vim.s3cfg
bucket_location = ZH 把 ZH改成US
[root@client ~]# s3cmd mb s3://first-bucket
ERROR: S3 error: 400 (InvalidLocationConstraint): The specified location-constraint is not valid
解決辦法：
vim.s3cfg
bucket_location = ZH 把 ZH改成US
[root@client ~]# s3cmd mb s3://first-bucket
ERROR: S3 error: 416 (InvalidRange)
解決辦法：
將其他對象存儲后台駐留程序（OSD）添加到群集或將“mon_max_pg_per_osd”的默認值增加到300以上。
修改配置文件（管理節點操作）：
[ceph-admin@c720181 my-cluster]$ vim ceph.conf
[mon]
mon allow pool delete = true
mon_max_pg_per_osd = 300
[ceph-admin@c720181 my-cluster]$ ceph-deploy --overwrite-conf config push c720181 c720182 c720183
重啟mon服務（集群mon節點都要操作）：
systemctl restart ceph-mon.target
[root@c720184 ~]# s3cmd mb s3://first-bucket #創建一個桶
Bucket 's3://first-bucket/' created
[root@c720184 ~]# s3cmd put /etc/hosts s3://first-bucket #把文件放入桶中
[root@c720184 ~]# s3cmd ls s3://first-bucket #查看桶里面的文件
正常情況應該是下面的截圖：
's3: //first-bucket/hosts ' 158 of 158 100% in 3s 52.55 B/S done [root@c720189 s3cmd Is s3://first-bucket 2018-11-26 14:29 [root@c720189 158 53 : / / first -bucket/hosts " v:shapes="圖片_x0020_19">
===============================================================================================
使用Swift API訪問對象存儲
（1）創建swift api自用戶
radosgw-admin subuser create --uid=radosgw --subuser=radosgw:swift --access=full --display-name="radosgw-sub" #display-name 可選 --access=full 必選，否則報錯
刪除用戶：
radosgw-admin subuser rm --uid=radosgw --subuser=radosgw:swift
[ceph-admin@c720181 my-cluster]$ radosgw-admin subuser create --uid=radosgw --subuser=radosgw:swift --access=full --display-name="radosgw-sub"
注意：請把secret_key保存下來，如果沒記住可使用命令查看：
{
"user_id": "radosgw",
"display_name": "Radosgw",
"email": "",
"suspended": 0,
"max_buckets": 1000,
"auid": 0,
"subusers": [
{
"id": "radosgw:swift",
"permissions": "<none>"
}
],
"keys": [
{
"user": "radosgw",
"access_key": "XBKOS3SHEB0KZ86U8QN6",
"secret_key": "kK9Rq7o1Wm8oBe02I35uqt3yIB0YOQOb4xHCVBnV"
}
],
"swift_keys": [
{
"user": "radosgw:swift",
"secret_key": "HHTS1whkPd46EM7CLOCRD7Mm3ATHN7nwBpqg8Iys"
}
],
"caps": [],
"op_mask": "read, write, delete",
"default_placement": "",
"placement_tags": [],
"bucket_quota": {
"enabled": false,
"check_on_raw": false,
"max_size": -1,
"max_size_kb": 0,
"max_objects": -1
},
"user_quota": {
"enabled": false,
"check_on_raw": false,
"max_size": -1,
"max_size_kb": 0,
"max_objects": -1
},
"temp_url_keys": [],
"type": "rgw"
}
（2）安裝swift客戶端
[root@c720184 ~]# yum install -y python-pip
[root@c720184 ~]# pip install --upgrade python-swiftclient
查看
[root@c720184 ~]# swift -A http://c720181.kevin.cn:7480/auth/1.0/ -U radosgw:swift -K HHTS1whkPd46EM7CLOCRD7Mm3ATHN7nwBpqg8Iys list #查看桶
Auth GET failed: http://c720181.kevin.cn:7480/auth/1.0/ 403 Forbidden [first 60 chars of response] {"Code":"AccessDenied","RequestId":"tx000000000000000000002-
Failed Transaction ID: tx000000000000000000002-005d595d78-1080-default
報錯原因：
a.對象存儲需要用到域名，所以在配置集群主機名之前需要配置好主機名和域名。
b.沒有觸發該用戶寫入數據庫，可以多試試幾遍，或者再添加一個其他用戶試試。
解決辦法：
vim /etc/hosts #添加如下行
172.22.69.63 c720181.kevin.cn
[root@c720184 ~]# swift -A http://c720181.kevin.cn:7480/auth/1.0/ -U radosgw:swift -K HHTS1whkPd46EM7CLOCRD7Mm3ATHN7nwBpqg8Iys post second-bucket #創建一個桶
[root@c720184 ~]# swift -A http://c720181,kevin.cn:7480/auth/1.0/ -U radosgw:swift -K HHTS1whkPd46EM7CLOCRD7Mm3ATHN7nwBpqg8Iys list #查看桶
正常情況的話如下圖所示：
