问题导读： 1.Ceph是什么？ 2.1.CEPH的底层是什么？ 3.CEPH存储集群中包含哪些层次结构？ 4.本文是如何建立Nova，Glance与Cinder 基于Ceph的统一存储方案的？ mds 192.168.122.149 装一个mds 一个mon， 一个osd
osd 192.168.122.169 装一个mon， 一个osd
mon 192.168.122.41 装 一个mon， 一个osd
client 192.168.122.104 上安装openstack all-in-one，管理节点 复制代码 三台机器组成ceph存储集群，hostname分别为mds，osd，mon，下文将都是使用这些短的hostname代表节点，其中在这三台机器上都部署monitor和对象存储osd，在mds上部署metadata服务器mds，另外一台机器作为openstack all-in-one环境节点 hostname:client
采用ceph-deploy来部署安装ceph， 这个类似与我们部署openstack用的chef。非常方便。
第一步: 在管理节点上修改/etc/hosts，ceph-deploy 后面的节点参数必须使用hostname，为了能够解析hostname，需要配置/etc/hosts，为下面粘贴部分的后四行。
[root@client ceph ]# cat /etc/hosts
127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
::1 localhost localhost.localdomain localhost6 localhost6.localdomain6
192.168.122.149 mds
192.168.122.169 osd
192.168.122.41 mon
192.168.122.104 client 复制代码 [root@client install]# ssh-keygen
[root@client install]# ssh-copy-id mds
[root@client install]# ssh-copy-id ods
[root@client install]# ssh-copy-id mon 复制代码 [root@client~]# cat /etc/yum.repos.d/ceph.repo
[Ceph]
name=Cephpackages for $basearch
gpgkey=https://ceph.com/git/?p=ceph.git;a=blob_plain;f=keys/release.asc
enabled=1
baseurl=http://ceph.com/rpm-firefly/rhel6/x86_64
priority=1
gpgcheck=1
type=rpm-md
[ceph-source]
name=Cephsource packages
gpgkey=https://ceph.com/git/?p=ceph.git;a=blob_plain;f=keys/release.asc
enabled=1
baseurl=http://ceph.com/rpm-firefly/rhel6/SRPMS
priority=1
gpgcheck=1
type=rpm-md
[Ceph-noarch]
name=Cephnoarch packages
gpgkey=https://ceph.com/git/?p=ceph.git;a=blob_plain;f=keys/release.asc
enabled=1
baseurl=http://ceph.com/rpm-firefly/rhel6/noarch
priority=1
gpgcheck=1
type=rpm-md 复制代码 [root@client~]# yum -y install ceph-deploy 复制代码 [root@client ~]# mkdir ceph
[root@client ~]# cd ceph 复制代码 [root@client ceph]# ceph-deploy new mds mon osd # 必须使用hostname 复制代码 [root@client ceph]# ceph-deploy install mds mon osd 复制代码 [root@client ceph]# ceph-deploy mon create mds mon osd 复制代码 不关报的错是：[ceph_deploy.gatherkeys][WARNIN]Unable to find /var/lib/ceph/bootstrap-mds/ceph.keyring
[root@client ceph]# ceph-deploy gatherkeys mds #用其中一个节点即可
[root@client ceph]# ls
ceph.bootstrap-mds.keyring ceph.bootstrap-osd.keyring ceph.client.admin.keyring ceph.conf ceph.log ceph.mon.keyring
建立osd，默认是基于xfs文件系统，并激活。
复制代码 [root@client ceph]# ceph-deploy osd prepare mds:/opt/ceph mds:/opt/cephmon:/opt/ceph
[root@client ceph]# ceph-deploy osd activate mds:/opt/ceph mds:/opt/cephmon:/opt/ceph 复制代码 [root@client ceph]# ceph-deploy mds create mds 复制代码 * 如果到此位止，我们仅想把这个文件系统mount到client端，我们需要安装ceph-fuse
[root@client ceph]# yum -y install ceph-fuse
[root@client ceph]#
[root@client ceph]# ceph-fuse -m 192.168.122.169:6789 /mnt/ceph
ceph-fuse[24569]:starting ceph client
ceph-fuse[24569]:starting fuse
[root@client ceph]# df
Filesystem 1K-blocks Used Available Use%Mounted on
/dev/mapper/vg_client-lv_root 18069936 2791420 14360604 17% /
tmpfs 812188 4 812184 1%/dev/shm
/dev/vda1 495844 34541 435703 8% /boot
/etc/swift/data/drives/images/swift.img 1038336 32976 1005360 4% /etc/swift/data/drives/sdb1
ceph-fuse 54304768 25591808 28712960 48%/mnt/ceph #这一行 复制代码 [root@client ceph]# yum install ceph
[root@client ceph]# rados mkpool volumes
[root@client ceph]# rados mkpool images
[root@client ceph]# ceph osd pool set volumes size 3
[root@client ceph]# ceph osd pool set images size 3
[root@client ceph]# ceph osd lspools
0data,1 metadata,2 rbd,4 volumes,5 images, 复制代码 [root@client ceph]# ceph auth get-or-create client.volumes mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=volumes,allow rx pool=images' -o /etc/ceph/client.volumes.keyring
[root@client ceph]# ceph auth get-or-create client.images mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=images' -o /etc/ceph/client.images.keyring 复制代码 [root@clientceph]# cat /etc/ceph/ceph.conf
[global]
auth_service_required= cephx
filestore_xattr_use_omap= true
auth_client_required= cephx
auth_cluster_required= cephx
mon_host= 192.168.122.149,192.168.122.169,192.168.122.41
mon_initial_members= mds, osd, mon
fsid= 3493ee7b-ce67-47ce-9ce1-f5d6b219a709
[client.volumes] #此行开始是加的
keyring= /etc/ceph/client.volumes.keyring
[client.images]
keyring= /etc/ceph/client.images.keyring
复制代码 Requestreturned failure status.
500Internal Server Error
GL-F9EE247Failed to upload image 1c3d2523-9752-4644-89c2-b066428144fd
(HTTP500)
本文中安装的qemu，libvirt均为新版本，版本过低可能存在rbd的支持问题。编译安装的方法为qemu安装libvirt安装
【问题: qemu版本问题，必须能支持rbd格式的，因为libvirt就是通过qemu相关命令与ceph存储交互，可通过"qemu-img–help”察看。
qemu-img version 0.12.1,
Supported formats: raw cow qcow vdi vmdk cloop dmg bochs vpc vvfat qcow2 qedvhdx parallels nbd blkdebug host_cdrom host_floppy host_device filegluster gluster gluster gluster
可以看到0.12.1不支持rbd，要装0.15以上的】
为了集成nova， 先做如下给libvirt创建密钥的操作，这个密钥在qemu执行创建image命令时会使用到，应在nova-compute服务运行的节点上执行如下操作。本文使用all-in-one，所以直接在client节点执行
[root@client ~]#ceph auth get-key client.volumes | ssh client tee client.volumes.key #ssh后紧跟的是本机的hostname
[root@client ~]# cat > secret.xml << EOF
<secret ephemeral = 'no' private = 'no'>
<usage type = 'ceph'>
<name>client.volumes secret</name>
</usage>
</secret>
EOF
[root@client ~]# sudo virsh secret-define --file secret.xml
Secret ce31d8b1-62b5-1561-a489-be305336520a created
[root@client ~]# sudo virsh secret-set-value --secret ce31d8b1-62b5-1561-a489-be305336520a --base64 $(cat client.volumes.key) &&rm client.volumes.key secret.xml
Secret value set
rm：是否删除普通文件 "client.volumes.key"？y
rm：是否删除普通文件 "secret.xml"？y 复制代码 /etc/glance/glance-api.conf
default_store = rbd
show_image_direct_url = True
bd_store_user = images
rbd_store_pool = images 复制代码 volume_driver=cinder.volume.drivers.rbd.RBDDriver
rbd_pool=volumes
rbd_user=volumes
rbd_secret_uuid=ce31d8b1-62b5-1561-a489-be305336520a 复制代码 images_type=rbd
images_rbd_pool=volumes
rbd_user=volumes
rbd_secret_uuid=ce31d8b1-62b5-1561-a489-be305336520a
复制代码 [root@clientceph]# service openstack-glance-api restart
Stoppingopenstack-glance-api: [ OK ]
Startingopenstack-glance-api: [ OK ]
[root@clientceph]# glance image-create --disk-format qcow2 --is-public True --container-format bare --file cirros-0.3.1-x86_64-disk.img --name cirros
+------------------+--------------------------------------+
|Property | Value |
+------------------+--------------------------------------+
|checksum | d972013792949d0d3ba628fbe8685bce |
|container_format | bare |
|created_at | 2014-06-24T08:49:43 |
|deleted | False |
|deleted_at | None |
|disk_format | qcow2 |
|id | 77b79879-addb-4a22-b750-7f0ef51ec154 |
|is_public | True |
|min_disk | 0 |
|min_ram | 0 |
|name | cirros |
|owner | f17fbd28fa184a39830f14a2e01a3b70 |
|protected | False |
|size | 13147648 |
|status | active |
|updated_at | 2014-06-24T08:50:01 |
|virtual_size | None |
+------------------+--------------------------------------+
[root@clientceph]# glance index
ID Name DiskFormat Container Format Size
------------------------------------------------------------------ ---------------------------------------- --------------
77b79879-addb-4a22-b750-7f0ef51ec154cirros qcow2 bare 13147648
[root@client ceph]# service openstack-cinder-volume restart
Stopping openstack-cinder-volume: [ OK ]
Starting openstack-cinder-volume: [ OK ]
[root@client ceph]# cinder create –display-name test-ceph 1
[root@client ceph]# cinder list
+--------------------------------------+-----------+--------------+------+-------------+----------+-------------+
| ID | Status | Display Name | Size| Volume Type | Bootable | Attached to |
+--------------------------------------+-----------+--------------+------+-------------+----------+-------------+
|1cc908d0-bbe9-4008-a10f-80cf1aa53afb | available | test-ceph | 1 | None | false | |
+--------------------------------------+-----------+--------------+------+-------------+----------+-------------+
[root@client ceph]# service openstack-nova-compute restart
Stopping openstack-nova-compute: [ OK ]
Starting openstack-nova-compute: [ OK ]
[root@client ceph]# nova boot --image image1 --flavor 1 xiao-new
[root@client ceph]# nova list
+--------------------------------------+-----------+--------+------------+-------------+---------------------+
| ID | Name | Status | Task State | Power State | Networks |
+--------------------------------------+-----------+--------+------------+-------------+---------------------+
| f6b04300-2d60-47d3-a65b-2d4ce32eeced | xiao-new | ACTIVE | - | Running | net_local=10.0.1.33 |
+--------------------------------------+-----------+--------+------------+-------------+---------------------+
复制代码
/2 
京ICP备2020039040号
简书网举报电话：021-34700000
提升卡
置顶卡
沉默卡
喧嚣卡
变色卡
千斤顶
显身卡
