MFS分布式文件系统

集群部署

Master Servers

Chunkservers

编辑Clients

Storage Classes

LABEL

mfs高可用

pacemaker高可用

编辑ISCSI

添加集群资源

主机	ip	角色
server1	192.168.81.11	Master Servers
server2	192.168.81.12	Chunkservers
server3	192.168.81.13	Chunkservers
server4	192.168.81.14	Chunkservers
server5	192.168.81.15	Clients

集群部署

Download current version of MooseFS | MooseFS Distributed File System

所有节点添加密钥和软件源库

[root@server1 ~]# curl "https://repository.moosefs.com/RPM-GPG-KEY-MooseFS" > /etc/pki/rpm-gpg/RPM-GPG-KEY-MooseFS
[root@server1 ~]# curl "http://repository.moosefs.com/MooseFS-3-el7.repo" > /etc/yum.repos.d/MooseFS.repo

Master Servers

[root@server1 ~]#yum install moosefs-master moosefs-cgi moosefs-cgiserv moosefs-cli
[root@server1 ~]#systemctl enable --now moosefs-master.service
[root@server1 ~]#systemctl enable --now moosefs-cgiserv.service

访问http://192.168.81.11:9425/

添加解析

Chunkservers

server2、server3、server4

[root@server2 ~]# yum install moosefs-chunkserver

[root@server2 ~]# cd /etc/mfs/
[root@server2 mfs]# vim mfshdd.cfg
/mnt/chunk1                //指定挂载点
[root@server2 ~]# cd /mnt
[root@server2 mnt]# mkdir chunk1

[root@server2 mnt]# chown -R mfs.mfs chunk1/
[root@server2 ~]# systemctl enable --now moosefs-chunkserver

Clients

[root@server5 ~]# yum install moosefs-client

[root@server5 ~]# vim /etc/mfs/mfsmount.cfg
/mnt/mfs
[root@server5 ~]# mkdir /mnt/mfs
[root@server5 ~]# mfsmount

[root@server5 ~]# cd /mnt/mfs/
[root@server5 mfs]# mkdir dir1
[root@server5 mfs]# mkdir dir2
[root@server5 mfs]# cd dir1/
[root@server5 dir1]# vim file1
[root@server5 dir1]# mfsfileinfo file1

[root@server5 dir1]# cd -
/mnt/mfs
[root@server5 mfs]# mfsgetgoal dir2/
dir2/: 2
[root@server5 mfs]# mfssetgoal -r 1 dir2/
dir2/:
 inodes with goal changed:                       1
 inodes with goal not changed:                   0
 inodes with permission denied:                  0
[root@server5 mfs]# mfsgetgoal -r 1 dir2/
1: realpath error on (/mnt/mfs/1): ENOENT (No such file or directory)
dir2/:
 directories with goal          1 :          1
[root@server5 mfs]# cd dir2/
[root@server5 dir2]# vim file1
[root@server5 dir2]# mfsfileinfo file1
file1:
        chunk 0: 0000000000000004_00000001 / (id:4 ver:1)
                copy 1: 192.168.81.12:9422 (status:VALID)

Storage Classes

LABEL

[root@server2 ~]# vim /etc/mfs/mfschunkserver.cfg

[root@server5 mfs]# mfsscadmin create 2S class_2s
storage class make class_2s: ok
[root@server5 mfs]# cd dir1/
[root@server5 dir1]# ls
file1
[root@server5 dir1]# mfssetsclass -r class_2s file1
file1:
 inodes with storage class changed:              1
 inodes with storage class not changed:          0
 inodes with permission denied:                  0
[root@server5 dir1]# mfsfile
mfsfileinfo    mfsfilepaths   mfsfilerepair
[root@server5 dir1]# mfsfileinfo file1
file1:
        chunk 0: 0000000000000002_00000001 / (id:2 ver:1)
                copy 1: 192.168.81.12:9422 (status:VALID)
                copy 2: 192.168.81.14:9422 (status:VALID)

[root@server5 dir1]# mfsscadmin create A,B class_ab
storage class make class_ab: ok
[root@server5 dir1]# mfssetsclass -r class_ab file1
file1:
 inodes with storage class changed:              1
 inodes with storage class not changed:          0
 inodes with permission denied:                  0
[root@server5 dir1]# mfsfileinfo file1
file1:
        chunk 0: 0000000000000002_00000001 / (id:2 ver:1)
                copy 1: 192.168.81.12:9422 (status:VALID)
                copy 2: 192.168.81.13:9422 (status:VALID)

[root@server5 mnt]# cd mfs/
[root@server5 mfs]# mkdir dir3
[root@server5 mfs]# mfsscadmin create S,H class_sh
storage class make class_sh: ok
[root@server5 mfs]# cd dir3/
[root@server5 dir3]# vim file1
[root@server5 dir3]# mfssetsclass -r class_sh file1
file1:
 inodes with storage class changed:              1
 inodes with storage class not changed:          0
 inodes with permission denied:                  0
[root@server5 dir3]# mfsfileinfo file1
file1:
        chunk 0: 0000000000000006_00000001 / (id:6 ver:1)
                copy 1: 192.168.81.12:9422 (status:VALID)
                copy 2: 192.168.81.13:9422 (status:VALID)

列出当前系统中所有的存储类 (class)

[root@server5 dir3]# mfsscadmin list

mfs高可用

新建虚机server6

[root@server6 ~]# curl "https://repository.moosefs.com/RPM-GPG-KEY-MooseFS" > /etc/pki/rpm-gpg/RPM-GPG-KEY-MooseFS
[root@server6 ~]# curl "http://repository.moosefs.com/MooseFS-3-el7.repo" > /etc/yum.repos.d/MooseFS.repo
[root@server6 ~]# yum install moosefs-master moosefs-cgi moosefs-cgiserv moosefs-cli
[root@server6 ~]# systemctl enable --now moosefs-master.service
[root@server6 ~]# systemctl enable --now moosefs-cgiserv.service

pacemaker高可用

[root@server1 ~]# vim /etc/yum.repos.d/dvd.repo
[HighAvailability]
name=rhel7.6 HighAvailability
baseurl=file:///media/addons/HighAvailability
gpgcheck=0

[root@server1 ~]# yum install -y pacemaker pcs psmisc policycoreutils-python

server6同步

[root@server1 ~]# scp /etc/yum.repos.d/dvd.repo server6:/etc/yum.repos.d/
[root@server1 ~]# ssh server6 yum install -y pacemaker pcs psmisc policycoreutils-python

节点认证

[root@server1 ~]# systemctl enable --now pcsd.service
[root@server1 ~]# ssh server6 systemctl enable --now pcsd.service

[root@server1 ~]# echo westos | passwd --stdin hacluster
[root@server1 ~]# ssh server6 'echo westos | passwd --stdin hacluster'

[root@server1 ~]# pcs cluster auth server1 server6
Username: hacluster
Password: westos

创建集群

[root@server1 ~]# pcs cluster setup --name mycluster server1 server6
[root@server1 ~]# pcs cluster start --all
[root@server1 ~]# pcs cluster enable --all

禁用stonith

[root@server1 ~]# pcs property set stonith-enabled=false

ISCSI

server4添加硬盘分区

[root@server5 ~]# yum install targetcli -y
[root@server5 ~]# systemctl enable --now target

[root@server5 ~]# targetcli
/> /backstores/block create westos:storage /dev/sdb1
/> /iscsi create iqn.2023-11.org.westos:storage
/> /iscsi/iqn.2023-11.org.westos:storage/tpg1/luns create /backstores/block/westos:storage
/> /iscsi/iqn.2023-11.org.westos:storage/tpg1/acls create iqn.2023-11.org.westos:westoskey1

客户端

[root@server1 ~]# yum install iscsi-initiator-utils.x86_64 -y
[root@server1 ~]# vim /etc/iscsi/initiatorname.iscsi
InitiatorName=iqn.2023-11.org.westos:westoskey1
[root@server1 ~]# systemctl restart iscsid

[root@server1 ~]# iscsiadm -m discovery -t st -p 192.168.81.15
[root@server1 ~]# iscsiadm -m node -T iqn.2023-11.org.westos:storage -p 192.168.81.15 -l

创建分区

[root@server1 ~]# fdisk /dev/sdb
[root@server1 ~]# mkfs.xfs /dev/sdb1

[root@server1 ~]# mount /dev/sdb1 /mnt/
[root@server1 ~]# cd /var/lib/mfs/
[root@server1 mfs]# systemctl disable --now moosefs-master.service
[root@server1 mfs]# cp * /mnt/
[root@server1 mnt]# chown -R mfs.mfs /mnt/
[root@server1 mnt]# ll -d /mnt/
[root@server1 ~]# umount /mnt/
[root@server1 ~]# ll -d /mnt/

server6类推

[root@server6 mfs]# mount /dev/sdb1 /var/lib/mfs/
[root@server6 mfs]# chown -R mfs.mfs .
[root@server6 ~]# umount /var/lib/mfs/

添加集群资源

[root@server1 ~]# pcs resource create vip ocf:heartbeat:IPaddr2 ip=192.168.81.200 op monitor interval=30s
[root@server1 ~]# pcs resource create mfs ocf:heartbeat:Filesystem device=/dev/sdb1 directory=/var/lib/mfs fstype=xfs
[root@server1 ~]# pcs resource create mfs-master systemd:moosefs-master op monitor interval=60s
[root@server1 ~]# pcs resource group add groupmfs vip mfs mfs-master