This is the last part of clustering series post regarding Red Hat HA cluster. In this post, HA NFS will be created in the cluster.
Creating a Fail-over Domain
[root@node01 ~]# ccs -h localhost --addfailoverdomain NFS
[root@node01 ~]# ccs -h localhost --addfailoverdomainnode NFS node01-hb.cls.local
[root@node01 ~]# ccs -h localhost --addfailoverdomainnode NFS node02-hb.cls.local
Creating Resources.
As you see the section nfs client we specify the target *(all). It is not secure configuration, so you should specify network addresses that you want to allow.
[root@node01 ~]# ccs -h localhost --addresource lvm name=nfs-halvm vg_name=vgcls_nfs lv_name=lvcls_nfs
[root@node01 ~]# ccs -h localhost --addresource fs name=nfs-fs mountpoint="/mnt/nfs-export" fstype="ext4" device="/dev/vgcls_nfs/lvcls_nfs" force_fsck="0" force_unmount="0"
[root@node01 ~]# ccs -h localhost --addresource nfsserver name=nfs-server
[root@node01 ~]# ccs -h localhost --addresource nfsclient name=nfs-client options="rw,no_root_squash" target="*"
[root@node01 ~]# ccs -h localhost --addresource ip address="192.168.122.20" monitor_link="1"
Creating Service Groups
[root@node01 ~]# ccs -h localhost --addservice HANFS domain="NFS" nfslock=1 exclusive=0 recovery=relocate autostart=1
[root@node01 ~]# ccs -h localhost --addsubservice HANFS ip ref=192.168.122.20
[root@node01 ~]# ccs -h localhost --addsubservice HANFS lvm ref=nfs-halvm
[root@node01 ~]# ccs -h localhost --addsubservice HANFS lvm:fs ref=nfs-fs
[root@node01 ~]# ccs -h localhost --addsubservice HANFS lvm:fs:nfsserver ref=nfs-server
[root@node01 ~]# ccs -h localhost --addsubservice HANFS lvm:fs:nfsserver:nfsclient ref=nfs-client
Sync cluster configuration to the other nodes.
[root@node01 ~]# ccs -h localhost --sync --activate
Final Cluster Configuration:
[root@node01 ~]# ccs -h localhost --getconf
<cluster config_version="127" name="ankara-cluster">
<fence_daemon/>
<clusternodes>
<clusternode name="node01-hb.cls.local" nodeid="1">
<fence>
<method name="FMET_XVM">
<device domain="node01" name="FDEV_XVM1"/>
</method>
</fence>
</clusternode>
<clusternode name="node02-hb.cls.local" nodeid="2">
<fence>
<method name="FMET_XVM">
<device domain="node02" name="FDEV_XVM2"/>
</method>
</fence>
</clusternode>
</clusternodes>
<cman expected_votes="1" two_node="1"/>
<fencedevices>
<fencedevice agent="fence_xvm" name="FDEV_XVM1"/>
<fencedevice agent="fence_xvm" name="FDEV_XVM2"/>
</fencedevices>
<rm>
<failoverdomains>
<failoverdomain name="name=httpd" nofailback="0" ordered="0" restricted="0"/>
<failoverdomain name="NFS" nofailback="0" ordered="0" restricted="0">
<failoverdomainnode name="node01-hb.cls.local"/>
<failoverdomainnode name="node02-hb.cls.local"/>
</failoverdomain>
</failoverdomains>
<resources>
<clusterfs device="UUID=996a0360-1895-2c53-b4ed-876151027b61" fstype="gfs2" mountpoint="/data/httpd" name="httpdgfs2"/>
<ip address="192.168.122.10" monitor_link="yes" sleeptime="10"/>
<lvm lv_name="lvcls_nfs" name="nfs-halvm" vg_name="vgcls_nfs"/>
<fs device="/dev/vgcls_nfs/lvcls_nfs" force_fsck="0" force_unmount="0" fstype="ext4" mountpoint="/mnt/nfs-export" name="nfs-fs"/>
<nfsserver name="nfs-server"/>
<nfsclient name="nfs-client" options="rw,no_root_squash" target="*"/>
<ip address="192.168.122.20" monitor_link="1"/>
</resources>
<service domain="httpd" name="httpd-resources" recovery="relocate">
<ip ref="192.168.122.10">
<clusterfs ref="httpdgfs2"/>
</ip>
</service>
<service autostart="1" domain="NFS" exclusive="0" name="HANFS" nfslock="1" recovery="relocate">
<ip ref="192.168.122.20"/>
<lvm ref="nfs-halvm">
<fs ref="nfs-fs">
<nfsserver ref="nfs-server">
<nfsclient ref="nfs-client"/>
</nfsserver>
</fs>
</lvm>
</service>
</rm>
<quorumd label="qdisk"/>
</cluster>
Cluster Status:
[root@node01 ~]# clustat
Cluster Status for ankara-cluster @ Sat Jan 13 19:54:54 2018
Member Status: Quorate
Member Name ID Status
------ ---- ---- ------
node01-hb.cls.local 1 Online, Local, rgmanager
node02-hb.cls.local 2 Online, rgmanager
/dev/block/8:16 0 Online, Quorum Disk
Service Name Owner (Last) State
------- ---- ----- ------ -----
service:HANFS node01-hb.cls.local started
service:httpd-resources node01-hb.cls.local started
[root@node01 ~]#
dh -hP on node1
/dev/mapper/vgcls_httpd-lv_httpd 1020M 131M 890M 13% /data/httpd
/dev/mapper/vgcls_nfs-lvcls_nfs 988M 1.4M 936M 1% /mnt/nfs-export
Cluster IP NFS HA
[root@node01 ~]# ping -c3 192.168.122.20
PING 192.168.122.20 (192.168.122.20) 56(84) bytes of data.
64 bytes from 192.168.122.20: icmp_seq=1 ttl=64 time=0.035 ms
64 bytes from 192.168.122.20: icmp_seq=2 ttl=64 time=0.023 ms
64 bytes from 192.168.122.20: icmp_seq=3 ttl=64 time=0.042 ms
Mounting HA NFS:
[root@KVMHOST ~]# mount.nfs nfscls:/mnt/nfs-export /mnt/nfscls/
[root@oc7133075241 ~]# df -hP
(...omitted)
Filesystem Size Used Avail Use% Mounted on
nfscls:/mnt/nfs-export 988M 1.3M 936M 1% /mnt/nfscls
Migrating Cluster Service to Other Node:
[root@node01 ~]# clusvcadm
usage: clusvcadm [command]
Resource Group Control Commands:
-v Display version and exit
-d <group> Disable <group>. This stops a group
until an administrator enables it again,
the cluster loses and regains quorum, or
an administrator-defined event script
explicitly enables it again.
-e <group> Enable <group>
-e <group> -F Enable <group> according to failover
domain rules (deprecated; always the
case when using central processing)
-e <group> -m <member> Enable <group> on <member>
-r <group> -m <member> Relocate <group> [to <member>]
Stops a group and starts it on another
cluster member.
-M <group> -m <member> Migrate <group> to <member>
(e.g. for live migration of VMs)
-q Quiet operation
-R <group> Restart a group in place.
-s <group> Stop <group>. This temporarily stops
a group. After the next group or
or cluster member transition, the group
will be restarted (if possible).
-Z <group> Freeze resource group. This prevents
transitions and status checks, and is
useful if an administrator needs to
administer part of a service without
stopping the whole service.
-U <group> Unfreeze (thaw) resource group. Restores
a group to normal operation.
-c <group> Convalesce (repair, fix) resource group.
Attempts to start failed, non-critical
resources within a resource group.
Resource Group Locking (for cluster Shutdown / Debugging):
-l Lock local resource group managers.
This prevents resource groups from
starting.
-S Show lock state
-u Unlock resource group managers.
This allows resource groups to start.
Migrating httpd-resources service to node02.
[root@node01 ~]# clusvcadm -r httpd-resources -m node02-hb.cls.local
Trying to relocate service:httpd-resources to node02-hb.cls.local...Success
service:httpd-resources is now running on node02-hb.cls.local
[root@node01 ~]# clustat
Cluster Status for ankara-cluster @ Sat Jan 13 20:11:10 2018
Member Status: Quorate
Member Name ID Status
------ ---- ---- ------
node01-hb.cls.local 1 Online, Local, rgmanager
node02-hb.cls.local 2 Online, rgmanager
/dev/block/8:16 0 Online, Quorum Disk
Service Name Owner (Last) State
------- ---- ----- ------ -----
service:HANFS node01-hb.cls.local started
service:httpd-resources node02-hb.cls.local
Testing
[root@oc7133075241 ~]# curl http://ankara-cluster
<h1> Hello Ankara-Cluster</h1>
<h2>Tue, 14 2017</h2>
Cluster Operations:
Checking fence agent. You can run on this command only on the master node.
[root@node01 ~]# fence_check
fence_check run at Sat Jan 13 20:05:49 CET 2018 pid: 1022
Testing node01-hb.cls.local method 1: success
Testing node02-hb.cls.local method 1: success
Finding Master Nodeid:
[root@node01 ~]# cman_tool services
fence domain
member count 2
victim count 0
victim now 0
master nodeid 1
wait state none
members 1 2
dlm lockspaces
name httpdgfs2
id 0x2b180060
flags 0x00000008 fs_reg
change member 1 joined 1 remove 0 failed 0 seq 1,1
members 1
name rgmanager
id 0x5231f3eb
flags 0x00000000
change member 2 joined 1 remove 0 failed 0 seq 2,2
members 1 2
name clvmd
id 0x4104eefa
flags 0x00000000
change member 2 joined 1 remove 0 failed 0 seq 2,2
members 1 2
gfs mountgroups
name httpdgfs2
id 0xf8e8df07
flags 0x00000048 mounted
change member 1 joined 1 remove 0 failed 0 seq 1,1
members 1
Showing Nodes:
[root@node01 ~]# cman_tool nodes
Node Sts Inc Joined Name
0 M 0 2018-01-13 19:41:11 /dev/block/8:16
1 M 728 2018-01-13 19:41:06 node01-hb.cls.local
2 M 732 2018-01-13 19:41:30 node02-hb.cls.local