Verifica che i nodi che apparterranno al cluster possano eseguire ssh senza password

# for i in 1 2 3; do for j in 1 2 3; do ssh c16-serv${i} "ssh c16-serv${j} hostname"; done; done
c16-serv1
c16-serv2
c16-serv3
c16-serv1
c16-serv2
c16-serv3
c16-serv1
c16-serv2
c16-serv3
#

Crea il file di input per mmcrcluster

# cat node.lis
c16-serv1:quorum-manager:
c16-serv2::
c16-serv3::
#

Crea il cluster

# mmcrcluster -N node
node node.lis
[root@c16-serv1 ~]# mmcrcluster -N node.lis -p c16-serv1 -r /usr/bin/ssh -R /usr/bin/scp -C c16 -A
Tue Nov 25 20:57:29 CET 2014: mmcrcluster: Processing node c16-serv1.openlab.infn.it
Tue Nov 25 20:57:31 CET 2014: mmcrcluster: Processing node c16-serv2.openlab.infn.it
Tue Nov 25 20:57:34 CET 2014: mmcrcluster: Processing node c16-serv3.openlab.infn.it
mmcrcluster: Command successfully completed
mmcrcluster: Warning: Not all nodes have proper GPFS license designations.
Use the mmchlicense command to designate licenses as needed.
mmcrcluster: Propagating the cluster configuration data to all
affected nodes. This is an asynchronous process.
#

Configura le licenze per i nodi del cluster

# mmchlicense server --accept -N c16-serv1,c16-serv2,c16-serv3

The following nodes will be designated as possessing GPFS server licenses:
c16-serv1.openlab.infn.it
c16-serv2.openlab.infn.it
c16-serv3.openlab.infn.it
mmchlicense: Command successfully completed
mmchlicense: Propagating the cluster configuration data to all
affected nodes. This is an asynchronous process.
#

Visualizza il cluster

# mmlscluster

GPFS cluster information
========================
GPFS cluster name: c16.openlab.infn.it
GPFS cluster id: 7610410449082048296
GPFS UID domain: c16.openlab.infn.it
Remote shell command: /usr/bin/ssh
Remote file copy command: /usr/bin/scp

GPFS cluster configuration servers:
----------------------------------- Primary server: c16-serv1.openlab.infn.it
Secondary server: (none)

Node Daemon node name IP address Admin node name Designation
------------------------------------------------------------------------------------------
1 c16-serv1.openlab.infn.it 192.168.200.246 c16-serv1.openlab.infn.it quorum-manager
2 c16-serv2.openlab.infn.it 192.168.200.247 c16-serv2.openlab.infn.it
3 c16-serv3.openlab.infn.it 192.168.200.248 c16-serv3.openlab.infn.it
#

Fai partire GPFS su tutto il cluster e verifica

# mmstartup -a
Tue Nov 25 21:02:29 CET 2014: mmstartup: Starting GPFS ...

# mmgetstate -a

Node number Node name GPFS state
------------------------------------------
1 c16-serv1 active
2 c16-serv2 active
3 c16-serv3 active
#

Modifica il tipo di nodo di c16-serv2 e c16-serv3 in quorum e manager, e verifica

# mmchnode --manager --quorum -N c16-serv2,c16-serv3
Tue Nov 25 21:04:30 CET 2014: mmchnode: Processing node c16-serv2.openlab.infn.it
Tue Nov 25 21:04:30 CET 2014: mmchnode: Processing node c16-serv3.openlab.infn.it
mmchnode: Propagating the cluster configuration data to all
affected nodes. This is an asynchronous process.

# mmlscluster

GPFS cluster information
========================
GPFS cluster name: c16.openlab.infn.it
GPFS cluster id: 7610410449082048296
GPFS UID domain: c16.openlab.infn.it
Remote shell command: /usr/bin/ssh
Remote file copy command: /usr/bin/scp

GPFS cluster configuration servers:
-----------------------------------
Primary server: c16-serv1.openlab.infn.it
Secondary server: (none)

Node Daemon node name IP address Admin node name Designation
------------------------------------------------------------------------------------------
1 c16-serv1.openlab.infn.it 192.168.200.246 c16-serv1.openlab.infn.it quorum-manager
2 c16-serv2.openlab.infn.it 192.168.200.247 c16-serv2.openlab.infn.it quorum-manager
3 c16-serv3.openlab.infn.it 192.168.200.248 c16-serv3.openlab.infn.it quorum-manager
#

Aggiungi c16-serv2 come secondary configuration server

# mmchcluster -s c16-serv2
mmchcluster: GPFS cluster configuration servers:
mmchcluster: Primary server: c16-serv1.openlab.infn.it
mmchcluster: Secondary server: c16-serv2.openlab.infn.it
mmchcluster: Propagating the new server information to the rest of the nodes.
mmchcluster: Command successfully completed

# mmlscluster

GPFS cluster information
========================
GPFS cluster name: c16.openlab.infn.it
GPFS cluster id: 7610410449082048296
GPFS UID domain: c16.openlab.infn.it
Remote shell command: /usr/bin/ssh
Remote file copy command: /usr/bin/scp

GPFS cluster configuration servers:
-----------------------------------
Primary server: c16-serv1.openlab.infn.it
Secondary server: c16-serv2.openlab.infn.it

Node Daemon node name IP address Admin node name Designation
------------------------------------------------------------------------------------------
1 c16-serv1.openlab.infn.it 192.168.200.246 c16-serv1.openlab.infn.it quorum-manager
2 c16-serv2.openlab.infn.it 192.168.200.247 c16-serv2.openlab.infn.it quorum-manager
3 c16-serv3.openlab.infn.it 192.168.200.248 c16-serv3.openlab.infn.it quorum-manager
#

Visualizza la configurazione del cluster

# mmlsconfig
Configuration data for cluster c16.openlab.infn.it:
---------------------------------------------------
myNodeConfigNumber 1
clusterName c16.openlab.infn.it
clusterId 7610410449082048296
autoload yes
dmapiFileHandleSize 32
minReleaseLevel 3.5.0.11
adminMode central

File systems in cluster c16.openlab.infn.it:
--------------------------------------------
(none)
#

Visualizza i dischi visti dal nodo

# fdisk -l | grep "^Disk /dev"
Disk /dev/vda: 21.5 GB, 21474836480 bytes
Disk /dev/vdb: 1073.7 GB, 1073741824000 bytes
Disk /dev/vdc: 1073.7 GB, 1073741824000 bytes
Disk /dev/vdd: 1073.7 GB, 1073741824000 bytes
Disk /dev/vde: 1073.7 GB, 1073741824000 bytes
Disk /dev/vdf: 42.9 GB, 42949673984 bytes
#

Crea il file di input per mmcrnsd: crea quattro nsd su vdb, vdc, vdd, vde

# cat nsd.lis
%nsd: device=/dev/vdb
nsd=nsd1
usage=dataAndMetadata
failureGroup=1
pool=system
%nsd: device=/dev/vdc
nsd=nsd2
usage=dataAndMetadata
failureGroup=2
pool=system
%nsd: device=/dev/vdd
nsd=nsd3
usage=dataAndMetadata
failureGroup=3
pool=system
%nsd: device=/dev/vde
nsd=nsd4
usage=dataAndMetadata
failureGroup=4
pool=system
#

Crea gli NSD

# mmcrnsd -F nsd.lis
mmcrnsd: Processing disk vdb
mmcrnsd: Processing disk vdc
mmcrnsd: Processing disk vdd
mmcrnsd: Processing disk vde
mmcrnsd: Propagating the cluster configuration data to all
affected nodes. This is an asynchronous process.
#

Verifica la visibilita’ degli NSD sui nodi del cluster

# mmlsnsd -m

Disk name NSD volume ID Device Node name Remarks
---------------------------------------------------------------------------------------
nsd1 C0A8C8F65474E307 /dev/vdb c16-serv1.openlab.infn.it
nsd2 C0A8C8F65474E308 /dev/vdc c16-serv1.openlab.infn.it
nsd3 C0A8C8F65474E309 /dev/vdd c16-serv1.openlab.infn.it
nsd4 C0A8C8F65474E30A /dev/vde c16-serv1.openlab.infn.it
#

Modifica gli NSD configurando NSD server per ciascuno, e verifica

# mmchnsd "nsd1:c16-serv1,c16-serv2,c16-serv3:;nsd2:c16-serv2,c16-serv3,c16-serv1:;nsd3:c16-serv3,c16-serv1,c16-serv2:;nsd4:c16-serv1,c16-serv3,c16-serv2:"
mmchnsd: Processing disk nsd1
mmchnsd: Processing disk nsd2
mmchnsd: Processing disk nsd3
mmchnsd: Processing disk nsd4
mmchnsd: Propagating the cluster configuration data to all
affected nodes. This is an asynchronous process.

# mmlsnsd

File system Disk name NSD servers
---------------------------------------------------------------------------
(free disk) nsd1 c16-serv1.openlab.infn.it,c16-serv2.openlab.infn.it,c16-serv3.openlab.infn.it
(free disk) nsd2 c16-serv2.openlab.infn.it,c16-serv3.openlab.infn.it,c16-serv1.openlab.infn.it
(free disk) nsd3 c16-serv3.openlab.infn.it,c16-serv1.openlab.infn.it,c16-serv2.openlab.infn.it
(free disk) nsd4 c16-serv1.openlab.infn.it,c16-serv3.openlab.infn.it,c16-serv2.openlab.infn.it

#

Crea il file system /dev/c16fs, mount point /gpfs/c16fs

# mmcrfs /dev/c16fs -F nsd.lis -A yes -B 1M -j scatter -M 2 -R 2 -T /gpfs/c16fs

The following disks of c16fs will be formatted on node c16-serv1:
nsd1: size 1048576000 KB
nsd2: size 1048576000 KB
nsd3: size 1048576000 KB
nsd4: size 1048576000 KB
Formatting file system ...
Disks up to size 9.1 TB can be added to storage pool system.
Creating Inode File
Creating Allocation Maps
Creating Log Files
Clearing Inode Allocation Map
Clearing Block Allocation Map
Formatting Allocation Map for storage pool system
Completed creation of file system /dev/c16fs.
mmcrfs: Propagating the cluster configuration data to all
affected nodes. This is an asynchronous process.
#

Visualizza NSD (ora appartengono ad un file system), dischi e file system

# mmlsnsd

File system Disk name NSD servers
---------------------------------------------------------------------------
c16fs nsd1 c16-serv1.openlab.infn.it,c16-serv2.openlab.infn.it,c16-serv3.openlab.infn.it
c16fs nsd2 c16-serv2.openlab.infn.it,c16-serv3.openlab.infn.it,c16-serv1.openlab.infn.it
c16fs nsd3 c16-serv3.openlab.infn.it,c16-serv1.openlab.infn.it,c16-serv2.openlab.infn.it
c16fs nsd4 c16-serv1.openlab.infn.it,c16-serv3.openlab.infn.it,c16-serv2.openlab.infn.it

# mmlsdisk /dev/c16fs
disk driver sector failure holds holds storage
name type size group metadata data status availability pool
------------ -------- ------ ----------- -------- ----- ------------- ------------ ------------
nsd1 nsd 512 1 Yes Yes ready up system
nsd2 nsd 512 2 Yes Yes ready up system
nsd3 nsd 512 3 Yes Yes ready up system
nsd4 nsd 512 4 Yes Yes ready up system

# mmlsfs /dev/c16fs
flag value description
------------------- ------------------------ -----------------------------------
-f 32768 Minimum fragment size in bytes
-i 512 Inode size in bytes
-I 32768 Indirect block size in bytes
-m 1 Default number of metadata replicas
-M 2 Maximum number of metadata replicas
-r 1 Default number of data replicas
-R 2 Maximum number of data replicas
-j scatter Block allocation type
-D nfs4 File locking semantics in effect
-k all ACL semantics in effect
-n 32 Estimated number of nodes that will mount file system
-B 1048576 Block size
-Q none Quotas enforced
none Default quotas enabled
--filesetdf No Fileset df enabled?
-V 13.23 (3.5.0.7) File system version
--create-time Tue Nov 25 21:24:23 2014 File system creation time
-u Yes Support for large LUNs?
-z No Is DMAPI enabled?
-L 4194304 Logfile size
-E Yes Exact mtime mount option
-S No Suppress atime mount option
-K whenpossible Strict replica allocation option
--fastea Yes Fast external attributes enabled?
--inode-limit 4098048 Maximum number of inodes
-P system Disk storage pools in file system
-d nsd1;nsd2;nsd3;nsd4 Disks in file system
--perfileset-quota no Per-fileset quota enforcement
-A yes Automatic mount option
-o none Additional mount options
-T /gpfs/c16fs Default mount point
--mount-priority 0 Mount priority
#

Monta il file system su tutti i nodi

# mmmount /dev/c16fs -a
Tue Nov 25 21:26:48 CET 2014: mmmount: Mounting file systems ...

# mmlsmount /dev/c16fs
File system c16fs is mounted on 3 nodes.

# mmlsmount /dev/c16fs -L

File system c16fs is mounted on 3 nodes:
192.168.200.246 c16-serv1
192.168.200.248 c16-serv3
192.168.200.247 c16-serv2
#

Crea un file

# dd if=/dev/zero of=/gpfs/c16fs/datafile bs=1M count=1K
1024+0 records in
1024+0 records out
1073741824 bytes (1.1 GB) copied, 5.75137 s, 187 MB/s
#

Verifica l’occupazione del file system

# df /gpfs/c16fs/
Filesystem 1K-blocks Used Available Use% Mounted on
/dev/c16fs 4194304000 1529856 4192774144 1% /gpfs/c16fs

# mmdf /dev/c16fs
disk disk size failure holds holds free KB free KB
name in KB group metadata data in full blocks in fragments
--------------- ------------- -------- -------- ----- -------------------- -------------------
Disks in storage pool: system (Maximum disk size allowed is 8.3 TB)
nsd1 1048576000 1 Yes Yes 1048194048 (100%) 1952 ( 0%)
nsd2 1048576000 2 Yes Yes 1048193024 (100%) 992 ( 0%)
nsd3 1048576000 3 Yes Yes 1048193024 (100%) 992 ( 0%)
nsd4 1048576000 4 Yes Yes 1048194048 (100%) 2976 ( 0%)
------------- -------------------- -------------------
(pool total) 4194304000 4192774144 (100%) 6912 ( 0%)

(total) 4194304000 4192774144 (100%) 6912 ( 0%)

Inode Information
-----------------
Number of used inodes: 4039
Number of free inodes: 503865
Number of allocated inodes: 507904
Maximum number of inodes: 4098048