Project

General

Profile

Grunt Setup

General notes

  • turn off network boot in the BIOS
  • disable firewall, iptables, ip6tables, selinux, NetManager
  • use /usr/bin/system-config-firewall-tui
  • use chkconfig
  • edit /etc/selinux/config
  • add definitions to /etc/hosts
  • configure eth1 and eth2
  • use /usr/bin/system-config-network
  • udev might reassign these, so be careful

services

chkconfig NetworkManager off
chkconfig ip6tables off
chkconfig iptables off
chkconfig rsh on
chkconfig rlogin on
chkconfig rexec on

/etc/yum.conf

add a proxy line to the bottom:
proxy=http://192.168.76.79:3128/
OFED protection:

# This is to avoid problems with the IB drivers from OFED.org.
exclude=compat-dapl,compat-dapl-devel,dapl,dapl-debuginfo,dapl-devel,dapl-devel-static,dapl-utils,ibacm,ibsim,ibsim-debuginfo,ibutils,infiniband-diags,infinipath-psm,infinipath-psm-devel,kernel-ib,kernel-ib-devel,libcxgb3,libcxgb3-debuginfo,libcxgb3-devel,libcxgb4,libcxgb4-debuginfo,libcxgb4-devel,libibcm,libibcm-debuginfo,libibcm-devel,libibmad,libibmad-debuginfo,libibmad-devel,libibmad-static,libibumad,libibumad-debuginfo,libibumad-devel,libibumad-static,libibverbs,libibverbs-debuginfo,libibverbs-devel,libibverbs-devel-static,libibverbs-utils,libipathverbs,libipathverbs-debuginfo,libipathverbs-devel,libmlx4,libmlx4-debuginfo,libmlx4-devel,libmthca,libmthca-static,libmthca-debuginfo,libmthca-devel-static,libnes,libnes-debuginfo,libnes-devel-static,librdmacm,librdmacm-debuginfo,librdmacm-devel,librdmacm-utils,libsdp,libsdp-debuginfo,libsdp-devel,mpi-selector,mpitests_mvapich2_gcc,mpitests_mvapich_gcc,mpitests_openmpi_gcc,mstflint,mvapich2_gcc,mvapich_gcc,ofed-docs,ofed-scripts,openmpi_gcc,opensm,opensm-debuginfo,opensm-devel,opensm-libs,opensm-static,perftest,qperf,qperf-debuginfo,rds-devel,rds-tools,sdpnetstat,srptools

/etc/sysconfig/network

NETWORKING=yes
HOSTNAME=gruntN

/etc/grub.conf

add console=tty0 console=ttyS2,115200N8 to the end of the kernel line

ntpd

  • edit /etc/ntp.conf
  • add two server lines to the bottom
  • server 192.168.76.79 iburst
  • server 192.168.76.26 iburst
  • if zz_ntp_configure was installed, remove the 131.225 lines
  • edit /etc/ntp/step-tickers
  • 192.168.76.26 192.168.76.79
  • edit /etc/sysconfig/ntpdate
  • set SYNC_HWCLOCK to yes
  • /etc/init.d/ntpd restart
  • verify status
  • /usr/bin/ntpstat should say "synchronised to NTP server.."
  • /usr/sbin/ntpq -p and /usr/sbin/ntpdc -p list the ntp servers

restore users, etc.

  • edit /etc/idmapd.conf
  • add Domain = private.net near the top
  • restart idmapd or reboot
  • if you don't do this, all uids and gids will be squashed to nobody
  • mount cluckp:/mnt/disk1/grunt/scratch /home-cluck
  • backup files are in /scratch-shared/backup/gruntN/
  • make sure /home, /usr/local, and /opt are empty
  • mkdir /products /home-cluck /scratch-shared
  • recover appropriate lines from /scratch-shared/backup/gruntN/etc/passwd and /scratch-shared/backup/gruntN/etc/group
  • pwconv
  • recover /root/.rhosts

/etc/fstab

  • recover nfs mount lines from /scratch-shared/backup/gruntN/etc/fstab
  • mount -a
cluckp:/mnt/disk1/grunt/home    /home-grunt     nfs4    defaults,hard,intr,timeo=600 0 0
cluckp:/mnt/disk1/grunt/local   /usr/local      nfs4    defaults,hard,intr,timeo=600 0 0
cluckp:/mnt/disk1/grunt/opt     /opt            nfs4    defaults,hard,intr,timeo=600 0 0
cluckp:/mnt/disk1/products      /products       nfs4    defaults,hard,intr,timeo=600 0 0
cluckp:/home                    /home           nfs4    defaults,hard,intr,timeo=600 0 0
cluckp:/mnt/disk1/grunt/scratch /scratch-shared nfs4    defaults,hard,intr,timeo=600 0 0

ssh access

  • yum install openssh-server
  • edit /etc/ssh/sshd_config
  • RSAAuthentication yes
  • PubkeyAuthentication yes
  • UsePAM no
  • KerberosAuthentication no
  • GSSAPIAuthentication no
  • /etc/init.d/sshd start
  • recover /etc/ssh/ssh_config from backup
  • make sure /root/.ssh is recovered from backup

make totalview work

  • cd /etc/init.d; ln -s /usr/local/cet-chg/unix-admin/tv-ssh-tunnel
  • chkconfig --add tv-ssh-tunnel
  • edit /etc/hosts
  • add flmvm1.fnal.gov flmvm1 to the end of the 127.0.0.1 line
  • /etc/init.d/tv-ssh-tunnel start

/etc/hosts

192.168.76.79   cluckp.private.net cluck.fnal.gov cluckp cluck

# Cluster head node
192.168.76.26   tev.fnal.gov tev
192.168.77.26   ipmitev.private.net ipmitev
192.168.176.26  ibtev.private.net ibtev

# NFS File server
192.168.76.27   tevnfs.private.net tevnfs
192.168.176.27  ibtevnfs.private.net ibtevnfs

# FY10 worker nodes
192.168.76.1  tev0101.private.net tev0101
192.168.76.2  tev0102.private.net tev0102
192.168.76.3  tev0103.private.net tev0103
192.168.76.4  tev0104.private.net tev0104
192.168.76.5  tev0105.private.net tev0105
192.168.76.6  tev0106.private.net tev0106
192.168.76.7  tev0107.private.net tev0107
192.168.76.8  tev0108.private.net tev0108
192.168.76.9  tev0109.private.net tev0109
192.168.76.10  tev0110.private.net tev0110
192.168.76.11  tev0111.private.net tev0111
192.168.76.12  tev0112.private.net tev0112
192.168.76.13  tev0113.private.net tev0113
192.168.76.14  tev0201.private.net tev0201
192.168.76.15  tev0202.private.net tev0202
192.168.76.16  tev0203.private.net tev0203
192.168.76.17  tev0204.private.net tev0204
192.168.76.18  tev0205.private.net tev0205
192.168.76.19  tev0206.private.net tev0206
192.168.76.20  tev0207.private.net tev0207
192.168.76.21  tev0208.private.net tev0208
192.168.76.22  tev0209.private.net tev0209
192.168.76.23  tev0210.private.net tev0210
192.168.76.24  tev0211.private.net tev0211
192.168.76.25  tev0212.private.net tev0212

# FY11 cluster nodes
192.168.76.40  tev0301.private.net tev0301
192.168.76.41  tev0302.private.net tev0302
192.168.76.42  tev0303.private.net tev0303
192.168.76.43  tev0304.private.net tev0304
192.168.76.44  tev0305.private.net tev0305
192.168.76.45  tev0306.private.net tev0306
192.168.76.46  tev0307.private.net tev0307
192.168.76.47  tev0308.private.net tev0308
192.168.76.48  tev0309.private.net tev0309
192.168.76.49  tev0310.private.net tev0310
192.168.76.50  tev0311.private.net tev0311
192.168.76.51  tev0312.private.net tev0312
192.168.76.52  tev0401.private.net tev0401
192.168.76.53  tev0402.private.net tev0402
192.168.76.54  tev0403.private.net tev0403
192.168.76.55  tev0404.private.net tev0404
192.168.76.56  tev0405.private.net tev0405
192.168.76.57  tev0406.private.net tev0406
192.168.76.58  tev0407.private.net tev0407
192.168.76.59  tev0408.private.net tev0408
192.168.76.60  tev0409.private.net tev0409
192.168.76.61  tev0410.private.net tev0410
192.168.76.62  tev0411.private.net tev0411
192.168.76.63  tev0412.private.net tev0412
192.168.76.64  tev0501.private.net tev0501
192.168.76.65  tev0502.private.net tev0502
192.168.76.66  tev0503.private.net tev0503
192.168.76.67  tev0504.private.net tev0504
192.168.76.68  tev0505.private.net tev0505
192.168.76.69  tev0506.private.net tev0506
192.168.76.70  tev0507.private.net tev0507
192.168.76.71  tev0508.private.net tev0508
192.168.76.72  tev0509.private.net tev0509
192.168.76.73  tev0510.private.net tev0510

# FY10 cluster nodes ipmi network
192.168.77.1 ipmitev0101.private.net ipmitev0101
192.168.77.2 ipmitev0102.private.net ipmitev0102
192.168.77.3 ipmitev0103.private.net ipmitev0103
192.168.77.4 ipmitev0104.private.net ipmitev0104
192.168.77.5 ipmitev0105.private.net ipmitev0105
192.168.77.6 ipmitev0106.private.net ipmitev0106
192.168.77.7 ipmitev0107.private.net ipmitev0107
192.168.77.8 ipmitev0108.private.net ipmitev0108
192.168.77.9 ipmitev0109.private.net ipmitev0109
192.168.77.10 ipmitev0110.private.net ipmitev0110
192.168.77.11 ipmitev0111.private.net ipmitev0111
192.168.77.12 ipmitev0112.private.net ipmitev0112
192.168.77.13 ipmitev0113.private.net ipmitev0113
192.168.77.14 ipmitev0201.private.net ipmitev0201
192.168.77.15 ipmitev0202.private.net ipmitev0202
192.168.77.16 ipmitev0203.private.net ipmitev0203
192.168.77.17 ipmitev0204.private.net ipmitev0204
192.168.77.18 ipmitev0205.private.net ipmitev0205
192.168.77.19 ipmitev0206.private.net ipmitev0206
192.168.77.20 ipmitev0207.private.net ipmitev0207
192.168.77.21 ipmitev0208.private.net ipmitev0208
192.168.77.22 ipmitev0209.private.net ipmitev0209
192.168.77.23 ipmitev0210.private.net ipmitev0210
192.168.77.24 ipmitev0211.private.net ipmitev0211
192.168.77.25 ipmitev0212.private.net ipmitev0212

# FY11 cluster nodes ipmi network
192.168.77.40 ipmitev0301.private.net ipmitev0301
192.168.77.41 ipmitev0302.private.net ipmitev0302
192.168.77.42 ipmitev0303.private.net ipmitev0303
192.168.77.43 ipmitev0304.private.net ipmitev0304
192.168.77.44 ipmitev0305.private.net ipmitev0305
192.168.77.45 ipmitev0306.private.net ipmitev0306
192.168.77.46 ipmitev0307.private.net ipmitev0307
192.168.77.47 ipmitev0308.private.net ipmitev0308
192.168.77.48 ipmitev0309.private.net ipmitev0309
192.168.77.49 ipmitev0310.private.net ipmitev0310
192.168.77.50 ipmitev0311.private.net ipmitev0311
192.168.77.51 ipmitev0312.private.net ipmitev0312
192.168.77.52 ipmitev0401.private.net ipmitev0401
192.168.77.53 ipmitev0402.private.net ipmitev0402
192.168.77.54 ipmitev0403.private.net ipmitev0403
192.168.77.55 ipmitev0404.private.net ipmitev0404
192.168.77.56 ipmitev0405.private.net ipmitev0405
192.168.77.57 ipmitev0406.private.net ipmitev0406
192.168.77.58 ipmitev0407.private.net ipmitev0407
192.168.77.59 ipmitev0408.private.net ipmitev0408
192.168.77.60 ipmitev0409.private.net ipmitev0409
192.168.77.61 ipmitev0410.private.net ipmitev0410
192.168.77.62 ipmitev0411.private.net ipmitev0411
192.168.77.63 ipmitev0412.private.net ipmitev0412
192.168.77.64 ipmitev0501.private.net ipmitev0501
192.168.77.65 ipmitev0502.private.net ipmitev0502
192.168.77.66 ipmitev0503.private.net ipmitev0503
192.168.77.67 ipmitev0504.private.net ipmitev0504
192.168.77.68 ipmitev0505.private.net ipmitev0505
192.168.77.69 ipmitev0506.private.net ipmitev0506
192.168.77.70 ipmitev0507.private.net ipmitev0507
192.168.77.71 ipmitev0508.private.net ipmitev0508
192.168.77.72 ipmitev0509.private.net ipmitev0509
192.168.77.73 ipmitev0510.private.net ipmitev0510

# FY10 cluster nodes Infiniband network
192.168.176.1  ibtev0101.private.net ibtev0101
192.168.176.2  ibtev0102.private.net ibtev0102
192.168.176.3  ibtev0103.private.net ibtev0103
192.168.176.4  ibtev0104.private.net ibtev0104
192.168.176.5  ibtev0105.private.net ibtev0105
192.168.176.6  ibtev0106.private.net ibtev0106
192.168.176.7  ibtev0107.private.net ibtev0107
192.168.176.8  ibtev0108.private.net ibtev0108
192.168.176.9  ibtev0109.private.net ibtev0109
192.168.176.10  ibtev0110.private.net ibtev0110
192.168.176.11  ibtev0111.private.net ibtev0111
192.168.176.12  ibtev0112.private.net ibtev0112
192.168.176.13  ibtev0113.private.net ibtev0113
192.168.176.14  ibtev0201.private.net ibtev0201
192.168.176.15  ibtev0202.private.net ibtev0202
192.168.176.16  ibtev0203.private.net ibtev0203
192.168.176.17  ibtev0204.private.net ibtev0204
192.168.176.18  ibtev0205.private.net ibtev0205
192.168.176.19  ibtev0206.private.net ibtev0206
192.168.176.20  ibtev0207.private.net ibtev0207
192.168.176.21  ibtev0208.private.net ibtev0208
192.168.176.22  ibtev0209.private.net ibtev0209
192.168.176.23  ibtev0210.private.net ibtev0210
192.168.176.24  ibtev0211.private.net ibtev0211
192.168.176.25  ibtev0212.private.net ibtev0212

# FY11 cluster nodes Infiniband network
192.168.176.40  ibtev0301.private.net ibtev0301
192.168.176.41  ibtev0302.private.net ibtev0302
192.168.176.42  ibtev0303.private.net ibtev0303
192.168.176.43  ibtev0304.private.net ibtev0304
192.168.176.44  ibtev0305.private.net ibtev0305
192.168.176.45  ibtev0306.private.net ibtev0306
192.168.176.46  ibtev0307.private.net ibtev0307
192.168.176.47  ibtev0308.private.net ibtev0308
192.168.176.48  ibtev0309.private.net ibtev0309
192.168.176.49  ibtev0310.private.net ibtev0310
192.168.176.50  ibtev0311.private.net ibtev0311
192.168.176.51  ibtev0312.private.net ibtev0312
192.168.176.52  ibtev0401.private.net ibtev0401
192.168.176.53  ibtev0402.private.net ibtev0402
192.168.176.54  ibtev0403.private.net ibtev0403
192.168.176.55  ibtev0404.private.net ibtev0404
192.168.176.56  ibtev0405.private.net ibtev0405
192.168.176.57  ibtev0406.private.net ibtev0406
192.168.176.58  ibtev0407.private.net ibtev0407
192.168.176.59  ibtev0408.private.net ibtev0408
192.168.176.60  ibtev0409.private.net ibtev0409
192.168.176.61  ibtev0410.private.net ibtev0410
192.168.176.62  ibtev0411.private.net ibtev0411
192.168.176.63  ibtev0412.private.net ibtev0412
192.168.176.64  ibtev0501.private.net ibtev0501
192.168.176.65  ibtev0502.private.net ibtev0502
192.168.176.66  ibtev0503.private.net ibtev0503
192.168.176.67  ibtev0504.private.net ibtev0504
192.168.176.68  ibtev0505.private.net ibtev0505
192.168.176.69  ibtev0506.private.net ibtev0506
192.168.176.70  ibtev0507.private.net ibtev0507
192.168.176.71  ibtev0508.private.net ibtev0508
192.168.176.72  ibtev0509.private.net ibtev0509
192.168.176.73  ibtev0510.private.net ibtev0510

# FPE grunt nodes
192.168.76.74 grunt1.private.net grunt1
192.168.76.75 grunt2.private.net grunt2
192.168.76.76 grunt3.private.net grunt3
192.168.76.77 grunt4.private.net grunt4
192.168.76.78 grunt5.private.net grunt5

# FPE grunt nodes
192.168.77.74 ipmigrunt1.private.net ipmigrunt1
192.168.77.75 ipmigrunt2.private.net ipmigrunt2
192.168.77.76 ipmigrunt3.private.net ipmigrunt3
192.168.77.77 ipmigrunt4.private.net ipmigrunt4
192.168.77.78 ipmigrunt5.private.net ipmigrunt5

# FPE grunt nodes
192.168.176.74  ibgrunt1.private.net ibgrunt1
192.168.176.75  ibgrunt2.private.net ibgrunt2
192.168.176.76  ibgrunt3.private.net ibgrunt3
192.168.176.77  ibgrunt4.private.net ibgrunt4
192.168.176.78  ibgrunt5.private.net ibgrunt5

/etc/hosts.equiv

tev
ibtev
tevnfs
ibtevnfs
cluckp
grunt1
grunt2
grunt3
grunt4
grunt5
ibgrunt1
ibgrunt2
ibgrunt3
ibgrunt4
ibgrunt5
tev0101
tev0102
tev0103
tev0104
tev0105
tev0106
tev0107
tev0108
tev0109
tev0110
tev0111
tev0112
tev0113
tev0201
tev0202
tev0203
tev0204
tev0205
tev0206
tev0207
tev0208
tev0209
tev0210
tev0211
tev0212
tev0301
tev0302
tev0303
tev0304
tev0305
tev0306
tev0307
tev0308
tev0309
tev0310
tev0311
tev0312
tev0401
tev0402
tev0403
tev0404
tev0405
tev0406
tev0407
tev0408
tev0409
tev0410
tev0411
tev0412
tev0501
tev0502
tev0503
tev0504
tev0505
tev0506
tev0507
tev0508
tev0509
tev0510
ibtev0101
ibtev0102
ibtev0103
ibtev0104
ibtev0105
ibtev0106
ibtev0107
ibtev0108
ibtev0109
ibtev0110
ibtev0111
ibtev0112
ibtev0113
ibtev0201
ibtev0202
ibtev0203
ibtev0204
ibtev0205
ibtev0206
ibtev0207
ibtev0208
ibtev0209
ibtev0210
ibtev0211
ibtev0212
ibtev0301
ibtev0302
ibtev0303
ibtev0304
ibtev0305
ibtev0306
ibtev0307
ibtev0308
ibtev0309
ibtev0310
ibtev0311
ibtev0312
ibtev0401
ibtev0402
ibtev0403
ibtev0404
ibtev0405
ibtev0406
ibtev0407
ibtev0408
ibtev0409
ibtev0410
ibtev0411
ibtev0412
ibtev0501
ibtev0502
ibtev0503
ibtev0504
ibtev0505
ibtev0506
ibtev0507
ibtev0508
ibtev0509
ibtev0510