Project

General

Profile

Necessary Maintenance #9789

down time on Aug 5th 2015

Added by Natalia Ratnikova over 4 years ago. Updated over 4 years ago.

Status:
Assigned
Priority:
High
Start date:
08/05/2015
Due date:
% Done:

0%

Estimated time:
Spent time:
Scope:
Internal
Experiment:
-
Stakeholders:
Duration:

Description

This is for the record of the work done.

History

#1 Updated by Natalia Ratnikova over 4 years ago

In check_mk go to host groups

click on the group name on top of the corresponding nodes list (dCache disk)
click on the tool in the top menu
declare downtime (2 hours)

Direct link to dCache:

https://ecfmon1.fnal.gov/dcsomon/check_mk/index.py?start_url=%2Fdcsomon%2Fcheck_mk%2Fview.py%3Fview_name%3Dhostgroup%26hostgroup%3DdCache-disk

Do the same for cmssrv228 and cmsphedex0disk, cmsphedex-lpc

#2 Updated by Natalia Ratnikova over 4 years ago

Chih-Hao reported that dcache tape instance is getting restarted.

Stopped phedex agents on cmssrv228, restart the node.

cmssrv228.fnal.gov - phedextape/production (SLF 6.6)
16-core Opteron 6320 (H8DGU); 62.90 GB RAM, 20.00 GB swap
[root@cmssrv228 ~]# su - cmsprod
bash-4.1$ cd /home/cmsprod/siteconf-git/
-bash-4.1$ ps -fu$USER | grep perl
cmsprod 3171 1 0 Apr21 ? 00:53:08 perl /home/cmsprod/phedex/4.1.3-comp3/sw/slc6_amd64_gcc461/cms/PHEDEX/PHEDEX_4.1.3-comp3/Toolkit/Transfer/FileExport -state /home/cmsprod/phedex/Dev_T1_US_FNAL_Buffer/state/exp-pfn-buffer/ -log /storage/local/data1/phedex/Dev_T1_US_FNAL_Buffer/logs/exp-pfn-buffer -db /home/cmsprod/phedex/info/DBParam:Dev/T1USFNAL -nodes T1_US_FNAL_Buffer,T1_US_FNAL_MSS -storagemap /home/cmsprod/siteconf-git//storage-dev.xml -protocols srmv2,direct
cmsprod 3183 1 0 Apr21 ? 00:54:50 perl /home/cmsprod/phedex/4.1.3-comp3/sw/slc6_amd64_gcc461/cms/PHEDEX/PHEDEX_4.1.3-comp3/Toolkit/Transfer/FileExport -state /home/cmsprod/phedex/Dev_T1_US_FNAL_Buffer/state/exp-pfn-mss/ -log /storage/local/data1/phedex/Dev_T1_US_FNAL_Buffer/logs/exp-pfn-mss -db /home/cmsprod/phedex/info/DBParam:Dev/T1USFNAL -nodes T1_US_FNAL_MSS -storagemap /home/cmsprod/siteconf-git//storage-dev.xml -protocols direct
cmsprod 3195 1 0 Apr21 ? 01:32:37 perl /home/cmsprod/phedex/4.1.3-comp3/sw/slc6_amd64_gcc461/cms/PHEDEX/PHEDEX_4.1.3-comp3/Toolkit/Transfer/FileRemove -state /home/cmsprod/phedex/Dev_T1_US_FNAL_Buffer/state/download-remove/ -log /storage/local/data1/phedex/Dev_T1_US_FNAL_Buffer/logs/download-remove -db /home/cmsprod/phedex/info/DBParam:Dev/T1USFNAL -nodes T1_US_FNAL_Buffer,T1_US_FNAL_MSS -delete /home/cmsprod/siteconf-git//plugins/FileDownloadDeleteDirect -protocol direct -jobs 4
cmsprod 3209 1 0 Apr21 ? 01:43:08 perl /home/cmsprod/phedex/4.1.3-comp3/sw/slc6_amd64_gcc461/cms/PHEDEX/PHEDEX_4.1.3-comp3/Toolkit/Transfer/FileDownload -state /home/cmsprod/phedex/Dev_T1_US_FNAL_Buffer/state/download-cern/ -log /storage/local/data1/phedex/Dev_T1_US_FNAL_Buffer/logs/download-cern -db /home/cmsprod/phedex/info/DBParam:Dev/T1USFNAL -nodes T1_US_FNAL_Buffer -accept CERN -delete /home/cmsprod/siteconf-git//plugins/FileDownloadDelete -validate /home/cmsprod/siteconf-git//plugins/FileDownloadVerify-chimera,-c,.*SFN=/11/,/pnfs/fs/usr/cms/WAX/11/,dcap -verbose -max-tasks 5000 -util-jobs 5000 -jobs 200 -util-timeout 3600 -backend FTS -protocols srmv2 -service https://cmsfts3.fnal.gov:8443 -q_interval 5 -j_interval 4 -max-active-files 1284 -batch-files 10 -link-pending-files 1284
cmsprod 3222 1 0 Apr21 ? 01:43:18 perl /home/cmsprod/phedex/4.1.3-comp3/sw/slc6_amd64_gcc461/cms/PHEDEX/PHEDEX_4.1.3-comp3/Toolkit/Transfer/FileDownload -state /home/cmsprod/phedex/Dev_T1_US_FNAL_Buffer/state/download-tier1/ -log /storage/local/data1/phedex/Dev_T1_US_FNAL_Buffer/logs/download-tier1 -db /home/cmsprod/phedex/info/DBParam:Dev/T1USFNAL -nodes T1_US_FNAL_Buffer -accept T1% -ignore CERN -delete /home/cmsprod/siteconf-git//plugins/FileDownloadDelete -validate /home/cmsprod/siteconf-git//plugins/FileDownloadVerify-chimera,-c,.*SFN=/11/,/pnfs/fs/usr/cms/WAX/11/,dcap -verbose -max-tasks 5000 -util-jobs 5000 -jobs 200 -util-timeout 3600 -backend FTS -protocols srmv2 -service https://cmsfts3.fnal.gov:8443 -q_interval 5 -j_interval 4 -max-active-files 5000 -batch-files 5 -link-pending-files 5000
cmsprod 3235 1 0 Apr21 ? 01:44:03 perl /home/cmsprod/phedex/4.1.3-comp3/sw/slc6_amd64_gcc461/cms/PHEDEX/PHEDEX_4.1.3-comp3/Toolkit/Transfer/FileDownload -state /home/cmsprod/phedex/Dev_T1_US_FNAL_Buffer/state/download-tier2/ -log /storage/local/data1/phedex/Dev_T1_US_FNAL_Buffer/logs/download-tier2 -db /home/cmsprod/phedex/info/DBParam:Dev/T1USFNAL -nodes T1_US_FNAL_Buffer -accept T2% -ignore CERN -delete /home/cmsprod/siteconf-git//plugins/FileDownloadDelete -validate /home/cmsprod/siteconf-git//plugins/FileDownloadVerify-chimera,-c,.*SFN=/11/,/pnfs/fs/usr/cms/WAX/11/,dcap -verbose -max-tasks 5000 -util-jobs 5000 -jobs 200 -util-timeout 3600 -backend FTS -protocols srmv2 -service https://cmsfts3.fnal.gov:8443 -q_interval 5 -j_interval 4 -max-active-files 5000 -batch-files 5 -link-pending-files 5000
cmsprod 3248 1 0 Apr21 ? 01:44:02 perl /home/cmsprod/phedex/4.1.3-comp3/sw/slc6_amd64_gcc461/cms/PHEDEX/PHEDEX_4.1.3-comp3/Toolkit/Transfer/FileDownload -state /home/cmsprod/phedex/Dev_T1_US_FNAL_Buffer/state/download-tier3/ -log /storage/local/data1/phedex/Dev_T1_US_FNAL_Buffer/logs/download-tier3 -db /home/cmsprod/phedex/info/DBParam:Dev/T1USFNAL -nodes T1_US_FNAL_Buffer -accept T3% -ignore CERN -delete /home/cmsprod/siteconf-git//plugins/FileDownloadDelete -validate /home/cmsprod/siteconf-git//plugins/FileDownloadVerify-chimera,-c,.*SFN=/11/,/pnfs/fs/usr/cms/WAX/11/,dcap -verbose -max-tasks 5000 -util-jobs 5000 -jobs 200 -util-timeout 3600 -backend FTS -protocols srmv2 -service https://cmsfts3.fnal.gov:8443 -q_interval 5 -j_interval 4 -max-active-files 5000 -batch-files 5 -link-pending-files 5000
cmsprod 3261 1 0 Apr21 ? 01:26:21 perl /home/cmsprod/phedex/4.1.3-comp3/sw/slc6_amd64_gcc461/cms/PHEDEX/PHEDEX_4.1.3-comp3/Toolkit/Transfer/FileStager -state /home/cmsprod/phedex/Dev_T1_US_FNAL_Buffer/state/mss-stage-local/ -log /storage/local/data1/phedex/Dev_T1_US_FNAL_Buffer/logs/mss-stage-local -accept T1_US_FNAL_Disk -db /home/cmsprod/phedex/info/DBParam:Dev/T1USFNAL -nodes T1_US_FNAL_Buffer -storagemap /home/cmsprod/siteconf-git//storage-dev.xml -protocol srmv2 -stage /usr/bin/srm-bring-online -status /home/cmsprod/siteconf-git//plugins/chimeradirect_prestage-status-2disk.py -maxfiles 4000 -batch-files 400 -waittime 60
cmsprod 3273 1 0 Apr21 ? 01:32:06 perl /home/cmsprod/phedex/4.1.3-comp3/sw/slc6_amd64_gcc461/cms/PHEDEX/PHEDEX_4.1.3-comp3/Toolkit/Transfer/FileStager -state /home/cmsprod/phedex/Dev_T1_US_FNAL_Buffer/state/mss-stage/ -log /storage/local/data1/phedex/Dev_T1_US_FNAL_Buffer/logs/mss-stage -ignore T1_US_FNAL_Disk,%CERN% -db /home/cmsprod/phedex/info/DBParam:Dev/T1USFNAL -nodes T1_US_FNAL_Buffer -storagemap /home/cmsprod/siteconf-git//storage-dev.xml -protocol srmv2 -stage /usr/bin/srm-bring-online -status /home/cmsprod/siteconf-git//plugins/chimeradirect_prestage-status.py -maxfiles 4000 -batch-files 400 -waittime 60
cmsprod 3285 1 0 Apr21 ? 01:21:23 perl /home/cmsprod/phedex/4.1.3-comp3/sw/slc6_amd64_gcc461/cms/PHEDEX/PHEDEX_4.1.3-comp3/Toolkit/Transfer/FileStager -state /home/cmsprod/phedex/Dev_T1_US_FNAL_Buffer/state/mss-stage-cern/ -log /storage/local/data1/phedex/Dev_T1_US_FNAL_Buffer/logs/mss-stage-cern -accept CERN -db /home/cmsprod/phedex/info/DBParam:Dev/T1USFNAL -nodes T1_US_FNAL_Buffer -storagemap /home/cmsprod/siteconf-git//storage-dev.xml -protocol srmv2 -stage /usr/bin/srm-bring-online -status /home/cmsprod/siteconf-git//plugins/chimeradirect_prestage-status.py -maxfiles 4000 -batch-files 400 -waittime 60
cmsprod 3299 1 0 Apr21 ? 01:06:53 perl /home/cmsprod/phedex/4.1.3-comp3/sw/slc6_amd64_gcc461/cms/PHEDEX/PHEDEX_4.1.3-comp3/Toolkit/Transfer/FileMSSMigrate -state /home/cmsprod/phedex/Dev_T1_US_FNAL_Buffer/state/mss-migrate/ -log /storage/local/data1/phedex/Dev_T1_US_FNAL_Buffer/logs/mss-migrate -db /home/cmsprod/phedex/info/DBParam:Dev/T1USFNAL -nodes T1_US_FNAL_MSS -protocols srmv2 -checkroutine /home/cmsprod/siteconf-git//plugins/chimeradirectMigrationCheck.pm
cmsprod 3382 1 0 Apr21 ? 01:02:20 perl /home/cmsprod/phedex/4.1.3-comp3/sw/slc6_amd64_gcc461/cms/PHEDEX/PHEDEX_4.1.3-comp3/Toolkit/Verify/BlockDownloadVerify -state /home/cmsprod/phedex/Dev_T1_US_FNAL_Buffer/state/blockverify/ -log /storage/local/data1/phedex/Dev_T1_US_FNAL_Buffer/logs/blockverify -db /home/cmsprod/phedex/info/DBParam:Dev/T1USFNAL -nodes T1_US_FNAL_Buffer,T1_US_FNAL_MSS -namespace dcache -
NOCACHE 1
cmsprod 3957 1 0 Apr21 ? 01:03:45 perl /home/cmsprod/phedex/4.1.3-comp3/sw/slc6_amd64_gcc461/cms/PHEDEX/PHEDEX_4.1.3-comp3/Utilities/AgentFactory.pl state /home/cmsprod/phedex/Dev_T1_US_FNAL_Buffer/state/watchdog/ -log /storage/local/data1/phedex/Dev_T1_US_FNAL_Buffer/logs/watchdog -db /home/cmsprod/phedex/info/DBParam:Dev/T1USFNAL -node T1_US_FNAL_Buffer -agent_list download-cern -agent_list download-tier1 -agent_list download-tier2 -agent_list download-tier3 -agent_list download-remove -agent_list exp-pfn-buffer -agent_list exp-pfn-mss -agent_list mss-migrate -agent_list mss-stage-local -agent_list mss-stage -agent_list mss-stage-cern -agent_list blockverify
cmsprod 4061 1 0 Apr21 ? 00:53:43 perl /home/cmsprod/phedex/4.1.3-comp3/sw/slc6_amd64_gcc461/cms/PHEDEX/PHEDEX_4.1.3-comp3/Toolkit/Transfer/FileExport -state /home/cmsprod/phedex/Debug_T1_US_FNAL_Buffer/state/exp-pfn-buffer/ -log /storage/local/data1/phedex/Debug_T1_US_FNAL_Buffer/logs/exp-pfn-buffer -db /home/cmsprod/phedex/info/DBParam:Debug/T1USFNAL -nodes T1_US_FNAL_Buffer,T1_US_FNAL_MSS -storagemap /home/cmsprod/siteconf-git//storage.xml -protocols srmv2,direct
cmsprod 4074 1 0 Apr21 ? 00:53:31 perl /home/cmsprod/phedex/4.1.3-comp3/sw/slc6_amd64_gcc461/cms/PHEDEX/PHEDEX_4.1.3-comp3/Toolkit/Transfer/FileExport -state /home/cmsprod/phedex/Debug_T1_US_FNAL_Buffer/state/exp-pfn-mss/ -log /storage/local/data1/phedex/Debug_T1_US_FNAL_Buffer/logs/exp-pfn-mss -db /home/cmsprod/phedex/info/DBParam:Debug/T1USFNAL -nodes T1_US_FNAL_MSS -storagemap /home/cmsprod/siteconf-git//storage.xml -protocols direct
cmsprod 4087 1 0 Apr21 ? 01:35:38 perl /home/cmsprod/phedex/4.1.3-comp3/sw/slc6_amd64_gcc461/cms/PHEDEX/PHEDEX_4.1.3-comp3/Toolkit/Transfer/FileRemove -state /home/cmsprod/phedex/Debug_T1_US_FNAL_Buffer/state/download-remove/ -log /storage/local/data1/phedex/Debug_T1_US_FNAL_Buffer/logs/download-remove -db /home/cmsprod/phedex/info/DBParam:Debug/T1USFNAL -nodes T1_US_FNAL_Buffer,T1_US_FNAL_MSS -delete /home/cmsprod/siteconf-git//plugins/FileDownloadDeleteDirect -protocol direct -jobs 4
cmsprod 4100 1 0 Apr21 ? 11:03:49 perl /home/cmsprod/phedex/4.1.3-comp3/sw/slc6_amd64_gcc461/cms/PHEDEX/PHEDEX_4.1.3-comp3/Toolkit/Transfer/FileDownload -state /home/cmsprod/phedex/Debug_T1_US_FNAL_Buffer/state/download-cern/ -log /storage/local/data1/phedex/Debug_T1_US_FNAL_Buffer/logs/download-cern -db /home/cmsprod/phedex/info/DBParam:Debug/T1USFNAL -nodes T1_US_FNAL_Buffer -accept CERN -delete /home/cmsprod/siteconf-git//plugins/FileDownloadDelete -validate /home/cmsprod/siteconf-git//plugins/FileDownloadVerify-chimera,-c,.*SFN=/11/,/pnfs/fs/usr/cms/WAX/11/,dcap -verbose -max-tasks 5000 -util-jobs 5000 -jobs 200 -util-timeout 3600 -backend FTS -protocols srmv2 -service https://cmsfts3.fnal.gov:8443 -q_interval 5 -j_interval 4 -max-active-files 1284 -batch-files 10 -link-pending-files 1284
cmsprod 4114 1 0 Apr21 ? 05:59:30 perl /home/cmsprod/phedex/4.1.3-comp3/sw/slc6_amd64_gcc461/cms/PHEDEX/PHEDEX_4.1.3-comp3/Toolkit/Transfer/FileDownload -state /home/cmsprod/phedex/Debug_T1_US_FNAL_Buffer/state/download-tier1/ -log /storage/local/data1/phedex/Debug_T1_US_FNAL_Buffer/logs/download-tier1 -db /home/cmsprod/phedex/info/DBParam:Debug/T1USFNAL -nodes T1_US_FNAL_Buffer -accept T1% -ignore CERN -delete /home/cmsprod/siteconf-git//plugins/FileDownloadDelete -validate /home/cmsprod/siteconf-git//plugins/FileDownloadVerify-chimera,-c,.*SFN=/11/,/pnfs/fs/usr/cms/WAX/11/,dcap -verbose -max-tasks 5000 -util-jobs 5000 -jobs 200 -util-timeout 3600 -backend FTS -protocols srmv2 -service https://cmsfts3.fnal.gov:8443 -q_interval 5 -j_interval 4 -max-active-files 5000 -batch-files 5 -link-pending-files 5000
cmsprod 4127 1 0 Apr21 ? 17:18:14 perl /home/cmsprod/phedex/4.1.3-comp3/sw/slc6_amd64_gcc461/cms/PHEDEX/PHEDEX_4.1.3-comp3/Toolkit/Transfer/FileDownload -state /home/cmsprod/phedex/Debug_T1_US_FNAL_Buffer/state/download-tier2/ -log /storage/local/data1/phedex/Debug_T1_US_FNAL_Buffer/logs/download-tier2 -db /home/cmsprod/phedex/info/DBParam:Debug/T1USFNAL -nodes T1_US_FNAL_Buffer -accept T2% -ignore CERN -delete /home/cmsprod/siteconf-git//plugins/FileDownloadDelete -validate /home/cmsprod/siteconf-git//plugins/FileDownloadVerify-chimera,-c,.*SFN=/11/,/pnfs/fs/usr/cms/WAX/11/,dcap -verbose -max-tasks 5000 -util-jobs 5000 -jobs 200 -util-timeout 3600 -backend FTS -protocols srmv2 -service https://cmsfts3.fnal.gov:8443 -q_interval 5 -j_interval 4 -max-active-files 5000 -batch-files 5 -link-pending-files 5000
cmsprod 4140 1 0 Apr21 ? 01:57:37 perl /home/cmsprod/phedex/4.1.3-comp3/sw/slc6_amd64_gcc461/cms/PHEDEX/PHEDEX_4.1.3-comp3/Toolkit/Transfer/FileDownload -state /home/cmsprod/phedex/Debug_T1_US_FNAL_Buffer/state/download-tier3/ -log /storage/local/data1/phedex/Debug_T1_US_FNAL_Buffer/logs/download-tier3 -db /home/cmsprod/phedex/info/DBParam:Debug/T1USFNAL -nodes T1_US_FNAL_Buffer -accept T3% -ignore CERN -delete /home/cmsprod/siteconf-git//plugins/FileDownloadDelete -validate /home/cmsprod/siteconf-git//plugins/FileDownloadVerify-chimera,-c,.*SFN=/11/,/pnfs/fs/usr/cms/WAX/11/,dcap -verbose -max-tasks 5000 -util-jobs 5000 -jobs 200 -util-timeout 3600 -backend FTS -protocols srmv2 -service https://cmsfts3.fnal.gov:8443 -q_interval 5 -j_interval 4 -max-active-files 5000 -batch-files 5 -link-pending-files 5000
cmsprod 4153 1 0 Apr21 ? 01:18:58 perl /home/cmsprod/phedex/4.1.3-comp3/sw/slc6_amd64_gcc461/cms/PHEDEX/PHEDEX_4.1.3-comp3/Toolkit/Transfer/FileStager -state /home/cmsprod/phedex/Debug_T1_US_FNAL_Buffer/state/mss-stage-local/ -log /storage/local/data1/phedex/Debug_T1_US_FNAL_Buffer/logs/mss-stage-local -accept T1_US_FNAL_Disk -db /home/cmsprod/phedex/info/DBParam:Debug/T1USFNAL -nodes T1_US_FNAL_Buffer -storagemap /home/cmsprod/siteconf-git//storage.xml -protocol srmv2 -stage /usr/bin/srm-bring-online -status /home/cmsprod/siteconf-git//plugins/chimeradirect_prestage-status-2disk.py -maxfiles 4000 -batch-files 400 -waittime 60
cmsprod 4165 1 0 Apr21 ? 02:57:38 perl /home/cmsprod/phedex/4.1.3-comp3/sw/slc6_amd64_gcc461/cms/PHEDEX/PHEDEX_4.1.3-comp3/Toolkit/Transfer/FileStager -state /home/cmsprod/phedex/Debug_T1_US_FNAL_Buffer/state/mss-stage/ -log /storage/local/data1/phedex/Debug_T1_US_FNAL_Buffer/logs/mss-stage -ignore T1_US_FNAL_Disk,%CERN% -db /home/cmsprod/phedex/info/DBParam:Debug/T1USFNAL -nodes T1_US_FNAL_Buffer -storagemap /home/cmsprod/siteconf-git//storage.xml -protocol srmv2 -stage /usr/bin/srm-bring-online -status /home/cmsprod/siteconf-git//plugins/chimeradirect_prestage-status.py -maxfiles 4000 -batch-files 400 -waittime 60
cmsprod 4177 1 0 Apr21 ? 01:21:43 perl /home/cmsprod/phedex/4.1.3-comp3/sw/slc6_amd64_gcc461/cms/PHEDEX/PHEDEX_4.1.3-comp3/Toolkit/Transfer/FileStager -state /home/cmsprod/phedex/Debug_T1_US_FNAL_Buffer/state/mss-stage-cern/ -log /storage/local/data1/phedex/Debug_T1_US_FNAL_Buffer/logs/mss-stage-cern -accept CERN -db /home/cmsprod/phedex/info/DBParam:Debug/T1USFNAL -nodes T1_US_FNAL_Buffer -storagemap /home/cmsprod/siteconf-git//storage.xml -protocol srmv2 -stage /usr/bin/srm-bring-online -status /home/cmsprod/siteconf-git//plugins/chimeradirect_prestage-status.py -maxfiles 4000 -batch-files 400 -waittime 60
cmsprod 4193 1 1 Apr21 ? 1-10:24:16 perl /home/cmsprod/phedex/4.1.3-comp3/sw/slc6_amd64_gcc461/cms/PHEDEX/PHEDEX_4.1.3-comp3/Toolkit/Transfer/FileMSSMigrate -state /home/cmsprod/phedex/Debug_T1_US_FNAL_Buffer/state/mss-migrate/ -log /storage/local/data1/phedex/Debug_T1_US_FNAL_Buffer/logs/mss-migrate -db /home/cmsprod/phedex/info/DBParam:Debug/T1USFNAL -nodes T1_US_FNAL_MSS -protocols srmv2 -checkroutine /home/cmsprod/siteconf-git//plugins/chimeradirectMigrationCheck.pm
cmsprod 4275 1 0 Apr21 ? 01:01:56 perl /home/cmsprod/phedex/4.1.3-comp3/sw/slc6_amd64_gcc461/cms/PHEDEX/PHEDEX_4.1.3-comp3/Toolkit/Verify/BlockDownloadVerify -state /home/cmsprod/phedex/Debug_T1_US_FNAL_Buffer/state/blockverify/ -log /storage/local/data1/phedex/Debug_T1_US_FNAL_Buffer/logs/blockverify -db /home/cmsprod/phedex/info/DBParam:Debug/T1USFNAL -nodes T1_US_FNAL_Buffer,T1_US_FNAL_MSS -namespace dcache -
NOCACHE 1
cmsprod 4847 1 0 Apr21 ? 01:08:13 perl /home/cmsprod/phedex/4.1.3-comp3/sw/slc6_amd64_gcc461/cms/PHEDEX/PHEDEX_4.1.3-comp3/Utilities/AgentFactory.pl state /home/cmsprod/phedex/Debug_T1_US_FNAL_Buffer/state/watchdog/ -log /storage/local/data1/phedex/Debug_T1_US_FNAL_Buffer/logs/watchdog -db /home/cmsprod/phedex/info/DBParam:Debug/T1USFNAL -node T1_US_FNAL_Buffer -agent_list download-cern -agent_list download-tier1 -agent_list download-tier2 -agent_list download-tier3 -agent_list download-remove -agent_list exp-pfn-buffer -agent_list exp-pfn-mss -agent_list mss-migrate -agent_list mss-stage-local -agent_list mss-stage -agent_list mss-stage-cern -agent_list blockverify
cmsprod 5027 1 0 Apr21 ? 00:51:43 perl /home/cmsprod/phedex/4.1.3-comp3/sw/slc6_amd64_gcc461/cms/PHEDEX/PHEDEX_4.1.3-comp3/Toolkit/Transfer/FileExport -state /home/cmsprod/phedex/Prod_T1_US_FNAL_Buffer/state/exp-pfn-buffer/ -log /storage/local/data1/phedex/Prod_T1_US_FNAL_Buffer/logs/exp-pfn-buffer -db /home/cmsprod/phedex/info/DBParam:Prod/T1USFNAL -nodes T1_US_FNAL_Buffer,T1_US_FNAL_MSS -storagemap /home/cmsprod/siteconf-git//storage.xml -protocols srmv2,direct
cmsprod 5044 1 0 Apr21 ? 00:52:22 perl /home/cmsprod/phedex/4.1.3-comp3/sw/slc6_amd64_gcc461/cms/PHEDEX/PHEDEX_4.1.3-comp3/Toolkit/Transfer/FileExport -state /home/cmsprod/phedex/Prod_T1_US_FNAL_Buffer/state/exp-pfn-mss/ -log /storage/local/data1/phedex/Prod_T1_US_FNAL_Buffer/logs/exp-pfn-mss -db /home/cmsprod/phedex/info/DBParam:Prod/T1USFNAL -nodes T1_US_FNAL_MSS -storagemap /home/cmsprod/siteconf-git//storage.xml -protocols direct
cmsprod 5258 1 0 Apr21 ? 01:36:14 perl /home/cmsprod/phedex/4.1.3-comp3/sw/slc6_amd64_gcc461/cms/PHEDEX/PHEDEX_4.1.3-comp3/Toolkit/Transfer/FileRemove -state /home/cmsprod/phedex/Prod_T1_US_FNAL_Buffer/state/download-remove/ -log /storage/local/data1/phedex/Prod_T1_US_FNAL_Buffer/logs/download-remove -db /home/cmsprod/phedex/info/DBParam:Prod/T1USFNAL -nodes T1_US_FNAL_Buffer,T1_US_FNAL_MSS -delete /home/cmsprod/siteconf-git//plugins/FileDownloadDeleteDirect -protocol direct -jobs 4
cmsprod 5487 1 0 Apr21 ? 14:17:03 perl /home/cmsprod/phedex/4.1.3-comp3/sw/slc6_amd64_gcc461/cms/PHEDEX/PHEDEX_4.1.3-comp3/Toolkit/Transfer/FileDownload -state /home/cmsprod/phedex/Prod_T1_US_FNAL_Buffer/state/download-tier1/ -log /storage/local/data1/phedex/Prod_T1_US_FNAL_Buffer/logs/download-tier1 -db /home/cmsprod/phedex/info/DBParam:Prod/T1USFNAL -nodes T1_US_FNAL_Buffer -accept T1% -ignore CERN -delete /home/cmsprod/siteconf-git//plugins/FileDownloadDelete -validate /home/cmsprod/siteconf-git//plugins/FileDownloadVerify-chimera,-c,.*SFN=/11/,/pnfs/fs/usr/cms/WAX/11/,dcap -verbose -max-tasks 5000 -util-jobs 5000 -jobs 200 -util-timeout 3600 -backend FTS -protocols srmv2 -service https://cmsfts3.fnal.gov:8443 -q_interval 5 -j_interval 4 -max-active-files 5000 -batch-files 5 -link-pending-files 5000
cmsprod 5522 1 0 Apr21 ? 07:44:26 perl /home/cmsprod/phedex/4.1.3-comp3/sw/slc6_amd64_gcc461/cms/PHEDEX/PHEDEX_4.1.3-comp3/Toolkit/Transfer/FileDownload -state /home/cmsprod/phedex/Prod_T1_US_FNAL_Buffer/state/download-tier2/ -log /storage/local/data1/phedex/Prod_T1_US_FNAL_Buffer/logs/download-tier2 -db /home/cmsprod/phedex/info/DBParam:Prod/T1USFNAL -nodes T1_US_FNAL_Buffer -accept T2% -ignore CERN -delete /home/cmsprod/siteconf-git//plugins/FileDownloadDelete -validate /home/cmsprod/siteconf-git//plugins/FileDownloadVerify-chimera,-c,.*SFN=/11/,/pnfs/fs/usr/cms/WAX/11/,dcap -verbose -max-tasks 5000 -util-jobs 5000 -jobs 200 -util-timeout 3600 -backend FTS -protocols srmv2 -service https://cmsfts3.fnal.gov:8443 -q_interval 5 -j_interval 4 -max-active-files 5000 -batch-files 5 -link-pending-files 5000
cmsprod 5591 1 0 Apr21 ? 01:43:56 perl /home/cmsprod/phedex/4.1.3-comp3/sw/slc6_amd64_gcc461/cms/PHEDEX/PHEDEX_4.1.3-comp3/Toolkit/Transfer/FileDownload -state /home/cmsprod/phedex/Prod_T1_US_FNAL_Buffer/state/download-tier3/ -log /storage/local/data1/phedex/Prod_T1_US_FNAL_Buffer/logs/download-tier3 -db /home/cmsprod/phedex/info/DBParam:Prod/T1USFNAL -nodes T1_US_FNAL_Buffer -accept T3% -ignore CERN -delete /home/cmsprod/siteconf-git//plugins/FileDownloadDelete -validate /home/cmsprod/siteconf-git//plugins/FileDownloadVerify-chimera,-c,.*SFN=/11/,/pnfs/fs/usr/cms/WAX/11/,dcap -verbose -max-tasks 5000 -util-jobs 5000 -jobs 200 -util-timeout 3600 -backend FTS -protocols srmv2 -service https://cmsfts3.fnal.gov:8443 -q_interval 5 -j_interval 4 -max-active-files 5000 -batch-files 5 -link-pending-files 5000
cmsprod 5610 1 0 Apr21 ? 01:29:31 perl /home/cmsprod/phedex/4.1.3-comp3/sw/slc6_amd64_gcc461/cms/PHEDEX/PHEDEX_4.1.3-comp3/Toolkit/Transfer/FileStager -state /home/cmsprod/phedex/Prod_T1_US_FNAL_Buffer/state/mss-stage-local/ -log /storage/local/data1/phedex/Prod_T1_US_FNAL_Buffer/logs/mss-stage-local -accept T1_US_FNAL_Disk -db /home/cmsprod/phedex/info/DBParam:Prod/T1USFNAL -nodes T1_US_FNAL_Buffer -storagemap /home/cmsprod/siteconf-git//storage.xml -protocol srmv2 -stage /usr/bin/srm-bring-online -status /home/cmsprod/siteconf-git//plugins/chimeradirect_prestage-status-2disk.py -maxfiles 4000 -batch-files 400 -waittime 60
cmsprod 5628 1 0 Apr21 ? 05:50:16 perl /home/cmsprod/phedex/4.1.3-comp3/sw/slc6_amd64_gcc461/cms/PHEDEX/PHEDEX_4.1.3-comp3/Toolkit/Transfer/FileStager -state /home/cmsprod/phedex/Prod_T1_US_FNAL_Buffer/state/mss-stage/ -log /storage/local/data1/phedex/Prod_T1_US_FNAL_Buffer/logs/mss-stage -ignore T1_US_FNAL_Disk,%CERN% -db /home/cmsprod/phedex/info/DBParam:Prod/T1USFNAL -nodes T1_US_FNAL_Buffer -storagemap /home/cmsprod/siteconf-git//storage.xml -protocol srmv2 -stage /usr/bin/srm-bring-online -status /home/cmsprod/siteconf-git//plugins/chimeradirect_prestage-status.py -maxfiles 4000 -batch-files 400 -waittime 60
cmsprod 5652 1 0 Apr21 ? 01:23:53 perl /home/cmsprod/phedex/4.1.3-comp3/sw/slc6_amd64_gcc461/cms/PHEDEX/PHEDEX_4.1.3-comp3/Toolkit/Transfer/FileStager -state /home/cmsprod/phedex/Prod_T1_US_FNAL_Buffer/state/mss-stage-cern/ -log /storage/local/data1/phedex/Prod_T1_US_FNAL_Buffer/logs/mss-stage-cern -accept CERN -db /home/cmsprod/phedex/info/DBParam:Prod/T1USFNAL -nodes T1_US_FNAL_Buffer -storagemap /home/cmsprod/siteconf-git//storage.xml -protocol srmv2 -stage /usr/bin/srm-bring-online -status /home/cmsprod/siteconf-git//plugins/chimeradirect_prestage-status.py -maxfiles 4000 -batch-files 400 -waittime 60
cmsprod 5779 1 0 Apr21 ? 16:27:40 perl /home/cmsprod/phedex/4.1.3-comp3/sw/slc6_amd64_gcc461/cms/PHEDEX/PHEDEX_4.1.3-comp3/Toolkit/Verify/BlockDownloadVerify -state /home/cmsprod/phedex/Prod_T1_US_FNAL_Buffer/state/blockverify/ -log /storage/local/data1/phedex/Prod_T1_US_FNAL_Buffer/logs/blockverify -db /home/cmsprod/phedex/info/DBParam:Prod/T1USFNAL -nodes T1_US_FNAL_Buffer,T1_US_FNAL_MSS -namespace dcache -
NOCACHE 1
cmsprod 6540 1 0 Apr21 ? 01:23:45 perl /home/cmsprod/phedex/4.1.3-comp3/sw/slc6_amd64_gcc461/cms/PHEDEX/PHEDEX_4.1.3-comp3/Utilities/AgentFactory.pl -state /home/cmsprod/phedex/Prod_T1_US_FNAL_Buffer/state/watchdog/ -log /storage/local/data1/phedex/Prod_T1_US_FNAL_Buffer/logs/watchdog -db /home/cmsprod/phedex/info/DBParam:Prod/T1USFNAL -node T1_US_FNAL_Buffer -agent_list download-cern -agent_list download-tier1 -agent_list download-tier2 -agent_list download-tier3 -agent_list download-remove -agent_list exp-pfn-buffer -agent_list exp-pfn-mss -agent_list mss-migrate -agent_list mss-stage-local -agent_list mss-stage -agent_list mss-stage-cern -agent_list blockverify
cmsprod 10761 1 0 May11 ? 17:08:24 perl /home/cmsprod/phedex/4.1.3-comp3/sw/slc6_amd64_gcc461/cms/PHEDEX/PHEDEX_4.1.3-comp3/Toolkit/Transfer/FileDownload -state /home/cmsprod/phedex/Prod_T1_US_FNAL_Buffer/state/download-cern/ -log /storage/local/data1/phedex/Prod_T1_US_FNAL_Buffer/logs/download-cern -db /home/cmsprod/phedex/info/DBParam:Prod/T1USFNAL -nodes T1_US_FNAL_Buffer -accept CERN -delete /home/cmsprod/siteconf-git//plugins/FileDownloadDelete -validate /home/cmsprod/siteconf-git//plugins/FileDownloadVerify-chimera,-c,.*SFN=/11/,/pnfs/fs/usr/cms/WAX/11/,dcap -verbose -max-tasks 5000 -util-jobs 5000 -jobs 200 -util-timeout 3600 -backend FTS -protocols srmv2 -service https://cmsfts3.fnal.gov:8443 -q_interval 5 -j_interval 4 -max-active-files 1284 -batch-files 10 -link-pending-files 1284
cmsprod 20911 1 0 Jul08 ? 02:20:52 perl /home/cmsprod/phedex/4.1.3-comp3/sw/slc6_amd64_gcc461/cms/PHEDEX/PHEDEX_4.1.3-comp3/Toolkit/Transfer/FileMSSMigrate -state /home/cmsprod/phedex/Prod_T1_US_FNAL_Buffer/state/mss-migrate/ -log /storage/local/data1/phedex/Prod_T1_US_FNAL_Buffer/logs/mss-migrate -db /home/cmsprod/phedex/info/DBParam:Prod/T1USFNAL -nodes T1_US_FNAL_MSS -protocols srmv2 -checkroutine /home/cmsprod/siteconf-git//plugins/chimeradirectMigrationCheck.pm
cmsprod 25329 4165 0 09:26 ? 00:00:00 perl /home/cmsprod/phedex/4.1.3-comp3/sw/slc6_amd64_gcc461/cms/PHEDEX/PHEDEX_4.1.3-comp3/Toolkit/Transfer/../../Utilities/RunWithTimeout 600 /usr/bin/srm-bring-online srm://cmssrm.fnal.gov:8443/srm/managerv2?SFN=/11/store/PhEDEx_LoadTest07/LoadTest07_Debug_T1_US_FNAL_Disk/T1_US_FNAL_Buffer/19/LoadTest07_LoadTest07_FNAL_75_xjUnifsw4Pwp7R0C_19
cmsprod 27911 27539 0 09:34 pts/1 00:00:00 grep perl
-bash-4.1$ ps -fu$USER | grep perl | wc -l
41
-bash-4.1$ ./pmgr Dev stop
Wed Aug 5 09:35:04 2015: Waiting for agents: mss-migrate(3299), mss-stage-cern(3285), blockverify(3382), watchdog(3957), download-tier1(3222), exp-pfn-mss(3183), exp-pfn-buffer(3171), download-tier3(3248), mss-stage-local(3261), mss-stage(3273), download-remove(3195), download-tier2(3235), download-cern(3209)
Wed Aug 5 09:35:07 2015: Waiting for agents: mss-migrate(3299), watchdog(3957), download-tier1(3222), exp-pfn-mss(3183), exp-pfn-buffer(3171), download-tier3(3248), download-remove(3195), download-cern(3209)
All agents sucessfully terminated
T1 tape pmgr: stop Dev
-bash-4.1$ ./pmgr Prod stop
Wed Aug 5 09:35:16 2015: Waiting for agents: mss-stage-cern(5652), mss-stage-local(5610), download-tier1(5487), download-tier3(5591), download-remove(5258), exp-pfn-buffer(5027), download-cern(10761), blockverify(5779), watchdog(6540)
Wed Aug 5 09:35:19 2015: Waiting for agents: blockverify(5779)
Wed Aug 5 09:35:22 2015: Waiting for agents: blockverify(5779)
Wed Aug 5 09:35:25 2015: Waiting for agents: blockverify(5779)
Wed Aug 5 09:35:28 2015: Waiting for agents: blockverify(5779)
Wed Aug 5 09:35:31 2015: Waiting for agents: blockverify(5779)
Wed Aug 5 09:35:34 2015: Waiting for agents: blockverify(5779)
Wed Aug 5 09:35:37 2015: Waiting for agents: blockverify(5779)
Wed Aug 5 09:35:40 2015: Waiting for agents: blockverify(5779)
All agents sucessfully terminated
T1 tape pmgr: stop Prod
-bash-4.1$ ./pmgr Debug stop
Wed Aug 5 09:36:05 2015: Waiting for agents: mss-stage(4165), mss-stage-cern(4177), watchdog(4847), blockverify(4275), download-cern(4100), exp-pfn-mss(4074)
Wed Aug 5 09:36:08 2015: Waiting for agents: mss-stage(4165)
Wed Aug 5 09:36:11 2015: Waiting for agents: mss-stage(4165)
Wed Aug 5 09:36:14 2015: Waiting for agents: mss-stage(4165)
Wed Aug 5 09:36:17 2015: Waiting for agents: mss-stage(4165)
Wed Aug 5 09:36:20 2015: Waiting for agents: mss-stage(4165)
Wed Aug 5 09:36:23 2015: Waiting for agents: mss-stage(4165)
Wed Aug 5 09:36:26 2015: Waiting for agents: mss-stage(4165)
Wed Aug 5 09:36:29 2015: Waiting for agents: mss-stage(4165)
Wed Aug 5 09:36:32 2015: Waiting for agents: mss-stage(4165)
Wed Aug 5 09:36:35 2015: Waiting for agents: mss-stage(4165)
Wed Aug 5 09:36:38 2015: Waiting for agents: mss-stage(4165)
All agents sucessfully terminated
T1 tape pmgr: stop Debug
-bash-4.1$ ps -fu$USER | grep perl | wc -l
1
-bash-4.1$ ps -fu$USER | grep perl
cmsprod 28460 27539 0 09:38 pts/1 00:00:00 grep perl
-bash-4.1$ reboot now
reboot: Need to be root
-bash-4.1$ exit
logout
[root@cmssrv228 ~]# reboot now

Broadcast message from
(/dev/pts/1) at 9:38 ...

The system is going down for reboot NOW!
[root@cmssrv228 ~]# Connection to cmssrv228 closed by remote host.
Connection to cmssrv228 closed.
[root@cmsadmin1 ~]# ssh cmssrv228
Last login: Wed Aug 5 09:33:44 2015 from cmsadmin1.fnal.gov
NOTICE TO USERS

This  is a Federal computer (and/or it is directly connected to a
Fermilab local network system) that is the property of the United
States Government. It is for authorized use only. Users (autho-
rized or unauthorized) have no explicit or implicit expectation
of privacy.
Any  or  all uses of this system and all files on this system may
be intercepted, monitored, recorded, copied, audited, inspected,
and disclosed to authorized site, Department of Energy and law
enforcement personnel, as well as authorized officials of other
agencies, both domestic and foreign. By using this system, the
user consents to such interception, monitoring, recording, copy-
ing, auditing, inspection, and disclosure at the discretion of
authorized site or Department of Energy personnel.
Unauthorized or improper use of this system may result in  admin-
istrative disciplinary action and civil and criminal penalties.
By continuing to use this system you indicate your awareness of
and consent to these terms and conditions of use. LOG OFF IMME-
DIATELY if you do not agree to the conditions stated in this
warning.
Fermilab  policy  and  rules for computing, including appropriate
use, may be found at http://www.fnal.gov/cd/main/cpolicy.html

cmssrv228.fnal.gov - phedextape/production (SLF 6.6)
16-core Opteron 6320 (H8DGU); 62.90 GB RAM, 20.00 GB swap
[root@cmssrv228 ~]# uptime
09:45:33 up 4 min, 1 user, load average: 0.09, 0.21, 0.09
[root@cmssrv228 ~]#
[root@cmssrv228 ~]# ifconfig
eth0 Link encap:Ethernet HWaddr A0:36:9F:23:D0:70
inet addr:131.225.205.62 Bcast:131.225.207.255 Mask:255.255.252.0
inet6 addr: fe80::a236:9fff:fe23:d070/64 Scope:Link
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:139109 errors:0 dropped:0 overruns:0 frame:0
TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:19729958 (18.8 MiB) TX bytes:0 (0.0 b)

lo Link encap:Local Loopback
inet addr:127.0.0.1 Mask:255.0.0.0
inet6 addr: ::1/128 Scope:Host
UP LOOPBACK RUNNING MTU:65536 Metric:1
RX packets:1529 errors:0 dropped:0 overruns:0 frame:0
TX packets:1529 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:0
RX bytes:1626226 (1.5 MiB) TX bytes:1626226 (1.5 MiB)

[root@cmssrv228 ~]# ethtool -i eth0
driver: ixgbe
version: 3.19.1-k
firmware-version: 0x8000037c
bus-info: 0000:03:00.0
supports-statistics: yes
supports-test: yes
supports-eeprom-access: yes
supports-register-dump: yes
supports-priv-flags: no
[root@cmssrv228 ~]#
[root@cmssrv228 ~]# facter kernelrelease
2.6.32-504.12.2.el6.x86_64
[root@cmssrv228 ~]# df
Filesystem 1K-blocks Used Available Use% Mounted on
/dev/sda2 41284928 6774068 32413708 18% /
tmpfs 32975400 0 32975400 0% /dev/shm
/dev/sda1 1032088 122688 856972 13% /boot
/dev/sda5 898341304 142907900 709793572 17% /storage/local/data1
cmschimera.fnal.gov:/pnfs
1125899906842624 23071111835200 1102828795007424 3% /pnfs
cmschimeradisk.fnal.gov:/dcache
1125899906842624 11045826174976 1114854080667648 1% /dcache

=======================

cmsphedex-disk.fnal.gov - phedexdisk/production (SLF 6.6)
8-core Xeon E5430 @ 2.66GHz (PowerEdge 1950); 31.35 GB RAM, 16.00 GB swap
[root@cmsphedex-disk ~]# su - cmsprod
-bash-4.1$ cd siteconfs/siteconf-fnaldisk.4.1.3-comp3-1.0
-bash-4.1$ ./pmgr Dev stop
Wed Aug 5 10:01:19 2015: Waiting for agents: download-tier1(10808), blockverify(10832)
All agents sucessfully terminated
T1 disk pmgr: stop Dev
-bash-4.1$ ./pmgr Debug stop
Wed Aug 5 10:01:28 2015: Waiting for agents: watchdog(1564), download-remove(1115), blockverify(1155), download-tier1(1131)
All agents sucessfully terminated
T1 disk pmgr: stop Debug
-bash-4.1$ ./pmgr Prod stop
Wed Aug 5 10:01:39 2015: Waiting for agents: download-tier1(1627), download-remove(1591), download-tier2(1635), watchdog(2083)
All agents sucessfully terminated
T1 disk pmgr: stop Prod
-bash-4.1$ exit
logout
[root@cmsphedex-disk ~]# reboot now

Broadcast message from
(/dev/pts/0) at 10:01 ...

The system is going down for reboot NOW!
[root@cmsphedex-disk ~]# Connection to cmsphedex-disk closed by remote host.
Connection to cmsphedex-disk closed.

Looking at the console: failed to mount /dcache disk

ADDRCONF: eth1: link becomes ready
Starting rpcbind: [ OK ]
Starting NFS statd: [ OK ]
Starting kdump:[ OK ]
Starting system message bus: [ OK ]
Starting cups: [ OK ]
NFS filesystems queued to be mounted
Mounting filesystems: mount.nfs: Failed to resolve server cmschimeradisk.fnal.gov: Temporary failure in name resolution
RPC: Registered named UNIX socket transport module.
RPC: Registered udp transport module.
RPC: Registered tcp transport module.
RPC: Registered tcp NFSv4.1 backchannel transport module.
Slow work thread pool: Starting up
Slow work thread pool: Ready
FS-Cache: Loaded
NFS: Registering the id_resolver key type
FS-Cache: Netfs 'nfs' registered for caching
[FAILED]

Fixed with a manual mount:
cmsphedex-disk.fnal.gov - phedexdisk/production (SLF 6.6)
8-core Xeon E5430 @ 2.66GHz (PowerEdge 1950); 31.35 GB RAM, 16.00 GB swap
[root@cmsphedex-disk ~]# df
Filesystem 1K-blocks Used Available Use% Mounted on
/dev/sda3 270691880 72515796 184419004 29% /
tmpfs 16437888 0 16437888 0% /dev/shm
/dev/sda1 999320 91712 855180 10% /boot
cmschimera.fnal.gov:/pnfs
1125899906842624 23071111835232 1102828795007392 3% /pnfs
[root@cmsphedex-disk ~]# mount
/dev/sda3 on / type ext4 (rw)
proc on /proc type proc (rw)
sysfs on /sys type sysfs (rw)
devpts on /dev/pts type devpts (rw,gid=5,mode=620)
tmpfs on /dev/shm type tmpfs (rw)
/dev/sda1 on /boot type ext4 (rw)
none on /proc/sys/fs/binfmt_misc type binfmt_misc (rw)
sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw)
cmschimera.fnal.gov:/pnfs on /pnfs type nfs (rw,noatime,nodiratime,nfsvers=3,intr,hard,noac,rsize=65536,wsize=65536,noacl,addr=131.225.204.184)
[root@cmsphedex-disk ~]# df
Filesystem 1K-blocks Used Available Use% Mounted on
/dev/sda3 270691880 72515796 184419004 29% /
tmpfs 16437888 0 16437888 0% /dev/shm
/dev/sda1 999320 91712 855180 10% /boot
cmschimera.fnal.gov:/pnfs
1125899906842624 23071111835232 1102828795007392 3% /pnfs
[root@cmsphedex-disk ~]# cat /etc/fstab
  1. HEADER: This file was autogenerated at Tue Apr 07 15:20:42 -0500 2015
  2. HEADER: by puppet. While it can still be managed manually, it
  3. HEADER: is definitely not recommended.
#
  1. /etc/fstab
  2. Created by anaconda on Wed Nov 20 02:02:16 2013 #
  3. Accessible filesystems, by reference, are maintained under '/dev/disk'
  4. See man pages fstab(5), findfs(8), mount(8) and/or blkid(8) for more info #
    UUID=40efab39-6270-4950-a83a-d094a21e1074 / ext4 defaults 1 1
    UUID=4460fe9c-6e1b-4eca-afe8-04458c802673 /boot ext4 defaults 1 2
    UUID=df1070a7-09ea-421b-8d9a-b6d138e092d6 swap swap defaults 0 0
    tmpfs /dev/shm tmpfs defaults 0 0
    devpts /dev/pts devpts gid=5,mode=620 0 0
    sysfs /sys sysfs defaults 0 0
    proc /proc proc defaults 0 0
    #cmssrmdisk:/dcache /dcache nfs nfsvers=3,intr,hard,rw,noac,noacl,noatime,nodiratime 1 1
    #cmschimeradisk.fnal.gov:/dcache /dcache nfs minorversion=1,intr,hard,rw,noac,rsize=65536,wsize=65536,noacl,noatime,nodiratime 1 1
    cmschimeradisk.fnal.gov:/dcache /dcache nfs rw,noatime,nodiratime,nfsvers=3,intr,hard,noac,rsize=65536,wsize=65536,noacl 1 1
    #cmschimera.fnal.gov:/pnfs /pnfs nfs minorversion=1,intr,hard,rw,noac,rsize=65536,wsize=65536,noacl,noatime,nodiratime 1 1
    cmschimera.fnal.gov:/pnfs /pnfs nfs rw,noatime,nodiratime,nfsvers=3,intr,hard,noac,rsize=65536,wsize=65536,noacl 1 1
    [root@cmsphedex-disk ~]# mount -a
    [root@cmsphedex-disk ~]# df
    Filesystem 1K-blocks Used Available Use% Mounted on
    /dev/sda3 270691880 72515796 184419004 29% /
    tmpfs 16437888 0 16437888 0% /dev/shm
    /dev/sda1 999320 91712 855180 10% /boot
    cmschimera.fnal.gov:/pnfs
    1125899906842624 23071111835232 1102828795007392 3% /pnfs
    cmschimeradisk.fnal.gov:/dcache
    1125899906842624 11045826174976 1114854080667648 1% /dcache
    [root@cmsphedex-disk ~]# facter kernelrelease
    2.6.32-504.12.2.el6.x86_64

=====================
cmsphedex-lpc.fnal.gov - phedexlpc/production (SLF 6.6)
16-core Xeon E5530 @ 2.40GHz (PowerEdge R410); 23.46 GB RAM, 48.00 GB swap
[root@cmsphedex-lpc ~]# su - cmsprod
-bash-4.1$ cd /home/cmsprod/siteconfs/siteconf-fnallpc.4.1.3-comp3-1.0/
-bash-4.1$ ./pmgr Dev stop
All agents sucessfully terminated
T3 lpc pmgr: stop Dev
-bash-4.1$ ./pmgr Debug stop
Wed Aug 5 10:30:57 2015: Waiting for agents: blockverify(20766), watchdog(20989), download-remove(20747)
All agents sucessfully terminated
T3 lpc pmgr: stop Debug
-bash-4.1$ ./pmgr Prod stop
All agents sucessfully terminated
T3 lpc pmgr: stop Prod
-bash-4.1$ reboot now
reboot: Need to be root
-bash-4.1$ exit
logout
[root@cmsphedex-lpc ~]# reboot now

Broadcast message from
(/dev/pts/1) at 10:31 ...

The system is going down for reboot NOW!

Broadcast message from
(/dev/pts/1) at 10:31 ...

The system is going down for reboot NOW!
[root@cmsphedex-lpc ~]# Connection to cmsphedex-lpc closed by remote host.
Connection to cmsphedex-lpc closed.
[root@cmsadmin1 ~]#

cmsphedex-lpc.fnal.gov - phedexlpc/production (SLF 6.6)
16-core Xeon E5530 @ 2.40GHz (PowerEdge R410); 23.46 GB RAM, 48.00 GB swap
[root@cmsphedex-lpc ~]# df
Filesystem 1K-blocks Used Available Use% Mounted on
/dev/sda3 270691880 5880192 251054608 3% /
tmpfs 12297448 0 12297448 0% /dev/shm
/dev/sda1 999320 87264 859628 10% /boot
eosmain 7496794587136 2930322004548 4566472582588 40% /eos/uscms
[root@cmsphedex-lpc ~]# less /var/log/messages
[root@cmsphedex-lpc ~]# facter kernelrelease
2.6.32-504.12.2.el6.x86_64
[root@cmsphedex-lpc ~]# uptime
10:51:32 up 16 min, 2 users, load average: 0.01, 0.05, 0.07

The console has message about promiscuous mode - sent email to dcso/ecf-cis

[root@cmsphedex-lpc ~]# su - cmsprod
-bash-4.1$ cd /home/cmsprod/siteconfs/siteconf-fnallpc.4.1.3-comp3-1.0/
-bash-4.1$ ./pmgr Dev start
Testing your DB connection
2015-08-05 15:53:32: CheckDBConnection10342: (re)connecting to database
2015-08-05 15:53:34: CheckDBConnection10342: disconnected from database
Connection to TMDB with /home/cmsprod/phedex/info/DBParam:Dev/T3USFNALLPC succeeded
FileExport: removing old stop flag /home/cmsprod/phedex/Dev_T3_US_FNALLPC/state/exp-pfn//stop
FileExport: pid 10351 started in /home/cmsprod/phedex/Dev_T3_US_FNALLPC/state/exp-pfn/
FileRemove: removing old stop flag /home/cmsprod/phedex/Dev_T3_US_FNALLPC/state/download-remove//stop
FileRemove: pid 10358 started in /home/cmsprod/phedex/Dev_T3_US_FNALLPC/state/download-remove/
Unknown option: jobs
FileDownload: removing old stop flag /home/cmsprod/phedex/Dev_T3_US_FNALLPC/state/download//stop
FileDownload: pid 10366 started in /home/cmsprod/phedex/Dev_T3_US_FNALLPC/state/download/
BlockDownloadVerify: removing old stop flag /home/cmsprod/phedex/Dev_T3_US_FNALLPC/state/blockverify//stop
BlockDownloadVerify: pid 10374 started in /home/cmsprod/phedex/Dev_T3_US_FNALLPC/state/blockverify/
Watchdog: removing old stop flag /home/cmsprod/phedex/Dev_T3_US_FNALLPC/state/watchdog//stop
2015-08-05 15:53:38: Watchdog10380: starting Agent session (id=2)
2015-08-05 15:53:38: Watchdog10380: STATISTICS: Reporting every 43200 seconds, detail=0
2015-08-05 15:53:38: Watchdog10380: has successfully initialised
2015-08-05 15:53:38: Watchdog10380: debug: Agent was loaded DB=>1, CYCLE=1, DROPBOX=1
2015-08-05 15:53:38: Watchdog10380: Lookup agent "download-remove"
2015-08-05 15:53:38: Watchdog10380: Lookup agent "download"
2015-08-05 15:53:38: Watchdog10380: Lookup agent "exp-pfn"
2015-08-05 15:53:38: Watchdog10380: Lookup agent "blockverify"
2015-08-05 15:53:38: Watchdog10380: I am running these agents: Watchdog, blockverify, download, download-remove, exp-pfn
2015-08-05 15:53:38: Watchdog10380: (re)connecting to database
2015-08-05 15:53:38: Watchdog10380: Creating new private DBH
Watchdog: pid 10591 started in /home/cmsprod/phedex/Dev_T3_US_FNALLPC/state/watchdog/
T3 lpc pmgr: start Dev
-bash-4.1$ ./pmgr Debug start
Testing your DB connection
2015-08-05 15:53:45: CheckDBConnection10595: (re)connecting to database
2015-08-05 15:53:46: CheckDBConnection10595: disconnected from database
Connection to TMDB with /home/cmsprod/phedex/info/DBParam:Debug/T3USFNALLPC succeeded
FileExport: removing old stop flag /home/cmsprod/phedex/Debug_T3_US_FNALLPC/state/exp-pfn//stop
FileExport: pid 10605 started in /home/cmsprod/phedex/Debug_T3_US_FNALLPC/state/exp-pfn/
FileRemove: removing old stop flag /home/cmsprod/phedex/Debug_T3_US_FNALLPC/state/download-remove//stop
FileRemove: pid 10612 started in /home/cmsprod/phedex/Debug_T3_US_FNALLPC/state/download-remove/
Unknown option: jobs
FileDownload: removing old stop flag /home/cmsprod/phedex/Debug_T3_US_FNALLPC/state/download//stop
FileDownload: pid 10620 started in /home/cmsprod/phedex/Debug_T3_US_FNALLPC/state/download/
BlockDownloadVerify: removing old stop flag /home/cmsprod/phedex/Debug_T3_US_FNALLPC/state/blockverify//stop
BlockDownloadVerify: pid 10628 started in /home/cmsprod/phedex/Debug_T3_US_FNALLPC/state/blockverify/
Watchdog: removing old stop flag /home/cmsprod/phedex/Debug_T3_US_FNALLPC/state/watchdog//stop
2015-08-05 15:53:49: Watchdog10634: starting Agent session (id=2)
2015-08-05 15:53:49: Watchdog10634: STATISTICS: Reporting every 43200 seconds, detail=0
2015-08-05 15:53:49: Watchdog10634: has successfully initialised
2015-08-05 15:53:49: Watchdog10634: Lookup agent "download-remove"
2015-08-05 15:53:49: Watchdog10634: Lookup agent "download"
2015-08-05 15:53:49: Watchdog10634: Lookup agent "exp-pfn"
2015-08-05 15:53:49: Watchdog10634: Lookup agent "blockverify"
2015-08-05 15:53:49: Watchdog10634: I am running these agents: Watchdog, blockverify, download, download-remove, exp-pfn
2015-08-05 15:53:49: Watchdog10634: (re)connecting to database
Watchdog: pid 10846 started in /home/cmsprod/phedex/Debug_T3_US_FNALLPC/state/watchdog/
T3 lpc pmgr: start Debug
-bash-4.1$ ./pmgr Prod start
Testing your DB connection
2015-08-05 15:55:35: CheckDBConnection10965: (re)connecting to database
2015-08-05 15:55:37: CheckDBConnection10965: disconnected from database
Connection to TMDB with /home/cmsprod/phedex/info/DBParam:Prod/T3USFNALLPC succeeded
FileExport: removing old stop flag /home/cmsprod/phedex/Prod_T3_US_FNALLPC/state/exp-pfn//stop
FileExport: pid 10974 started in /home/cmsprod/phedex/Prod_T3_US_FNALLPC/state/exp-pfn/
FileRemove: removing old stop flag /home/cmsprod/phedex/Prod_T3_US_FNALLPC/state/download-remove//stop
FileRemove: pid 10981 started in /home/cmsprod/phedex/Prod_T3_US_FNALLPC/state/download-remove/
Unknown option: jobs
FileDownload: removing old stop flag /home/cmsprod/phedex/Prod_T3_US_FNALLPC/state/download//stop
FileDownload: pid 10989 started in /home/cmsprod/phedex/Prod_T3_US_FNALLPC/state/download/
BlockDownloadVerify: removing old stop flag /home/cmsprod/phedex/Prod_T3_US_FNALLPC/state/blockverify//stop
BlockDownloadVerify: pid 10997 started in /home/cmsprod/phedex/Prod_T3_US_FNALLPC/state/blockverify/
Watchdog: removing old stop flag /home/cmsprod/phedex/Prod_T3_US_FNALLPC/state/watchdog//stop
2015-08-05 15:55:40: Watchdog11003: starting Agent session (id=2)
2015-08-05 15:55:40: Watchdog11003: STATISTICS: Reporting every 43200 seconds, detail=0
2015-08-05 15:55:40: Watchdog11003: has successfully initialised
2015-08-05 15:55:40: Watchdog11003: Lookup agent "download-remove"
2015-08-05 15:55:40: Watchdog11003: Lookup agent "download"
2015-08-05 15:55:40: Watchdog11003: Lookup agent "exp-pfn"
2015-08-05 15:55:40: Watchdog11003: Lookup agent "blockverify"
2015-08-05 15:55:40: Watchdog11003: I am running these agents: Watchdog, blockverify, download, download-remove, exp-pfn
2015-08-05 15:55:40: Watchdog11003: (re)connecting to database
Watchdog: pid 11215 started in /home/cmsprod/phedex/Prod_T3_US_FNALLPC/state/watchdog/
T3 lpc pmgr: start Prod
-bash-4.1$

#3 Updated by Natalia Ratnikova over 4 years ago

Start at 11:00

Open shell sessions on cmsdcacheadmindisk cmssrmdisk cmschimeradisk

Disable puppet on all three nodes:

puppet agent --disable "Down time Aug5th"

puppet service still runs but does nothing.

Stop dcache services: srm, admin, chimera

[root@cmssrmdisk ~]# service dcache-server stop
Stopping transfermanagersDomain 0 1 done
Stopping srm-cmssrmdiskDomain 0 1 2 3 done
Stopping gPlazmaDomain 0 done
Stopping utilityDomain 0 done
Stopping xrootdLFNs-cmssrmdiskDomain 0 1 2 3 4 5 6 7 8 9 [K] 10 done
Stopping xrootd-cmssrmdiskDomain 0 done
Stopping gsidcap-cmssrmdiskDomain 0 1 done
Stopping authdcap-cmssrmdiskDomain 0 1 done
Stopping dcap-cmssrmdiskDomain 0 done
Stopping nfsDomain.v3 0 done
[root@cmssrmdisk ~]#

root@cmsdcacheadmindisk ~]# service dcache-server stop
Stopping dCacheDomain 0 1 2 3 4 5 done
Stopping adminDoorDomain 0 1 done
Stopping httpdDomain 0 1 2 3 4 done
Stopping billingDomain 0 1 done
Stopping dirDomain 0 done
[root@cmsdcacheadmindisk ~]#

[root@cmschimeradisk ~]# service dcache-server stop
Stopping nfsDomain 0 done
Stopping namespaceDomain 0 done
[root@cmschimeradisk ~]#

==============================

[root@cmsdcacheadmindisk ~]# yum clean all;
Loaded plugins: priorities, security
Cleaning repos: epel osg postgresql92 puppet puppet-deps slf slf-collections
: slf-security slf-source uscmst1
Cleaning up Everything
[root@cmsdcacheadmindisk ~]# yum update y
Loaded plugins: priorities, security
Setting up Update Process
epel | 3.8 kB 00:00
epel/primary_db | 6.4 MB 00:00
osg | 2.5 kB 00:00
osg/primary_db | 407 kB 00:00
postgresql92 | 2.5 kB 00:00
postgresql92/primary_db | 129 kB 00:00
puppet | 2.5 kB 00:00
puppet/primary_db | 17 kB 00:00
puppet-deps | 2.5 kB 00:00
puppet-deps/primary_db | 17 kB 00:00
slf | 3.6 kB 00:00
slf/primary_db | 4.2 MB 00:00
slf-collections | 2.9 kB 00:00
slf-collections/primary_db | 1.1 MB 00:00
slf-security | 2.9 kB 00:00
slf-security/primary_db | 4.1 MB 00:00
slf-source | 2.6 kB 00:00
slf-source/primary_db | 2.2 MB 00:00
uscmst1 | 2.5 kB 00:00
uscmst1/primary_db | 141 kB 00:00
582 packages excluded due to repository priority protections
Resolving Dependencies
-
> Running transaction check
---> Package kernel.x86_64 0:2.6.32-504.30.3.el6 will be installed
---> Package kernel-firmware.noarch 0:2.6.32-504.23.4.el6 will be updated
---> Package kernel-firmware.noarch 0:2.6.32-504.30.3.el6 will be an update
---> Package kernel-headers.x86_64 0:2.6.32-504.8.1.slf6 will be updated
---> Package kernel-headers.x86_64 0:2.6.32-504.30.3.el6 will be an update
--> Finished Dependency Resolution
--> Running transaction check
---> Package kernel.x86_64 0:2.6.32-504.8.1.el6 will be erased
--> Finished Dependency Resolution

Dependencies Resolved

================================================================================
Package Arch Version Repository Size ================================================================================
Installing:
kernel x86_64 2.6.32-504.30.3.el6 slf-security 29 M
Updating:
kernel-firmware noarch 2.6.32-504.30.3.el6 slf-security 14 M
kernel-headers x86_64 2.6.32-504.30.3.el6 slf-security 3.4 M
Removing:
kernel x86_64 2.6.32-504.8.1.el6 @slf-security 124 M

Transaction Summary ================================================================================
Install 1 Package(s)
Upgrade 2 Package(s)
Remove 1 Package(s)

Total download size: 47 M
Downloading Packages:
(1/3): kernel-2.6.32-504.30.3.el6.x86_64.rpm | 29 MB 00:00
(2/3): kernel-firmware-2.6.32-504.30.3.el6.noarch.rpm | 14 MB 00:00
(3/3): kernel-headers-2.6.32-504.30.3.el6.x86_64.rpm | 3.4 MB 00:00
--------------------------------------------------------------------------------
Total 52 MB/s | 47 MB 00:00
Running rpm_check_debug
Running Transaction Test
Transaction Test Succeeded
Running Transaction
Updating : kernel-firmware-2.6.32-504.30.3.el6.noarch 1/6
Installing : kernel-2.6.32-504.30.3.el6.x86_64 2/6
Updating : kernel-headers-2.6.32-504.30.3.el6.x86_64 3/6
Cleanup : kernel-2.6.32-504.8.1.el6.x86_64 4/6
Cleanup : kernel-firmware-2.6.32-504.23.4.el6.noarch 5/6
Cleanup : kernel-headers-2.6.32-504.8.1.slf6.x86_64 6/6
Verifying : kernel-headers-2.6.32-504.30.3.el6.x86_64 1/6
Verifying : kernel-2.6.32-504.30.3.el6.x86_64 2/6
Verifying : kernel-firmware-2.6.32-504.30.3.el6.noarch 3/6
Verifying : kernel-headers-2.6.32-504.8.1.slf6.x86_64 4/6
Verifying : kernel-2.6.32-504.8.1.el6.x86_64 5/6
Verifying : kernel-firmware-2.6.32-504.23.4.el6.noarch 6/6

Removed:
kernel.x86_64 0:2.6.32-504.8.1.el6

Installed:
kernel.x86_64 0:2.6.32-504.30.3.el6

Updated:
kernel-firmware.noarch 0:2.6.32-504.30.3.el6 kernel-headers.x86_64 0:2.6.32-504.30.3.el6

Complete!
[root@cmsdcacheadmindisk ~]# facter kernelrelease
2.6.32-504.8.1.slf6.x86_64
[root@cmsdcacheadmindisk ~]# reboot

Broadcast message from
(/dev/pts/0) at 11:19 ...

The system is going down for reboot NOW!
[root@cmsdcacheadmindisk ~]# Connection to cmsdcacheadmindisk closed by remote host.
Connection to cmsdcacheadmindisk closed.
[root@cmsadmin1 ~]#

root@cmsdcacheadmindisk ~]# dcache status
DOMAIN STATUS PID USER
dirDomain running 8152 root
billingDomain running 8226 root
httpdDomain running 8313 root
adminDoorDomain running 8398 root
dCacheDomain running 8487 root
[root@cmsdcacheadmindisk ~]# dcache stop
Stopping dCacheDomain 0 1 done
Stopping adminDoorDomain 0 done
Stopping httpdDomain 0 1 done
Stopping billingDomain 0 done
Stopping dirDomain 0 done
[root@cmsdcacheadmindisk ~]# facter kernelrelease
2.6.32-504.30.3.el6.x86_64
[root@cmsdcacheadmindisk ~]#
cmsdcacheadmindisk.fnal.gov - dcacheadmindisk/production (SLF 6.6)
32-core Opteron 6320 (H8QG6); 62.89 GB RAM, 16.00 GB swap
[root@cmsdcacheadmindisk ~]# dcache status
DOMAIN STATUS PID USER
dirDomain running 8152 root
billingDomain running 8226 root
httpdDomain running 8313 root
adminDoorDomain running 8398 root
dCacheDomain running 8487 root
[root@cmsdcacheadmindisk ~]# dcache stop
Stopping dCacheDomain 0 1 done
Stopping adminDoorDomain 0 done
Stopping httpdDomain 0 1 done
Stopping billingDomain 0 done
Stopping dirDomain 0 done
[root@cmsdcacheadmindisk ~]# facter kernelrelease
2.6.32-504.30.3.el6.x86_64
[root@cmsdcacheadmindisk ~]# service postgresql status
postgresql: unrecognized service
[root@cmsdcacheadmindisk ~]# service postgresql-9.2 status
(pid 7439) is running...
[root@cmsdcacheadmindisk ~]#
[root@cmsdcacheadmindisk ~]# uptime
11:32:11 up 9 min, 2 users, load average: 0.00, 0.14, 0.13

===========================

[root@cmssrmdisk ~]# yum clean all;
Loaded plugins: priorities, security
Cleaning repos: epel osg postgresql92 puppet puppet-deps slf slf-collections
: slf-security slf-source uscmst1
Cleaning up Everything
[root@cmssrmdisk ~]# yum update y
Loaded plugins: priorities, security
Setting up Update Process
epel | 3.8 kB 00:00
epel/primary_db | 6.4 MB 00:00
osg | 2.5 kB 00:00
osg/primary_db | 407 kB 00:00
postgresql92 | 2.5 kB 00:00
postgresql92/primary_db | 129 kB 00:00
puppet | 2.5 kB 00:00
puppet/primary_db | 17 kB 00:00
puppet-deps | 2.5 kB 00:00
puppet-deps/primary_db | 17 kB 00:00
slf | 3.6 kB 00:00
slf/primary_db | 4.2 MB 00:00
slf-collections | 2.9 kB 00:00
slf-collections/primary_db | 1.1 MB 00:00
slf-security | 2.9 kB 00:00
slf-security/primary_db | 4.1 MB 00:00
slf-source | 2.6 kB 00:00
slf-source/primary_db | 2.2 MB 00:00
uscmst1 | 2.5 kB 00:00
uscmst1/primary_db | 141 kB 00:00
582 packages excluded due to repository priority protections
Resolving Dependencies
-
> Running transaction check
---> Package kernel.x86_64 0:2.6.32-504.30.3.el6 will be installed
---> Package kernel-firmware.noarch 0:2.6.32-504.23.4.el6 will be updated
---> Package kernel-firmware.noarch 0:2.6.32-504.30.3.el6 will be an update
---> Package kernel-headers.x86_64 0:2.6.32-504.8.1.slf6 will be updated
---> Package kernel-headers.x86_64 0:2.6.32-504.30.3.el6 will be an update
--> Finished Dependency Resolution
--> Running transaction check
---> Package kernel.x86_64 0:2.6.32-504.8.1.el6 will be erased
facter kernelrelease
--> Finished Dependency Resolution

Dependencies Resolved

================================================================================
Package Arch Version Repository Size ================================================================================
Installing:
kernel x86_64 2.6.32-504.30.3.el6 slf-security 29 M
Updating:
kernel-firmware noarch 2.6.32-504.30.3.el6 slf-security 14 M
kernel-headers x86_64 2.6.32-504.30.3.el6 slf-security 3.4 M
Removing:
kernel x86_64 2.6.32-504.8.1.el6 @slf-security 124 M

Transaction Summary ================================================================================
Install 1 Package(s)
Upgrade 2 Package(s)
Remove 1 Package(s)

Total download size: 47 M
Downloading Packages:
(1/3): kernel-2.6.32-504.30.3.el6.x86_64.rpm | 29 MB 00:00
(2/3): kernel-firmware-2.6.32-504.30.3.el6.noarch.rpm | 14 MB 00:00
(3/3): kernel-headers-2.6.32-504.30.3.el6.x86_64.rpm | 3.4 MB 00:00
--------------------------------------------------------------------------------
Total 50 MB/s | 47 MB 00:00
Running rpm_check_debug
Running Transaction Test
Transaction Test Succeeded
Running Transaction
Updating : kernel-firmware-2.6.32-504.30.3.el6.noarch 1/6
Installing : kernel-2.6.32-504.30.3.el6.x86_64 2/6
Updating : kernel-headers-2.6.32-504.30.3.el6.x86_64 3/6
Cleanup : kernel-2.6.32-504.8.1.el6.x86_64 4/6
Cleanup : kernel-firmware-2.6.32-504.23.4.el6.noarch 5/6
Cleanup : kernel-headers-2.6.32-504.8.1.slf6.x86_64 6/6

Verifying  : kernel-headers-2.6.32-504.30.3.el6.x86_64                    1/6 
Verifying : kernel-2.6.32-504.30.3.el6.x86_64 2/6
Verifying : kernel-firmware-2.6.32-504.30.3.el6.noarch 3/6
Verifying : kernel-headers-2.6.32-504.8.1.slf6.x86_64 4/6
Verifying : kernel-2.6.32-504.8.1.el6.x86_64 5/6
Verifying : kernel-firmware-2.6.32-504.23.4.el6.noarch 6/6

Removed:
kernel.x86_64 0:2.6.32-504.8.1.el6

Installed:
kernel.x86_64 0:2.6.32-504.30.3.el6

Updated:
kernel-firmware.noarch 0:2.6.32-504.30.3.el6
kernel-headers.x86_64 0:2.6.32-504.30.3.el6

Complete!
[root@cmssrmdisk ~]# facter kernelrelease
2.6.32-504.8.1.slf6.x86_64
[root@cmssrmdisk ~]#
[root@cmssrmdisk ~]# reboot

Broadcast message from
(/dev/pts/0) at 11:28 ...

The system is going down for reboot NOW!
[root@cmssrmdisk ~]# Connection to cmssrmdisk closed by remote host.
Connection to cmssrmdisk closed.
[root@cmsadmin1 ~]#

======================================

cmschimeradisk.fnal.gov - chimeradisk/production (SLF 6.6)
64-core Opteron 6376 (H8QG6); 126.01 GB RAM, 16.00 GB swap
[root@cmschimeradisk ~]# puppet agent --disable "Down time Aug5th"
[root@cmschimeradisk ~]# service dcache-server stop
Stopping nfsDomain 0 done
Stopping namespaceDomain 0 done
[root@cmschimeradisk ~]# service postgresq-9.2 status
postgresq-9.2: unrecognized service
[root@cmschimeradisk ~]# service postgresql-9.2 status
(pid 8384) is running...
[root@cmschimeradisk ~]# service postgresql-9.2 stop
Stopping postgresql-9.2 service: [ OK ]
[root@cmschimeradisk ~]# yum clean all;
Loaded plugins: priorities, security
Cleaning repos: epel osg postgresql92 puppet puppet-deps slf slf-collections
: slf-security slf-source uscmst1
Cleaning up Everything
[root@cmschimeradisk ~]# yum update y
Loaded plugins: priorities, security
Setting up Update Process
epel | 3.8 kB 00:00
epel/primary_db | 6.4 MB 00:00
osg | 2.5 kB 00:00
osg/primary_db | 407 kB 00:00
postgresql92 | 2.5 kB 00:00
postgresql92/primary_db | 129 kB 00:00
puppet | 2.5 kB 00:00
puppet/primary_db | 17 kB 00:00
puppet-deps | 2.5 kB 00:00
puppet-deps/primary_db | 17 kB 00:00
slf | 3.6 kB 00:00
slf/primary_db | 4.2 MB 00:00
slf-collections | 2.9 kB 00:00
slf-collections/primary_db | 1.1 MB 00:00
slf-security | 2.9 kB 00:00
slf-security/primary_db | 4.1 MB 00:00
slf-source | 2.6 kB 00:00
slf-source/primary_db | 2.2 MB 00:00
uscmst1 | 2.5 kB 00:00
uscmst1/primary_db | 141 kB 00:00
582 packages excluded due to repository priority protections
Resolving Dependencies
-
> Running transaction check
---> Package kernel.x86_64 0:2.6.32-504.30.3.el6 will be installed
---> Package kernel-firmware.noarch 0:2.6.32-504.23.4.el6 will be updated
---> Package kernel-firmware.noarch 0:2.6.32-504.30.3.el6 will be an update
---> Package kernel-headers.x86_64 0:2.6.32-504.8.1.slf6 will be updated
---> Package kernel-headers.x86_64 0:2.6.32-504.30.3.el6 will be an update
--> Finished Dependency Resolution
--> Running transaction check
---> Package kernel.x86_64 0:2.6.32-504.8.1.el6 will be erased
facter kernelrelease
--> Finished Dependency Resolution

Dependencies Resolved

================================================================================
Package Arch Version Repository Size ================================================================================
Installing:
kernel x86_64 2.6.32-504.30.3.el6 slf-security 29 M
Updating:
kernel-firmware noarch 2.6.32-504.30.3.el6 slf-security 14 M
kernel-headers x86_64 2.6.32-504.30.3.el6 slf-security 3.4 M
Removing:
kernel x86_64 2.6.32-504.8.1.el6 @slf-security 124 M

Transaction Summary ================================================================================
Install 1 Package(s)
Upgrade 2 Package(s)
Remove 1 Package(s)

Total download size: 47 M
Downloading Packages:
(1/3): kernel-2.6.32-504.30.3.el6.x86_64.rpm | 29 MB 00:00
(2/3): kernel-firmware-2.6.32-504.30.3.el6.noarch.rpm | 14 MB 00:00
(3/3): kernel-headers-2.6.32-504.30.3.el6.x86_64.rpm | 3.4 MB 00:00
--------------------------------------------------------------------------------
Total 58 MB/s | 47 MB 00:00
Running rpm_check_debug
Running Transaction Test
Transaction Test Succeeded
Running Transaction
Updating : kernel-firmware-2.6.32-504.30.3.el6.noarch 1/6
Installing : kernel-2.6.32-504.30.3.el6.x [#### ] 2/6r Installing : kernel-2.6.32-504.30.3.el6.x86_64 2/6
Updating : kernel-headers-2.6.32-504.30.3.el6.x86_64 3/6
Cleanup : kernel-2.6.32-504.8.1.el6.x86_64 4/6
Cleanup : kernel-firmware-2.6.32-504.23.4.el6.noarch 5/6
Cleanup : kernel-headers-2.6.32-504.8.1.slf6.x86_64 6/6
Verifying : kernel-headers-2.6.32-504.30.3.el6.x86_64 1/6
Verifying : kernel-2.6.32-504.30.3.el6.x86_64 2/6
Verifying : kernel-firmware-2.6.32-504.30.3.el6.noarch 3/6
Verifying : kernel-headers-2.6.32-504.8.1.slf6.x86_64 4/6
Verifying : kernel-2.6.32-504.8.1.el6.x86_64 5/6
Verifying : kernel-firmware-2.6.32-504.23.4.el6.noarch 6/6

Removed:
kernel.x86_64 0:2.6.32-504.8.1.el6

Installed:
kernel.x86_64 0:2.6.32-504.30.3.el6

Updated:
kernel-firmware.noarch 0:2.6.32-504.30.3.el6
kernel-headers.x86_64 0:2.6.32-504.30.3.el6

Complete!
[root@cmschimeradisk ~]# facter kernelrelease
2.6.32-504.8.1.slf6.x86_64
[root@cmschimeradisk ~]# reboot

Broadcast message from
(/dev/pts/0) at 11:32 ...

The system is going down for reboot NOW!
[root@cmschimeradisk ~]#
[root@cmssrmdisk ~]# uptime
11:33:19 up 1 min, 1 user, load average: 1.28, 0.39, 0.13
[root@cmssrmdisk ~]# dcache stop
Stopping transfermanagersDomain 0 1 done
Stopping srm-cmssrmdiskDomain 0 1 2 3 done
Stopping gPlazmaDomain 0 done
Stopping utilityDomain 0 1 2 3 done
Stopping xrootdLFNs-cmssrmdiskDomain 0 1 done
Stopping xrootd-cmssrmdiskDomain 0 1 done
Stopping gsidcap-cmssrmdiskDomain 0 1 done
Stopping authdcap-cmssrmdiskDomain 0 1 done
Stopping dcap-cmssrmdiskDomain 0 1 done
Stopping nfsDomain.v3 0 done
[root@cmssrmdisk ~]# facter kernelrelease
2.6.32-504.30.3.el6.x86_64

=======================

cmschimeradisk.fnal.gov - chimeradisk/production (SLF 6.6)
64-core Opteron 6376 (H8QG6); 126.01 GB RAM, 16.00 GB swap
[root@cmschimeradisk ~]# puppet agent --disable "Down time Aug5th"
[root@cmschimeradisk ~]# service dcache-server stop
Stopping nfsDomain 0 done
Stopping namespaceDomain 0 done
[root@cmschimeradisk ~]# service postgresq-9.2 status
postgresq-9.2: unrecognized service
[root@cmschimeradisk ~]# service postgresql-9.2 status
(pid 8384) is running...
[root@cmschimeradisk ~]# service postgresql-9.2 stop
Stopping postgresql-9.2 service: [ OK ]
[root@cmschimeradisk ~]# yum clean all;
Loaded plugins: priorities, security
Cleaning repos: epel osg postgresql92 puppet puppet-deps slf slf-collections
: slf-security slf-source uscmst1
Cleaning up Everything
[root@cmschimeradisk ~]# yum update y
Loaded plugins: priorities, security
Setting up Update Process
epel | 3.8 kB 00:00
epel/primary_db | 6.4 MB 00:00
osg | 2.5 kB 00:00
osg/primary_db | 407 kB 00:00
postgresql92 | 2.5 kB 00:00
postgresql92/primary_db | 129 kB 00:00
puppet | 2.5 kB 00:00
puppet/primary_db | 17 kB 00:00
puppet-deps | 2.5 kB 00:00
puppet-deps/primary_db | 17 kB 00:00
slf | 3.6 kB 00:00
slf/primary_db | 4.2 MB 00:00
slf-collections | 2.9 kB 00:00
slf-collections/primary_db | 1.1 MB 00:00
slf-security | 2.9 kB 00:00
slf-security/primary_db | 4.1 MB 00:00
slf-source | 2.6 kB 00:00
slf-source/primary_db | 2.2 MB 00:00
uscmst1 | 2.5 kB 00:00
uscmst1/primary_db | 141 kB 00:00
582 packages excluded due to repository priority protections
Resolving Dependencies
-
> Running transaction check
---> Package kernel.x86_64 0:2.6.32-504.30.3.el6 will be installed
---> Package kernel-firmware.noarch 0:2.6.32-504.23.4.el6 will be updated
---> Package kernel-firmware.noarch 0:2.6.32-504.30.3.el6 will be an update
---> Package kernel-headers.x86_64 0:2.6.32-504.8.1.slf6 will be updated
---> Package kernel-headers.x86_64 0:2.6.32-504.30.3.el6 will be an update
--> Finished Dependency Resolution
--> Running transaction check
---> Package kernel.x86_64 0:2.6.32-504.8.1.el6 will be erased
facter kernelrelease
--> Finished Dependency Resolution

Dependencies Resolved

================================================================================
Package Arch Version Repository Size ================================================================================
Installing:
kernel x86_64 2.6.32-504.30.3.el6 slf-security 29 M
Updating:
kernel-firmware noarch 2.6.32-504.30.3.el6 slf-security 14 M
kernel-headers x86_64 2.6.32-504.30.3.el6 slf-security 3.4 M
Removing:
kernel x86_64 2.6.32-504.8.1.el6 @slf-security 124 M

Transaction Summary ================================================================================
Install 1 Package(s)
Upgrade 2 Package(s)
Remove 1 Package(s)

Total download size: 47 M
Downloading Packages:
(1/3): kernel-2.6.32-504.30.3.el6.x86_64.rpm | 29 MB 00:00
(2/3): kernel-firmware-2.6.32-504.30.3.el6.noarch.rpm | 14 MB 00:00
(3/3): kernel-headers-2.6.32-504.30.3.el6.x86_64.rpm | 3.4 MB 00:00
--------------------------------------------------------------------------------
Total 58 MB/s | 47 MB 00:00
Running rpm_check_debug
Running Transaction Test
Transaction Test Succeeded
Running Transaction
Updating : kernel-firmware-2.6.32-504.30.3.el6.noarch 1/6
Installing : kernel-2.6.32-504.30.3.el6.x [#### ] 2/6r Installing : kernel-2.6.32-504.30.3.el6.x86_64 2/6
Updating : kernel-headers-2.6.32-504.30.3.el6.x86_64 3/6
Cleanup : kernel-2.6.32-504.8.1.el6.x86_64 4/6
Cleanup : kernel-firmware-2.6.32-504.23.4.el6.noarch 5/6
Cleanup : kernel-headers-2.6.32-504.8.1.slf6.x86_64 6/6
Verifying : kernel-headers-2.6.32-504.30.3.el6.x86_64 1/6
Verifying : kernel-2.6.32-504.30.3.el6.x86_64 2/6
Verifying : kernel-firmware-2.6.32-504.30.3.el6.noarch 3/6
Verifying : kernel-headers-2.6.32-504.8.1.slf6.x86_64 4/6
Verifying : kernel-2.6.32-504.8.1.el6.x86_64 5/6
Verifying : kernel-firmware-2.6.32-504.23.4.el6.noarch 6/6

Removed:
kernel.x86_64 0:2.6.32-504.8.1.el6

Installed:
kernel.x86_64 0:2.6.32-504.30.3.el6

Updated:
kernel-firmware.noarch 0:2.6.32-504.30.3.el6
kernel-headers.x86_64 0:2.6.32-504.30.3.el6

Complete!
[root@cmschimeradisk ~]# facter kernelrelease
2.6.32-504.8.1.slf6.x86_64
[root@cmschimeradisk ~]# reboot

Broadcast message from
(/dev/pts/0) at 11:32 ...

The system is going down for reboot NOW!
[root@cmschimeradisk ~]# Connection to cmschimeradisk closed by remote host.
Connection to cmschimeradisk closed.
[root@cmsadmin1 ~]#
cmschimeradisk.fnal.gov - chimeradisk/production (SLF 6.6)
64-core Opteron 6376 (H8QG6); 126.01 GB RAM, 16.00 GB swap
[root@cmschimeradisk ~]# dcache status
DOMAIN STATUS PID USER
namespaceDomain running 8614 root
nfsDomain running 8716 root
[root@cmschimeradisk ~]# service postgresql-9.2 status
(pid 8144) is running...
[root@cmschimeradisk ~]# dcache stop
Stopping nfsDomain 0 done
Stopping namespaceDomain 0 upt1 ime
kdone
e[root@cmschimeradisk ~]# uptime
11:39:58 up 1 min, 1 user, load average: 0.55, 0.30, 0.11
[root@cmschimeradisk ~]# facter kernelrelease
2.6.32-504.30.3.el6.x86_64
[root@cmschimeradisk ~]# ====================================================

#4 Updated by Natalia Ratnikova over 4 years ago

Tape instance dcache is done: upgrade kernel on restart cmssrv228 abnd start agents again

cmssrv228.fnal.gov - phedextape/production (SLF 6.6)
16-core Opteron 6320 (H8DGU); 62.90 GB RAM, 20.00 GB swap
[root@cmssrv228 ~]# facter kernelrelease
2.6.32-504.12.2.el6.x86_64
[root@cmssrv228 ~]# yum clean all
Loaded plugins: priorities, security
Cleaning repos: eos epel osg puppet puppet-deps slf slf-collections
: slf-security slf-source slf6x slf6x-security uscmst1
Cleaning up Everything
[root@cmssrv228 ~]# yum update y
Loaded plugins: priorities, security
Setting up Update Process
eos | 2.5 kB 00:00
eos/primary_db | 332 kB 00:00
epel | 3.8 kB 00:00
epel/primary_db | 6.4 MB 00:00
osg | 2.5 kB 00:00
osg/primary_db | 407 kB 00:00
puppet | 2.5 kB 00:00
puppet/primary_db | 17 kB 00:00
puppet-deps | 2.5 kB 00:00
puppet-deps/primary_db | 17 kB 00:00
slf | 3.6 kB 00:00
slf/primary_db | 4.2 MB 00:00
slf-collections | 2.9 kB 00:00
slf-collections/primary_db | 1.1 MB 00:00
slf-security | 2.9 kB 00:00
slf-security/primary_db | 4.1 MB 00:00
slf-source | 2.6 kB 00:00
slf-source/primary_db | 2.2 MB 00:00
slf6x | 3.6 kB 00:00
slf6x/primary_db | 4.2 MB 00:00
slf6x-security | 2.9 kB 00:00
slf6x-security/primary_db | 4.1 MB 00:00
uscmst1 | 2.5 kB 00:00
uscmst1/primary_db | 141 kB 00:00
7924 packages excluded due to repository priority protections
Resolving Dependencies
-
> Running transaction check
---> Package eos-client.x86_64 0:0.3.111-aquamarine.slc6 will be updated
---> Package eos-client.x86_64 0:0.3.125-aquamarine.slc6 will be an update
---> Package kernel.x86_64 0:2.6.32-504.30.3.el6 will be installed
---> Package kernel-firmware.noarch 0:2.6.32-504.12.2.el6 will be updated
---> Package kernel-firmware.noarch 0:2.6.32-504.30.3.el6 will be an update
---> Package kernel-headers.x86_64 0:2.6.32-504.12.2.el6 will be updated
---> Package kernel-headers.x86_64 0:2.6.32-504.30.3.el6 will be an update
--> Finished Dependency Resolution
--> Running transaction check
---> Package kernel.x86_64 0:2.6.32-504.8.1.el6 will be erased
reboot--> Finished Dependency Resolution

Dependencies Resolved

=========================================================================================================================================================================================================================================================================================
Package Arch Version Repository Size =========================================================================================================================================================================================================================================================================================
Installing:
kernel x86_64 2.6.32-504.30.3.el6 slf-security 29 M
Updating:
eos-client x86_64 0.3.125-aquamarine.slc6 eos 622 k
kernel-firmware noarch 2.6.32-504.30.3.el6 slf-security 14 M
kernel-headers x86_64 2.6.32-504.30.3.el6 slf-security 3.4 M
Removing:
kernel x86_64 2.6.32-504.8.1.el6 @slf-security 124 M

Transaction Summary =========================================================================================================================================================================================================================================================================================
Install 1 Package(s)
Upgrade 3 Package(s)
Remove 1 Package(s)

Total download size: 48 M
Downloading Packages:
(1/4): eos-client-0.3.125-aquamarine.slc6.x86_64.rpm | 622 kB 00:00
(2/4): kernel-2.6.32-504.30.3.el6.x86_64.rpm | 29 MB 00:00
(3/4): kernel-firmware-2.6.32-504.30.3.el6.noarch.rpm | 14 MB 00:00
(4/4): kernel-headers-2.6.32-504.30.3.el6.x86_64.rpm | 3.4 MB 00:00
-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
Total 57 MB/s | 48 MB 00:00
Running rpm_check_debug
Running Transaction Test
Transaction Test Succeeded
Running Transaction
Updating : kernel-firmware-2.6.32-504.30.3.el6.noarch 1/8
Installing : kernel-2.6.32-504.30.3.el6.x86_64 2/8
Updating : eos-client-0.3.125-aquamarine.slc6.x86_64 3/8
Updating : kernel-headers-2.6.32-504.30.3.el6.x86_64 4/8
Cleanup : kernel-2.6.32-504.8.1.el6.x86_64 5/8
Cleanup : kernel-firmware-2.6.32-504.12.2.el6.noarch 6/8
Cleanup : kernel-headers-2.6.32-504.12.2.el6.x86_64 7/8
Cleanup : eos-client-0.3.111-aquamarine.slc6.x86_64 8/8
Verifying : kernel-headers-2.6.32-504.30.3.el6.x86_64 1/8
Verifying : eos-client-0.3.125-aquamarine.slc6.x86_64 2/8
Verifying : kernel-2.6.32-504.30.3.el6.x86_64 3/8
Verifying : kernel-firmware-2.6.32-504.30.3.el6.noarch 4/8
Verifying : kernel-headers-2.6.32-504.12.2.el6.x86_64 5/8
Verifying : eos-client-0.3.111-aquamarine.slc6.x86_64 6/8
Verifying : kernel-2.6.32-504.8.1.el6.x86_64 7/8
Verifying : kernel-firmware-2.6.32-504.12.2.el6.noarch 8/8

Removed:
kernel.x86_64 0:2.6.32-504.8.1.el6

Installed:
kernel.x86_64 0:2.6.32-504.30.3.el6

Updated:
eos-client.x86_64 0:0.3.125-aquamarine.slc6 kernel-firmware.noarch 0:2.6.32-504.30.3.el6 kernel-headers.x86_64 0:2.6.32-504.30.3.el6

Complete!
[root@cmssrv228 ~]# reboot

Broadcast message from
(/dev/pts/0) at 11:44 ...

The system is going down for reboot NOW!
[root@cmssrv228 ~]# Connection to cmssrv228 closed by remote host.
Connection to cmssrv228 closed.
[root@cmsadmin1 ~]#

Node did not actually reboot, so power cycle was done:

[root@cmsconsole ~]# cmspower-powerit -a status cmssrv228 === cmssrv228 ===
connecting to APC APCCMS1574-1, outlet 15
Outlet state: ON
[root@cmsconsole ~]# cmspower-powerit -a cycle -c "Reboot for kernel upgrade " cmssrv228 === cmssrv228 ===
connecting to APC APCCMS1574-1, outlet 15
Outlet state: OFF
connecting to APC APCCMS1574-1, outlet 15
Outlet state: ON

Meanwhile Chih-Hao said he needs to do more work on the Tape instance (test that postgresql-9.2 comes up properly on reboot) , so I defer agents start until then.

Also nfs mount for disk instance will fail, because chimera services are down.

The console shows a failure:
NFS filesystems queued to be mounted
Mounting filesystems: RPC: Registered named UNIX socket transport module.
RPC: Registered udp transport module.
RPC: Registered tcp transport module.
RPC: Registered tcp NFSv4.1 backchannel transport module.
Slow work thread pool: Starting up
Slow work thread pool: Ready
FS-Cache: Loaded
NFS: Registering the id_resolver key type
FS-Cache: Netfs 'nfs' registered for caching
nfs4filelayout_init: NFSv4 File Layout Driver Registering...
mount.nfs: Connection timed out
[FAILED]

The node is up now with tape instance mounted properly and kernel upgraded:
cmssrv228.fnal.gov - phedextape/production (SLF 6.6)
16-core Opteron 6320 (H8DGU); 62.90 GB RAM, 20.00 GB swap
[root@cmssrv228 ~]# df
Filesystem 1K-blocks Used Available Use% Mounted on
/dev/sda2 41284928 6663168 32524608 18% /
tmpfs 32975396 0 32975396 0% /dev/shm
/dev/sda1 1032088 128400 851260 14% /boot
/dev/sda5 898341304 142907900 709793572 17% /storage/local/data1
cmschimera.fnal.gov:/pnfs
1125899906842624 23070969961600 1102828936881024 3% /pnfs
[root@cmssrv228 ~]# facter kernelrelease
2.6.32-504.30.3.el6.x86_64

#5 Updated by Natalia Ratnikova over 4 years ago

cmsphedex-lpc.fnal.gov - phedexlpc/production (SLF 6.6)
16-core Xeon E5530 @ 2.40GHz (PowerEdge R410); 23.46 GB RAM, 48.00 GB swap
[root@cmsphedex-lpc ~]# history | grep kernel
925 facter kernelrelease
998 history | grep kernel
[root@cmsphedex-lpc ~]# facter kernelrelease
2.6.32-504.12.2.el6.x86_64
[root@cmsphedex-lpc ~]# yum clean all; yum update y; reboot
Loaded plugins: priorities, security
Cleaning repos: eos epel osg puppet puppet-deps slf slf-collections slf-security slf-source slf6x slf6x-security uscmst1
Cleaning up Everything
Loaded plugins: priorities, security
Setting up Update Process
eos | 2.5 kB 00:00
eos/primary_db | 332 kB 00:00
epel | 3.8 kB 00:00
epel/primary_db | 6.4 MB 00:00
osg | 2.5 kB 00:00
osg/primary_db | 407 kB 00:00
puppet | 2.5 kB 00:00
puppet/primary_db | 17 kB 00:00
puppet-deps | 2.5 kB 00:00
puppet-deps/primary_db | 17 kB 00:00
slf | 3.6 kB 00:00
slf/primary_db | 4.2 MB 00:00
slf-collections | 2.9 kB 00:00
slf-collections/primary_db | 1.1 MB 00:00
slf-security | 2.9 kB 00:00
slf-security/primary_db | 4.1 MB 00:00
slf-source | 2.6 kB 00:00
slf-source/primary_db | 2.2 MB 00:00
slf6x | 3.6 kB 00:00
slf6x/primary_db | 4.2 MB 00:00
slf6x-security | 2.9 kB 00:00
slf6x-security/primary_db | 4.1 MB 00:00
uscmst1 | 2.5 kB 00:00
uscmst1/primary_db | 141 kB 00:00
7924 packages excluded due to repository priority protections
Resolving Dependencies
-
> Running transaction check
---> Package eos-client.x86_64 0:0.3.111-aquamarine.slc6 will be updated
---> Package eos-client.x86_64 0:0.3.125-aquamarine.slc6 will be an update
---> Package eos-fuse.x86_64 0:0.3.111-aquamarine.slc6 will be updated
---> Package eos-fuse.x86_64 0:0.3.125-aquamarine.slc6 will be an update
---> Package kernel.x86_64 0:2.6.32-504.30.3.el6 will be installed
---> Package kernel-firmware.noarch 0:2.6.32-504.12.2.el6 will be updated
---> Package kernel-firmware.noarch 0:2.6.32-504.30.3.el6 will be an update
---> Package kernel-headers.x86_64 0:2.6.32-504.12.2.el6 will be updated
---> Package kernel-headers.x86_64 0:2.6.32-504.30.3.el6 will be an update
--> Finished Dependency Resolution
--> Running transaction check
---> Package kernel.x86_64 0:2.6.32-431.23.3.el6 will be erased
--> Finished Dependency Resolution

Dependencies Resolved

=========================================================================================================================================================================================================================================================================================
Package Arch Version Repository Size =========================================================================================================================================================================================================================================================================================
Installing:
kernel x86_64 2.6.32-504.30.3.el6 slf-security 29 M
Updating:
eos-client x86_64 0.3.125-aquamarine.slc6 eos 622 k
eos-fuse x86_64 0.3.125-aquamarine.slc6 eos 432 k
kernel-firmware noarch 2.6.32-504.30.3.el6 slf-security 14 M
kernel-headers x86_64 2.6.32-504.30.3.el6 slf-security 3.4 M
Removing:
kernel x86_64 2.6.32-431.23.3.el6 @slf-security/6.5 121 M

Transaction Summary =========================================================================================================================================================================================================================================================================================
Install 1 Package(s)
Upgrade 4 Package(s)
Remove 1 Package(s)

Total download size: 48 M
Downloading Packages:
(1/5): eos-client-0.3.125-aquamarine.slc6.x86_64.rpm | 622 kB 00:00
(2/5): eos-fuse-0.3.125-aquamarine.slc6.x86_64.rpm | 432 kB 00:00
(3/5): kernel-2.6.32-504.30.3.el6.x86_64.rpm | 29 MB 00:00
(4/5): kernel-firmware-2.6.32-504.30.3.el6.noarch.rpm | 14 MB 00:00
(5/5): kernel-headers-2.6.32-504.30.3.el6.x86_64.rpm | 3.4 MB 00:00
-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
Total 29 MB/s | 48 MB 00:01
Running rpm_check_debug
Running Transaction Test
Transaction Test Succeeded
Running Transaction
Updating : kernel-firmware-2.6.32-504.30.3.el6.noarch 1/10
Updating : eos-client-0.3.125-aquamarine.slc6.x86_64 2/10
Updating : eos-fuse-0.3.125-aquamarine.slc6.x86_64 3/10
Starting conditional EOS services
Installing : kernel-2.6.32-504.30.3.el6.x86_64 4/10
Updating : kernel-headers-2.6.32-504.30.3.el6.x86_64 5/10
Cleanup : kernel-2.6.32-431.23.3.el6.x86_64 6/10
Cleanup : kernel-firmware-2.6.32-504.12.2.el6.noarch 7/10
Cleanup : kernel-headers-2.6.32-504.12.2.el6.x86_64 8/10
Cleanup : eos-fuse-0.3.111-aquamarine.slc6.x86_64 9/10
Cleanup : eos-client-0.3.111-aquamarine.slc6.x86_64 10/10
Verifying : kernel-headers-2.6.32-504.30.3.el6.x86_64 1/10
Verifying : kernel-2.6.32-504.30.3.el6.x86_64 2/10
Verifying : eos-client-0.3.125-aquamarine.slc6.x86_64 3/10
Verifying : eos-fuse-0.3.125-aquamarine.slc6.x86_64 4/10
Verifying : kernel-firmware-2.6.32-504.30.3.el6.noarch 5/10
Verifying : eos-fuse-0.3.111-aquamarine.slc6.x86_64 6/10
Verifying : eos-client-0.3.111-aquamarine.slc6.x86_64 7/10
Verifying : kernel-2.6.32-431.23.3.el6.x86_64 8/10
Verifying : kernel-firmware-2.6.32-504.12.2.el6.noarch 9/10
Verifying : kernel-headers-2.6.32-504.12.2.el6.x86_64 10/10

Removed:
kernel.x86_64 0:2.6.32-431.23.3.el6

Installed:
kernel.x86_64 0:2.6.32-504.30.3.el6

Updated:
eos-client.x86_64 0:0.3.125-aquamarine.slc6 eos-fuse.x86_64 0:0.3.125-aquamarine.slc6 kernel-firmware.noarch 0:2.6.32-504.30.3.el6 kernel-headers.x86_64 0:2.6.32-504.30.3.el6

Complete!

Broadcast message from
(/dev/pts/0) at 12:02 ...

The system is going down for reboot NOW!
[root@cmsphedex-lpc ~]# Connection to cmsphedex-lpc closed by remote host.
Connection to cmsphedex-lpc closed.

cmsphedex-lpc.fnal.gov - phedexlpc/production (SLF 6.6)
16-core Xeon E5530 @ 2.40GHz (PowerEdge R410); 23.46 GB RAM, 48.00 GB swap
[root@cmsphedex-lpc ~]# facter kernelrelease
2.6.32-504.30.3.el6.x86_64
[root@cmsphedex-lpc ~]# uptime
12:11:57 up 6 min, 1 user, load average: 0.09, 0.29, 0.16
[root@cmsphedex-lpc ~]# su - cmsprod
-bash-4.1$ cd /home/cmsprod/siteconfs/siteconf-fnallpc.4.1.3-comp3-1.0/
-bash-4.1$ ./pmgr Dev start
Testing your DB connection
2015-08-05 17:12:23: CheckDBConnection11341: (re)connecting to database

2015-08-05 17:12:25: CheckDBConnection11341: disconnected from database
Connection to TMDB with /home/cmsprod/phedex/info/DBParam:Dev/T3USFNALLPC succeeded
FileExport: pid 10351 dead in /home/cmsprod/phedex/Dev_T3_US_FNALLPC/state/exp-pfn/, overwriting
FileExport: pid 11350 started in /home/cmsprod/phedex/Dev_T3_US_FNALLPC/state/exp-pfn/
FileRemove: pid 10358 dead in /home/cmsprod/phedex/Dev_T3_US_FNALLPC/state/download-remove/, overwriting
FileRemove: pid 11357 started in /home/cmsprod/phedex/Dev_T3_US_FNALLPC/state/download-remove/
Unknown option: jobs
FileDownload: pid 10366 dead in /home/cmsprod/phedex/Dev_T3_US_FNALLPC/state/download/, overwriting
FileDownload: pid 11365 started in /home/cmsprod/phedex/Dev_T3_US_FNALLPC/state/download/
BlockDownloadVerify: pid 10374 dead in /home/cmsprod/phedex/Dev_T3_US_FNALLPC/state/blockverify/, overwriting
BlockDownloadVerify: pid 11374 started in /home/cmsprod/phedex/Dev_T3_US_FNALLPC/state/blockverify/
Watchdog: pid 10591 dead in /home/cmsprod/phedex/Dev_T3_US_FNALLPC/state/watchdog/, overwriting
2015-08-05 17:12:29: Watchdog11380: starting Agent session (id=2)
2015-08-05 17:12:29: Watchdog11380: STATISTICS: Reporting every 43200 seconds, detail=0
2015-08-05 17:12:29: Watchdog11380: has successfully initialised
2015-08-05 17:12:29: Watchdog11380: debug: Agent was loaded DB=>1, CYCLE=1, DROPBOX=1
2015-08-05 17:12:29: Watchdog11380: Lookup agent "download-remove"
2015-08-05 17:12:29: Watchdog11380: Lookup agent "download"
2015-08-05 17:12:29: Watchdog11380: Lookup agent "exp-pfn"
2015-08-05 17:12:29: Watchdog11380: Lookup agent "blockverify"
2015-08-05 17:12:29: Watchdog11380: I am running these agents: Watchdog, blockverify, download, download-remove, exp-pfn
2015-08-05 17:12:29: Watchdog11380: (re)connecting to database
2015-08-05 17:12:29: Watchdog11380: Creating new private DBH
Watchdog: pid 11591 started in /home/cmsprod/phedex/Dev_T3_US_FNALLPC/state/watchdog/
T3 lpc pmgr: start Dev
-bash-4.1$ ./pmgr Debug start
Testing your DB connection
2015-08-05 17:12:35: CheckDBConnection11595: (re)connecting to database
2015-08-05 17:12:37: CheckDBConnection11595: disconnected from database
Connection to TMDB with /home/cmsprod/phedex/info/DBParam:Debug/T3USFNALLPC succeeded
FileExport: pid 10605 dead in /home/cmsprod/phedex/Debug_T3_US_FNALLPC/state/exp-pfn/, overwriting
FileExport: pid 11604 started in /home/cmsprod/phedex/Debug_T3_US_FNALLPC/state/exp-pfn/
FileRemove: pid 10612 dead in /home/cmsprod/phedex/Debug_T3_US_FNALLPC/state/download-remove/, overwriting
FileRemove: pid 11612 started in /home/cmsprod/phedex/Debug_T3_US_FNALLPC/state/download-remove/
Unknown option: jobs
FileDownload: pid 10620 dead in /home/cmsprod/phedex/Debug_T3_US_FNALLPC/state/download/, overwriting
FileDownload: pid 11620 started in /home/cmsprod/phedex/Debug_T3_US_FNALLPC/state/download/
BlockDownloadVerify: pid 10628 dead in /home/cmsprod/phedex/Debug_T3_US_FNALLPC/state/blockverify/, overwriting
BlockDownloadVerify: pid 11628 started in /home/cmsprod/phedex/Debug_T3_US_FNALLPC/state/blockverify/
Watchdog: pid 10846 dead in /home/cmsprod/phedex/Debug_T3_US_FNALLPC/state/watchdog/, overwriting
2015-08-05 17:12:40: Watchdog11634: starting Agent session (id=2)
2015-08-05 17:12:40: Watchdog11634: STATISTICS: Reporting every 43200 seconds, detail=0
2015-08-05 17:12:40: Watchdog11634: has successfully initialised
2015-08-05 17:12:40: Watchdog11634: Lookup agent "download-remove"
2015-08-05 17:12:40: Watchdog11634: Lookup agent "download"
2015-08-05 17:12:40: Watchdog11634: Lookup agent "exp-pfn"
2015-08-05 17:12:40: Watchdog11634: Lookup agent "blockverify"
2015-08-05 17:12:40: Watchdog11634: I am running these agents: Watchdog, blockverify, download, download-remove, exp-pfn
2015-08-05 17:12:40: Watchdog11634: (re)connecting to database
Watchdog: pid 11849 started in /home/cmsprod/phedex/Debug_T3_US_FNALLPC/state/watchdog/
T3 lpc pmgr: start Debug
-bash-4.1$ ./pmgr Prod start
Testing your DB connection
2015-08-05 17:12:47: CheckDBConnection11853: (re)connecting to database
2015-08-05 17:12:48: CheckDBConnection11853: disconnected from database
Connection to TMDB with /home/cmsprod/phedex/info/DBParam:Prod/T3USFNALLPC succeeded
FileExport: pid 10974 dead in /home/cmsprod/phedex/Prod_T3_US_FNALLPC/state/exp-pfn/, overwriting
FileExport: pid 11863 started in /home/cmsprod/phedex/Prod_T3_US_FNALLPC/state/exp-pfn/
FileRemove: pid 10981 dead in /home/cmsprod/phedex/Prod_T3_US_FNALLPC/state/download-remove/, overwriting
FileRemove: pid 11870 started in /home/cmsprod/phedex/Prod_T3_US_FNALLPC/state/download-remove/
Unknown option: jobs
FileDownload: pid 10989 dead in /home/cmsprod/phedex/Prod_T3_US_FNALLPC/state/download/, overwriting
FileDownload: pid 11878 started in /home/cmsprod/phedex/Prod_T3_US_FNALLPC/state/download/
BlockDownloadVerify: pid 10997 dead in /home/cmsprod/phedex/Prod_T3_US_FNALLPC/state/blockverify/, overwriting
BlockDownloadVerify: pid 11886 started in /home/cmsprod/phedex/Prod_T3_US_FNALLPC/state/blockverify/
Watchdog: pid 11215 dead in /home/cmsprod/phedex/Prod_T3_US_FNALLPC/state/watchdog/, overwriting
2015-08-05 17:12:51: Watchdog11892: starting Agent session (id=2)
2015-08-05 17:12:51: Watchdog11892: STATISTICS: Reporting every 43200 seconds, detail=0
2015-08-05 17:12:51: Watchdog11892: has successfully initialised
2015-08-05 17:12:51: Watchdog11892: Lookup agent "download-remove"
2015-08-05 17:12:51: Watchdog11892: Lookup agent "download"
2015-08-05 17:12:51: Watchdog11892: Lookup agent "exp-pfn"
2015-08-05 17:12:51: Watchdog11892: Lookup agent "blockverify"
2015-08-05 17:12:51: Watchdog11892: I am running these agents: Watchdog, blockverify, download, download-remove, exp-pfn
2015-08-05 17:12:51: Watchdog11892: (re)connecting to database
Watchdog: pid 12103 started in /home/cmsprod/phedex/Prod_T3_US_FNALLPC/state/watchdog/
T3 lpc pmgr: start Prod
-bash-4.1$

===================================
msphedex-lpc.fnal.gov - phedexlpc/production (SLF 6.6)
16-core Xeon E5530 @ 2.40GHz (PowerEdge R410); 23.46 GB RAM, 48.00 GB swap
[root@cmsphedex-lpc ~]# service puppet status
puppet (pid 7231) is running...
[root@cmsphedex-lpc ~]# puppet agent --noop
[root@cmsphedex-lpc ~]# puppet agent -t --noop
Info: Retrieving pluginfacts
Info: Retrieving plugin
Info: Loading facts in /var/lib/puppet/lib/facter/iptables_persistent_version.rb
Info: Loading facts in /var/lib/puppet/lib/facter/puppet_vardir.rb
Info: Loading facts in /var/lib/puppet/lib/facter/facter_dot_d.rb
Info: Loading facts in /var/lib/puppet/lib/facter/ip6tables_version.rb
Info: Loading facts in /var/lib/puppet/lib/facter/iptables_version.rb
Info: Loading facts in /var/lib/puppet/lib/facter/cvmfspartsize.rb
Info: Loading facts in /var/lib/puppet/lib/facter/postgres_default_version.rb
Info: Loading facts in /var/lib/puppet/lib/facter/concat_basedir.rb
Info: Loading facts in /var/lib/puppet/lib/facter/cvmfsversion.rb
Info: Loading facts in /var/lib/puppet/lib/facter/pe_version.rb
Info: Loading facts in /var/lib/puppet/lib/facter/code_server.rb
Info: Loading facts in /var/lib/puppet/lib/facter/root_home.rb
Info: Loading facts in /var/lib/puppet/lib/facter/certificate_facts.rb
Info: Loading facts in /var/lib/puppet/lib/facter/afs_cache_size.rb
Info: Loading facts in /var/lib/puppet/lib/facter/os_maj_version.rb
Info: Loading facts in /var/lib/puppet/lib/facter/rsyslog_version.rb
Info: Caching catalog for cmsphedex-lpc.fnal.gov
Info: Applying configuration version '1438794790'
Notice: Finished catalog run in 17.98 seconds
[root@cmsphedex-lpc ~]# puppet agent --enable
[root@cmsphedex-lpc ~]#

#6 Updated by Natalia Ratnikova over 4 years ago

start agents on Tape instance phedex server

cmssrv228.fnal.gov - phedextape/production (SLF 6.6)
16-core Opteron 6320 (H8DGU); 62.90 GB RAM, 20.00 GB swap
[root@cmssrv228 ~]# uptime
12:16:23 up 22 min, 1 user, load average: 0.12, 0.14, 0.27
[root@cmssrv228 ~]# facter kernelrelease
2.6.32-504.30.3.el6.x86_64
[root@cmssrv228 ~]# su - cmsprod
-bash-4.1$ cd /home/cmsprod/siteconf-git/
-bash-4.1$ ./pmgr Dev start
Testing your DB connection
2015-08-05 17:16:33: CheckDBConnection17990: (re)connecting to database

2015-08-05 17:16:35: CheckDBConnection17990: disconnected from database
Connection to TMDB with /home/cmsprod/phedex/info/DBParam:Dev/T1USFNAL succeeded
FileExport: removing old stop flag /home/cmsprod/phedex/Dev_T1_US_FNAL_Buffer/state/exp-pfn-buffer//stop
FileExport: pid 18007 started in /home/cmsprod/phedex/Dev_T1_US_FNAL_Buffer/state/exp-pfn-buffer/
FileExport: removing old stop flag /home/cmsprod/phedex/Dev_T1_US_FNAL_Buffer/state/exp-pfn-mss//stop
FileExport: pid 18018 started in /home/cmsprod/phedex/Dev_T1_US_FNAL_Buffer/state/exp-pfn-mss/
FileRemove: removing old stop flag /home/cmsprod/phedex/Dev_T1_US_FNAL_Buffer/state/download-remove//stop
FileRemove: pid 18029 started in /home/cmsprod/phedex/Dev_T1_US_FNAL_Buffer/state/download-remove/
Unknown option: jobs
FileDownload: removing old stop flag /home/cmsprod/phedex/Dev_T1_US_FNAL_Buffer/state/download-cern//stop
FileDownload: pid 18041 started in /home/cmsprod/phedex/Dev_T1_US_FNAL_Buffer/state/download-cern/
Unknown option: jobs
FileDownload: removing old stop flag /home/cmsprod/phedex/Dev_T1_US_FNAL_Buffer/state/download-tier1//stop
FileDownload: pid 18053 started in /home/cmsprod/phedex/Dev_T1_US_FNAL_Buffer/state/download-tier1/
Unknown option: jobs
FileDownload: removing old stop flag /home/cmsprod/phedex/Dev_T1_US_FNAL_Buffer/state/download-tier2//stop
FileDownload: pid 18066 started in /home/cmsprod/phedex/Dev_T1_US_FNAL_Buffer/state/download-tier2/
Unknown option: jobs
FileDownload: removing old stop flag /home/cmsprod/phedex/Dev_T1_US_FNAL_Buffer/state/download-tier3//stop
FileDownload: pid 18078 started in /home/cmsprod/phedex/Dev_T1_US_FNAL_Buffer/state/download-tier3/
FileStager: removing old stop flag /home/cmsprod/phedex/Dev_T1_US_FNAL_Buffer/state/mss-stage-local//stop
FileStager: pid 18090 started in /home/cmsprod/phedex/Dev_T1_US_FNAL_Buffer/state/mss-stage-local/
FileStager: removing old stop flag /home/cmsprod/phedex/Dev_T1_US_FNAL_Buffer/state/mss-stage//stop
FileStager: pid 18101 started in /home/cmsprod/phedex/Dev_T1_US_FNAL_Buffer/state/mss-stage/
FileStager: removing old stop flag /home/cmsprod/phedex/Dev_T1_US_FNAL_Buffer/state/mss-stage-cern//stop
FileStager: pid 18112 started in /home/cmsprod/phedex/Dev_T1_US_FNAL_Buffer/state/mss-stage-cern/
FileMSSMigrate: removing old stop flag /home/cmsprod/phedex/Dev_T1_US_FNAL_Buffer/state/mss-migrate//stop
FileMSSMigrate: pid 18123 started in /home/cmsprod/phedex/Dev_T1_US_FNAL_Buffer/state/mss-migrate/
BlockDownloadVerify: removing old stop flag /home/cmsprod/phedex/Dev_T1_US_FNAL_Buffer/state/blockverify//stop
BlockDownloadVerify: pid 18181 started in /home/cmsprod/phedex/Dev_T1_US_FNAL_Buffer/state/blockverify/
Watchdog: removing old stop flag /home/cmsprod/phedex/Dev_T1_US_FNAL_Buffer/state/watchdog//stop
2015-08-05 17:16:41: Watchdog18215: starting Agent session (id=2)
2015-08-05 17:16:41: Watchdog18215: STATISTICS: Reporting every 43200 seconds, detail=0
2015-08-05 17:16:41: Watchdog18215: has successfully initialised
2015-08-05 17:16:41: Watchdog18215: debug: Agent was loaded DB=>1, CYCLE=1, DROPBOX=1
2015-08-05 17:16:41: Watchdog18215: Lookup agent "download-cern"
2015-08-05 17:16:41: Watchdog18215: Lookup agent "download-tier1"
2015-08-05 17:16:41: Watchdog18215: Lookup agent "download-tier2"
2015-08-05 17:16:41: Watchdog18215: Lookup agent "download-tier3"
2015-08-05 17:16:41: Watchdog18215: Lookup agent "download-remove"
2015-08-05 17:16:41: Watchdog18215: Lookup agent "exp-pfn-buffer"
2015-08-05 17:16:41: Watchdog18215: Lookup agent "exp-pfn-mss"
2015-08-05 17:16:41: Watchdog18215: Lookup agent "mss-migrate"
2015-08-05 17:16:41: Watchdog18215: Lookup agent "mss-stage-local"
2015-08-05 17:16:41: Watchdog18215: Lookup agent "mss-stage"
2015-08-05 17:16:41: Watchdog18215: Lookup agent "mss-stage-cern"
2015-08-05 17:16:41: Watchdog18215: Lookup agent "blockverify"
2015-08-05 17:16:41: Watchdog18215: I am running these agents: Watchdog, blockverify, download-cern, download-remove, download-tier1, download-tier2, download-tier3, exp-pfn-buffer, exp-pfn-mss, mss-migrate, mss-stage, mss-stage-cern, mss-stage-local
2015-08-05 17:16:41: Watchdog18215: (re)connecting to database
2015-08-05 17:16:41: Watchdog18215: Creating new private DBH
Watchdog: pid 18783 started in /home/cmsprod/phedex/Dev_T1_US_FNAL_Buffer/state/watchdog/
T1 tape pmgr: start Dev
-bash-4.1$ ./pmgr Debug start
Testing your DB connection
2015-08-05 17:16:47: CheckDBConnection18791: (re)connecting to database
2015-08-05 17:16:49: CheckDBConnection18791: disconnected from database
Connection to TMDB with /home/cmsprod/phedex/info/DBParam:Debug/T1USFNAL succeeded
FileExport: removing old stop flag /home/cmsprod/phedex/Debug_T1_US_FNAL_Buffer/state/exp-pfn-buffer//stop
FileExport: pid 18849 started in /home/cmsprod/phedex/Debug_T1_US_FNAL_Buffer/state/exp-pfn-buffer/
FileExport: removing old stop flag /home/cmsprod/phedex/Debug_T1_US_FNAL_Buffer/state/exp-pfn-mss//stop
FileExport: pid 18860 started in /home/cmsprod/phedex/Debug_T1_US_FNAL_Buffer/state/exp-pfn-mss/
FileRemove: removing old stop flag /home/cmsprod/phedex/Debug_T1_US_FNAL_Buffer/state/download-remove//stop
FileRemove: pid 18874 started in /home/cmsprod/phedex/Debug_T1_US_FNAL_Buffer/state/download-remove/
Unknown option: jobs
FileDownload: removing old stop flag /home/cmsprod/phedex/Debug_T1_US_FNAL_Buffer/state/download-cern//stop
FileDownload: pid 18911 started in /home/cmsprod/phedex/Debug_T1_US_FNAL_Buffer/state/download-cern/
Unknown option: jobs
FileDownload: removing old stop flag /home/cmsprod/phedex/Debug_T1_US_FNAL_Buffer/state/download-tier1//stop
FileDownload: pid 18924 started in /home/cmsprod/phedex/Debug_T1_US_FNAL_Buffer/state/download-tier1/
Unknown option: jobs
FileDownload: removing old stop flag /home/cmsprod/phedex/Debug_T1_US_FNAL_Buffer/state/download-tier2//stop
FileDownload: pid 18936 started in /home/cmsprod/phedex/Debug_T1_US_FNAL_Buffer/state/download-tier2/
Unknown option: jobs
FileDownload: removing old stop flag /home/cmsprod/phedex/Debug_T1_US_FNAL_Buffer/state/download-tier3//stop
FileDownload: pid 18947 started in /home/cmsprod/phedex/Debug_T1_US_FNAL_Buffer/state/download-tier3/
FileStager: removing old stop flag /home/cmsprod/phedex/Debug_T1_US_FNAL_Buffer/state/mss-stage-local//stop
FileStager: pid 18960 started in /home/cmsprod/phedex/Debug_T1_US_FNAL_Buffer/state/mss-stage-local/
FileStager: removing old stop flag /home/cmsprod/phedex/Debug_T1_US_FNAL_Buffer/state/mss-stage//stop
FileStager: pid 18971 started in /home/cmsprod/phedex/Debug_T1_US_FNAL_Buffer/state/mss-stage/
FileStager: removing old stop flag /home/cmsprod/phedex/Debug_T1_US_FNAL_Buffer/state/mss-stage-cern//stop
FileStager: pid 18982 started in /home/cmsprod/phedex/Debug_T1_US_FNAL_Buffer/state/mss-stage-cern/
FileMSSMigrate: removing old stop flag /home/cmsprod/phedex/Debug_T1_US_FNAL_Buffer/state/mss-migrate//stop
FileMSSMigrate: pid 18994 started in /home/cmsprod/phedex/Debug_T1_US_FNAL_Buffer/state/mss-migrate/
BlockDownloadVerify: removing old stop flag /home/cmsprod/phedex/Debug_T1_US_FNAL_Buffer/state/blockverify//stop
BlockDownloadVerify: pid 19019 started in /home/cmsprod/phedex/Debug_T1_US_FNAL_Buffer/state/blockverify/
Watchdog: removing old stop flag /home/cmsprod/phedex/Debug_T1_US_FNAL_Buffer/state/watchdog//stop
2015-08-05 17:16:54: Watchdog19054: starting Agent session (id=2)
2015-08-05 17:16:54: Watchdog19054: STATISTICS: Reporting every 43200 seconds, detail=0
2015-08-05 17:16:54: Watchdog19054: has successfully initialised
2015-08-05 17:16:54: Watchdog19054: debug: Agent was loaded DB=>1, CYCLE=1, DROPBOX=1
2015-08-05 17:16:54: Watchdog19054: Lookup agent "download-cern"
2015-08-05 17:16:54: Watchdog19054: Lookup agent "download-tier1"
2015-08-05 17:16:54: Watchdog19054: Lookup agent "download-tier2"
2015-08-05 17:16:54: Watchdog19054: Lookup agent "download-tier3"
2015-08-05 17:16:54: Watchdog19054: Lookup agent "download-remove"
2015-08-05 17:16:54: Watchdog19054: Lookup agent "exp-pfn-buffer"
2015-08-05 17:16:54: Watchdog19054: Lookup agent "exp-pfn-mss"
2015-08-05 17:16:54: Watchdog19054: Lookup agent "mss-migrate"
2015-08-05 17:16:54: Watchdog19054: Lookup agent "mss-stage-local"
2015-08-05 17:16:54: Watchdog19054: Lookup agent "mss-stage"
2015-08-05 17:16:54: Watchdog19054: Lookup agent "mss-stage-cern"
2015-08-05 17:16:54: Watchdog19054: Lookup agent "blockverify"
2015-08-05 17:16:54: Watchdog19054: I am running these agents: Watchdog, blockverify, download-cern, download-remove, download-tier1, download-tier2, download-tier3, exp-pfn-buffer, exp-pfn-mss, mss-migrate, mss-stage, mss-stage-cern, mss-stage-local
2015-08-05 17:16:54: Watchdog19054: (re)connecting to database
2015-08-05 17:16:54: Watchdog19054: Creating new private DBH
Watchdog: pid 19653 started in /home/cmsprod/phedex/Debug_T1_US_FNAL_Buffer/state/watchdog/
T1 tape pmgr: start Debug
-bash-4.1$ ./pmgr Prod start
Testing your DB connection
2015-08-05 17:17:01: CheckDBConnection19662: (re)connecting to database
2015-08-05 17:17:02: CheckDBConnection19662: disconnected from database
Connection to TMDB with /home/cmsprod/phedex/info/DBParam:Prod/T1USFNAL succeeded
FileExport: removing old stop flag /home/cmsprod/phedex/Prod_T1_US_FNAL_Buffer/state/exp-pfn-buffer//stop
FileExport: pid 19949 started in /home/cmsprod/phedex/Prod_T1_US_FNAL_Buffer/state/exp-pfn-buffer/
FileExport: removing old stop flag /home/cmsprod/phedex/Prod_T1_US_FNAL_Buffer/state/exp-pfn-mss//stop
FileExport: pid 20001 started in /home/cmsprod/phedex/Prod_T1_US_FNAL_Buffer/state/exp-pfn-mss/
FileRemove: removing old stop flag /home/cmsprod/phedex/Prod_T1_US_FNAL_Buffer/state/download-remove//stop
FileRemove: pid 20069 started in /home/cmsprod/phedex/Prod_T1_US_FNAL_Buffer/state/download-remove/
Unknown option: jobs
FileDownload: removing old stop flag /home/cmsprod/phedex/Prod_T1_US_FNAL_Buffer/state/download-cern//stop
FileDownload: pid 20100 started in /home/cmsprod/phedex/Prod_T1_US_FNAL_Buffer/state/download-cern/
Unknown option: jobs
FileDownload: removing old stop flag /home/cmsprod/phedex/Prod_T1_US_FNAL_Buffer/state/download-tier1//stop
FileDownload: pid 20134 started in /home/cmsprod/phedex/Prod_T1_US_FNAL_Buffer/state/download-tier1/
Unknown option: jobs
FileDownload: removing old stop flag /home/cmsprod/phedex/Prod_T1_US_FNAL_Buffer/state/download-tier2//stop
FileDownload: pid 20179 started in /home/cmsprod/phedex/Prod_T1_US_FNAL_Buffer/state/download-tier2/
Unknown option: jobs
FileDownload: removing old stop flag /home/cmsprod/phedex/Prod_T1_US_FNAL_Buffer/state/download-tier3//stop
FileDownload: pid 20207 started in /home/cmsprod/phedex/Prod_T1_US_FNAL_Buffer/state/download-tier3/
FileStager: removing old stop flag /home/cmsprod/phedex/Prod_T1_US_FNAL_Buffer/state/mss-stage-local//stop
FileStager: pid 20219 started in /home/cmsprod/phedex/Prod_T1_US_FNAL_Buffer/state/mss-stage-local/
FileStager: removing old stop flag /home/cmsprod/phedex/Prod_T1_US_FNAL_Buffer/state/mss-stage//stop
FileStager: pid 20234 started in /home/cmsprod/phedex/Prod_T1_US_FNAL_Buffer/state/mss-stage/
FileStager: removing old stop flag /home/cmsprod/phedex/Prod_T1_US_FNAL_Buffer/state/mss-stage-cern//stop
FileStager: pid 20245 started in /home/cmsprod/phedex/Prod_T1_US_FNAL_Buffer/state/mss-stage-cern/
FileMSSMigrate: removing old stop flag /home/cmsprod/phedex/Prod_T1_US_FNAL_Buffer/state/mss-migrate//stop
FileMSSMigrate: pid 20258 started in /home/cmsprod/phedex/Prod_T1_US_FNAL_Buffer/state/mss-migrate/
BlockDownloadVerify: removing old stop flag /home/cmsprod/phedex/Prod_T1_US_FNAL_Buffer/state/blockverify//stop
BlockDownloadVerify: pid 20314 started in /home/cmsprod/phedex/Prod_T1_US_FNAL_Buffer/state/blockverify/
Watchdog: removing old stop flag /home/cmsprod/phedex/Prod_T1_US_FNAL_Buffer/state/watchdog//stop
2015-08-05 17:17:07: Watchdog20345: starting Agent session (id=2)
2015-08-05 17:17:07: Watchdog20345: STATISTICS: Reporting every 43200 seconds, detail=0
2015-08-05 17:17:07: Watchdog20345: has successfully initialised
2015-08-05 17:17:07: Watchdog20345: Lookup agent "download-cern"
2015-08-05 17:17:07: Watchdog20345: Lookup agent "download-tier1"
2015-08-05 17:17:07: Watchdog20345: Lookup agent "download-tier2"
2015-08-05 17:17:07: Watchdog20345: Lookup agent "download-tier3"
2015-08-05 17:17:07: Watchdog20345: Lookup agent "download-remove"
2015-08-05 17:17:07: Watchdog20345: Lookup agent "exp-pfn-buffer"
2015-08-05 17:17:07: Watchdog20345: Lookup agent "exp-pfn-mss"
2015-08-05 17:17:07: Watchdog20345: Lookup agent "mss-migrate"
2015-08-05 17:17:07: Watchdog20345: Lookup agent "mss-stage-local"
2015-08-05 17:17:07: Watchdog20345: Lookup agent "mss-stage"
2015-08-05 17:17:07: Watchdog20345: Lookup agent "mss-stage-cern"
2015-08-05 17:17:07: Watchdog20345: Lookup agent "blockverify"
2015-08-05 17:17:07: Watchdog20345: I am running these agents: Watchdog, blockverify, download-cern, download-remove, download-tier1, download-tier2, download-tier3, exp-pfn-buffer, exp-pfn-mss, mss-migrate, mss-stage, mss-stage-cern, mss-stage-local
2015-08-05 17:17:07: Watchdog20345: (re)connecting to database
Watchdog: pid 21033 started in /home/cmsprod/phedex/Prod_T1_US_FNAL_Buffer/state/watchdog/
T1 tape pmgr: start Prod
-bash-4.1$

Sent email to cms-t1 .

#7 Updated by Natalia Ratnikova over 4 years ago

[root@cmsadmin1 DOWNTIME-Aug-5-2015]# time for s in `cat all_dcache_disk_pools.1`; do echo "Processing $s"; ssh $s 'puppet agent --disable "Downtime work Aug 5 "'; ssh $s 'dcache stop' ; done

[root@cmsadmin1 DOWNTIME-Aug-5-2015]# time for s in `cat all_dcache_disk_pools.1`; do echo "Processing $s"; ssh $s 'puppet agent --disable "Downtime work Aug 5 "'; ssh $s 'dcache stop' ; done
Processing cmsstor169.fnal.gov
Stopping gridftp-cmsstor169Domain 0 1 done
Stopping w-cmsstor169-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor169-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor169-disk-disk1Domain 0 1 2 3 4 done
Processing cmsstor171.fnal.gov
Stopping gridftp-cmsstor171Domain 0 1 done
Stopping w-cmsstor171-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor171-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor171-disk-disk1Domain 0 1 2 3 4 done
Processing cmsstor172.fnal.gov
Stopping gridftp-cmsstor172Domain 0 1 done
Stopping w-cmsstor172-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor172-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor172-disk-disk1Domain 0 1 2 3 done
Processing cmsstor173.fnal.gov
Stopping gridftp-cmsstor173Domain 0 1 done
Stopping w-cmsstor173-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor173-disk-disk2Domain 0 1 2 3 4 done
Stopping w-cmsstor173-disk-disk1Domain 0 1 2 3 done
Processing cmsstor174.fnal.gov
Stopping gridftp-cmsstor174Domain 0 1 done
Stopping w-cmsstor174-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor174-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor174-disk-disk1Domain 0 1 2 3 done
Processing cmsstor175.fnal.gov
Stopping gridftp-cmsstor175Domain 0 1 2 done
Stopping w-cmsstor175-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor175-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor175-disk-disk1Domain 0 1 2 3 4 done
Processing cmsstor176.fnal.gov
Stopping gridftp-cmsstor176Domain 0 1 done
Stopping w-cmsstor176-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor176-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor176-disk-disk1Domain 0 1 2 3 done
Processing cmsstor177.fnal.gov
Stopping gridftp-cmsstor177Domain 0 1 done
Stopping w-cmsstor177-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor177-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor177-disk-disk1Domain 0 1 2 3 done
Processing cmsstor178.fnal.gov
Stopping gridftp-cmsstor178Domain 0 1 done
Stopping w-cmsstor178-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor178-disk-disk2Domain 0 1 2 3 4 done
Stopping w-cmsstor178-disk-disk1Domain 0 1 2 3 done
Processing cmsstor179.fnal.gov
Stopping gridftp-cmsstor179Domain 0 1 done
Stopping w-cmsstor179-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor179-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor179-disk-disk1Domain 0 1 2 3 done
Processing cmsstor181.fnal.gov
Stopping gridftp-cmsstor181Domain 0 1 done
Stopping w-cmsstor181-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor181-disk-disk2Domain 0 1 2 3 4 done
Stopping w-cmsstor181-disk-disk1Domain 0 1 2 3 4 done
Processing cmsstor182.fnal.gov
Stopping gridftp-cmsstor182Domain 0 1 done
Stopping w-cmsstor182-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor182-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor182-disk-disk1Domain 0 1 2 3 done
Processing cmsstor183.fnal.gov
Stopping gridftp-cmsstor183Domain 0 1 done
Stopping w-cmsstor183-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor183-disk-disk2Domain 0 1 2 3 4 done
Stopping w-cmsstor183-disk-disk1Domain 0 1 2 3 done
Processing cmsstor184.fnal.gov
Stopping gridftp-cmsstor184Domain 0 1 done
Stopping w-cmsstor184-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor184-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor184-disk-disk1Domain 0 1 2 3 4 done
Processing cmsstor185.fnal.gov
Stopping gridftp-cmsstor185Domain 0 1 done
Stopping w-cmsstor185-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor185-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor185-disk-disk1Domain 0 1 2 3 done
Processing cmsstor186.fnal.gov
Stopping gridftp-cmsstor186Domain 0 1 done
Stopping w-cmsstor186-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor186-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor186-disk-disk1Domain 0 1 2 3 4 done
Processing cmsstor187.fnal.gov
Stopping gridftp-cmsstor187Domain 0 1 done
Stopping w-cmsstor187-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor187-disk-disk2Domain 0 1 2 3 4 done
Stopping w-cmsstor187-disk-disk1Domain 0 1 2 3 done
Processing cmsstor188.fnal.gov
Stopping gridftp-cmsstor188Domain 0 1 done
Stopping w-cmsstor188-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor188-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor188-disk-disk1Domain 0 1 2 3 done
Processing cmsstor189.fnal.gov
Stopping gridftp-cmsstor189Domain 0 1 done
Stopping w-cmsstor189-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor189-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor189-disk-disk1Domain 0 1 2 3 done
Processing cmsstor191.fnal.gov
Stopping gridftp-cmsstor191Domain 0 1 2 done
Stopping w-cmsstor191-disk-disk3Domain 0 1 2 3 4 done
Stopping w-cmsstor191-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor191-disk-disk1Domain 0 1 2 3 done
Processing cmsstor192.fnal.gov
Stopping gridftp-cmsstor192Domain 0 1 done
Stopping w-cmsstor192-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor192-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor192-disk-disk1Domain 0 1 2 3 done
Processing cmsstor193.fnal.gov
Stopping gridftp-cmsstor193Domain 0 1 done
Stopping w-cmsstor193-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor193-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor193-disk-disk1Domain 0 1 2 3 4 done
Processing cmsstor194.fnal.gov
Stopping gridftp-cmsstor194Domain 0 1 2 done
Stopping w-cmsstor194-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor194-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor194-disk-disk1Domain 0 1 2 3 done
Processing cmsstor195.fnal.gov
Stopping gridftp-cmsstor195Domain 0 1 done
Stopping w-cmsstor195-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor195-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor195-disk-disk1Domain 0 1 2 3 done
Processing cmsstor196.fnal.gov
Stopping gridftp-cmsstor196Domain 0 1 done
Stopping w-cmsstor196-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor196-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor196-disk-disk1Domain 0 1 2 3 done
Processing cmsstor197.fnal.gov
Stopping gridftp-cmsstor197Domain 0 1 2 done
Stopping w-cmsstor197-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor197-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor197-disk-disk1Domain 0 1 2 3 done
Processing cmsstor198.fnal.gov
Stopping gridftp-cmsstor198Domain 0 1 done
Stopping w-cmsstor198-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor198-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor198-disk-disk1Domain 0 1 2 3 done
Processing cmsstor199.fnal.gov
Stopping gridftp-cmsstor199Domain 0 1 done
Stopping w-cmsstor199-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor199-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor199-disk-disk1Domain 0 1 2 3 done
Processing cmsstor201.fnal.gov
Stopping gridftp-cmsstor201Domain 0 1 done
Stopping w-cmsstor201-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor201-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor201-disk-disk1Domain 0 1 2 done
Processing cmsstor202.fnal.gov
Stopping gridftp-cmsstor202Domain 0 1 done
Stopping w-cmsstor202-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor202-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor202-disk-disk1Domain 0 1 2 3 done
Processing cmsstor203.fnal.gov
Stopping gridftp-cmsstor203Domain 0 1 done
Stopping w-cmsstor203-disk-disk3Domain 0 1 2 3 4 done
Stopping w-cmsstor203-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor203-disk-disk1Domain 0 1 2 3 done
Processing cmsstor204.fnal.gov
Stopping gridftp-cmsstor204Domain 0 1 done
Stopping w-cmsstor204-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor204-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor204-disk-disk1Domain 0 1 2 3 done
Processing cmsstor205.fnal.gov
Stopping gridftp-cmsstor205Domain 0 1 done
Stopping w-cmsstor205-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor205-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor205-disk-disk1Domain 0 1 2 3 done
Processing cmsstor206.fnal.gov
Stopping gridftp-cmsstor206Domain 0 1 done
Stopping w-cmsstor206-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor206-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor206-disk-disk1Domain 0 1 2 3 done
Processing cmsstor207.fnal.gov
Stopping gridftp-cmsstor207Domain 0 1 done
Stopping w-cmsstor207-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor207-disk-disk2Domain 0 1 2 3 4 done
Stopping w-cmsstor207-disk-disk1Domain 0 1 2 3 done
Processing cmsstor208.fnal.gov
Stopping gridftp-cmsstor208Domain 0 1 done
Stopping w-cmsstor208-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor208-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor208-disk-disk1Domain 0 1 2 3 done
Processing cmsstor209.fnal.gov
Stopping gridftp-cmsstor209Domain 0 1 2 done
Stopping w-cmsstor209-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor209-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor209-disk-disk1Domain 0 1 2 3 4 done
Processing cmsstor210.fnal.gov
Stopping gridftp-cmsstor210Domain 0 1 done
Stopping w-cmsstor210-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor210-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor210-disk-disk1Domain 0 1 2 3 done
Processing cmsstor211.fnal.gov
Stopping gridftp-cmsstor211Domain 0 1 done
Stopping w-cmsstor211-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor211-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor211-disk-disk1Domain 0 1 2 3 done
Processing cmsstor212.fnal.gov
Stopping gridftp-cmsstor212Domain 0 1 done
Stopping w-cmsstor212-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor212-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor212-disk-disk1Domain 0 1 2 3 done
Processing cmsstor213.fnal.gov
Stopping gridftp-cmsstor213Domain 0 1 done
Stopping w-cmsstor213-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor213-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor213-disk-disk1Domain 0 1 2 3 done
Processing cmsstor214.fnal.gov
Stopping gridftp-cmsstor214Domain 0 1 done
Stopping w-cmsstor214-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor214-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor214-disk-disk1Domain 0 1 2 3 done
Processing cmsstor215.fnal.gov
Stopping gridftp-cmsstor215Domain 0 1 done
Stopping w-cmsstor215-disk-disk3Domain 0 1 2 3 4 5 6 7 8 9 [K] 10 done
Stopping w-cmsstor215-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor215-disk-disk1Domain 0 1 2 3 done
Processing cmsstor216.fnal.gov
Stopping gridftp-cmsstor216Domain 0 done
Stopping w-cmsstor216-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor216-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor216-disk-disk1Domain 0 1 2 3 done
Processing cmsstor217.fnal.gov
Stopping gridftp-cmsstor217Domain 0 1 2 done
Stopping w-cmsstor217-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor217-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor217-disk-disk1Domain 0 1 2 3 done
A heap dump file, /var/log/dcache/w-cmsstor217-disk-disk2Domain-oom.hprof,
was found for domain w-cmsstor217-disk-disk2Domain. The file was generated
at 2014-12-30 09:51:39.074621366 -0600 as a result of an out of memory
failure in the domain.

The dump contains debugging information that may help a developer
determine the cause of high memory usage. Please note that the dump may
contain confidential information.

As long as the file exists no other dumps will be generated on out of
memory failures. Please move or delete
/var/log/dcache/w-cmsstor217-disk-disk2Domain-oom.hprof. Consider
increasing the dcache.java.memory.heap property. The current value is
5096m.

Processing cmsstor218.fnal.gov
Stopping gridftp-cmsstor218Domain 0 1 done
Stopping w-cmsstor218-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor218-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor218-disk-disk1Domain 0 1 2 3 done
Processing cmsstor219.fnal.gov
Stopping gridftp-cmsstor219Domain 0 done
Stopping w-cmsstor219-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor219-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor219-disk-disk1Domain 0 1 2 3 done
Processing cmsstor220.fnal.gov
Stopping gridftp-cmsstor220Domain 0 1 done
Stopping w-cmsstor220-disk-disk3Domain 0 1 2 3 4 done
Stopping w-cmsstor220-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor220-disk-disk1Domain 0 1 2 3 4 done
Processing cmsstor221.fnal.gov
Stopping gridftp-cmsstor221Domain 0 1 done
Stopping w-cmsstor221-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor221-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor221-disk-disk1Domain 0 1 2 3 done
Processing cmsstor222.fnal.gov
Stopping gridftp-cmsstor222Domain 0 1 done
Stopping w-cmsstor222-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor222-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor222-disk-disk1Domain 0 1 2 3 4 done
Processing cmsstor223.fnal.gov
Stopping gridftp-cmsstor223Domain 0 1 done
Stopping w-cmsstor223-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor223-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor223-disk-disk1Domain 0 1 2 3 done
Processing cmsstor224.fnal.gov
Stopping gridftp-cmsstor224Domain 0 1 done
Stopping w-cmsstor224-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor224-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor224-disk-disk1Domain 0 1 2 3 done
Processing cmsstor225.fnal.gov
Stopping gridftp-cmsstor225Domain 0 1 done
Stopping w-cmsstor225-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor225-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor225-disk-disk1Domain 0 1 2 3 done
Processing cmsstor226.fnal.gov
Stopping gridftp-cmsstor226Domain 0 1 done
Stopping w-cmsstor226-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor226-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor226-disk-disk1Domain 0 1 2 3 4 done
Processing cmsstor227.fnal.gov
Stopping gridftp-cmsstor227Domain 0 1 done
Stopping w-cmsstor227-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor227-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor227-disk-disk1Domain 0 1 2 3 done
Processing cmsstor228.fnal.gov
Stopping gridftp-cmsstor228Domain 0 1 done
Stopping w-cmsstor228-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor228-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor228-disk-disk1Domain 0 1 2 3 done
Processing cmsstor229.fnal.gov
Stopping gridftp-cmsstor229Domain 0 1 done
Stopping w-cmsstor229-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor229-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor229-disk-disk1Domain 0 1 2 3 done
Processing cmsstor230.fnal.gov
Stopping gridftp-cmsstor230Domain 0 1 done
Stopping w-cmsstor230-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor230-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor230-disk-disk1Domain 0 1 2 3 4 done
Processing cmsstor231.fnal.gov
Stopping gridftp-cmsstor231Domain 0 1 done
Stopping w-cmsstor231-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor231-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor231-disk-disk1Domain 0 1 2 3 done
Processing cmsstor232.fnal.gov
Stopping gridftp-cmsstor232Domain 0 1 done
Stopping w-cmsstor232-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor232-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor232-disk-disk1Domain 0 1 2 3 done
Processing cmsstor233.fnal.gov
Stopping gridftp-cmsstor233Domain 0 1 done
Stopping w-cmsstor233-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor233-disk-disk2Domain 0 1 2 done
Stopping w-cmsstor233-disk-disk1Domain 0 1 2 3 done
Processing cmsstor234.fnal.gov
Stopping gridftp-cmsstor234Domain 0 1 done
Stopping w-cmsstor234-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor234-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor234-disk-disk1Domain 0 1 2 3 done
Processing cmsstor235.fnal.gov
Stopping gridftp-cmsstor235Domain 0 1 done
Stopping w-cmsstor235-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor235-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor235-disk-disk1Domain 0 1 2 3 done
Processing cmsstor236.fnal.gov
Stopping gridftp-cmsstor236Domain 0 1 done
Stopping w-cmsstor236-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor236-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor236-disk-disk1Domain 0 1 2 3 done
Processing cmsstor237.fnal.gov
Stopping gridftp-cmsstor237Domain 0 1 done
Stopping w-cmsstor237-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor237-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor237-disk-disk1Domain 0 1 2 3 done
Processing cmsstor238.fnal.gov
Stopping gridftp-cmsstor238Domain 0 1 done
Stopping w-cmsstor238-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor238-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor238-disk-disk1Domain 0 1 2 3 done
Processing cmsstor239.fnal.gov
Stopping gridftp-cmsstor239Domain 0 1 done
Stopping w-cmsstor239-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor239-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor239-disk-disk1Domain 0 1 2 3 done
Processing cmsstor240.fnal.gov
Stopping gridftp-cmsstor240Domain 0 1 done
Stopping w-cmsstor240-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor240-disk-disk2Domain 0 1 2 3 4 done
Stopping w-cmsstor240-disk-disk1Domain 0 1 2 3 done
Processing cmsstor241.fnal.gov
Stopping gridftp-cmsstor241Domain 0 1 done
Stopping w-cmsstor241-disk-disk3Domain 0 1 2 3 4 done
Stopping w-cmsstor241-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor241-disk-disk1Domain 0 1 2 3 done
Processing cmsstor242.fnal.gov
Stopping gridftp-cmsstor242Domain 0 1 done
Stopping w-cmsstor242-disk-disk3Domain 0 1 2 done
Stopping w-cmsstor242-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor242-disk-disk1Domain 0 1 2 3 done
Processing cmsstor243.fnal.gov
Stopping gridftp-cmsstor243Domain 0 1 done
Stopping w-cmsstor243-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor243-disk-disk2Domain 0 1 2 done
Stopping w-cmsstor243-disk-disk1Domain 0 1 2 3 4 done
Processing cmsstor244.fnal.gov
Stopping gridftp-cmsstor244Domain 0 1 done
Stopping w-cmsstor244-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor244-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor244-disk-disk1Domain 0 1 2 3 done
Processing cmsstor245.fnal.gov
Stopping gridftp-cmsstor245Domain 0 1 done
Stopping w-cmsstor245-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor245-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor245-disk-disk1Domain 0 1 2 3 done
Processing cmsstor246.fnal.gov
Stopping gridftp-cmsstor246Domain 0 1 done
Stopping w-cmsstor246-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor246-disk-disk2Domain 0 1 2 3 4 done
Stopping w-cmsstor246-disk-disk1Domain 0 1 2 3 done
Processing cmsstor247.fnal.gov
Stopping gridftp-cmsstor247Domain 0 1 done
Stopping w-cmsstor247-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor247-disk-disk2Domain 0 1 2 3 4 done
Stopping w-cmsstor247-disk-disk1Domain 0 1 2 3 4 done
Processing cmsstor248.fnal.gov
Stopping gridftp-cmsstor248Domain 0 1 done
Stopping w-cmsstor248-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor248-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor248-disk-disk1Domain 0 1 2 3 done
Processing cmsstor249.fnal.gov
Stopping gridftp-cmsstor249Domain 0 1 done
Stopping w-cmsstor249-disk-disk3Domain 0 1 2 3 4 done
Stopping w-cmsstor249-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor249-disk-disk1Domain 0 1 2 3 done
Processing cmsstor250.fnal.gov
Stopping gridftp-cmsstor250Domain 0 1 done
Stopping w-cmsstor250-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor250-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor250-disk-disk1Domain 0 1 2 3 done
Processing cmsstor251.fnal.gov
Stopping gridftp-cmsstor251Domain 0 1 done
Stopping w-cmsstor251-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor251-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor251-disk-disk1Domain 0 1 2 3 done
Processing cmsstor261.fnal.gov
Stopping gridftp-cmsstor261Domain 0 done
Stopping w-cmsstor261-disk-disk3Domain 0 1 2 done
Stopping w-cmsstor261-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor261-disk-disk1Domain 0 1 2 3 done
Processing cmsstor262.fnal.gov
Stopping gridftp-cmsstor262Domain 0 1 done
Stopping w-cmsstor262-disk-disk3Domain 0 1 2 3 4 done
Stopping w-cmsstor262-disk-disk2Domain 0 1 2 done
Stopping w-cmsstor262-disk-disk1Domain 0 1 2 3 done
Processing cmsstor263.fnal.gov
Stopping gridftp-cmsstor263Domain 0 1 done
Stopping w-cmsstor263-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor263-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor263-disk-disk1Domain 0 1 2 3 done
Processing cmsstor264.fnal.gov
Stopping gridftp-cmsstor264Domain 0 done
Stopping w-cmsstor264-disk-disk3Domain 0 1 2 3 4 done
Stopping w-cmsstor264-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor264-disk-disk1Domain 0 1 2 3 done
Processing cmsstor265.fnal.gov
Stopping gridftp-cmsstor265Domain 0 1 done
Stopping w-cmsstor265-disk-disk3Domain 0 1 2 done
Stopping w-cmsstor265-disk-disk2Domain 0 1 2 done
Stopping w-cmsstor265-disk-disk1Domain 0 1 2 3 done
Processing cmsstor266.fnal.gov
Stopping gridftp-cmsstor266Domain 0 done
Stopping w-cmsstor266-disk-disk3Domain 0 1 2 done
Stopping w-cmsstor266-disk-disk2Domain 0 1 2 3 4 done
Stopping w-cmsstor266-disk-disk1Domain 0 1 2 done
Processing cmsstor267.fnal.gov
Stopping gridftp-cmsstor267Domain 0 done
Stopping w-cmsstor267-disk-disk3Domain 0 1 2 done
Stopping w-cmsstor267-disk-disk2Domain 0 1 2 done
Stopping w-cmsstor267-disk-disk1Domain 0 1 2 done
Processing cmsstor268.fnal.gov
Stopping gridftp-cmsstor268Domain 0 1 done
Stopping w-cmsstor268-disk-disk3Domain 0 1 2 done
Stopping w-cmsstor268-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor268-disk-disk1Domain 0 1 2 3 4 done
Processing cmsstor269.fnal.gov
Stopping gridftp-cmsstor269Domain 0 1 done
Stopping w-cmsstor269-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor269-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor269-disk-disk1Domain 0 1 2 done
Processing cmsstor270.fnal.gov
Stopping gridftp-cmsstor270Domain 0 done
Stopping w-cmsstor270-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor270-disk-disk2Domain 0 1 2 3 4 done
Stopping w-cmsstor270-disk-disk1Domain 0 1 2 3 4 done
Processing cmsstor271.fnal.gov
Stopping gridftp-cmsstor271Domain 0 done
Stopping w-cmsstor271-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor271-disk-disk2Domain 0 1 2 done
Stopping w-cmsstor271-disk-disk1Domain 0 1 2 3 done
Processing cmsstor272.fnal.gov
Stopping gridftp-cmsstor272Domain 0 done
Stopping w-cmsstor272-disk-disk3Domain 0 1 2 done
Stopping w-cmsstor272-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor272-disk-disk1Domain 0 1 2 3 4 done
Processing cmsstor273.fnal.gov
Stopping gridftp-cmsstor273Domain 0 done
Stopping w-cmsstor273-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor273-disk-disk2Domain 0 1 2 3 4 done
Stopping w-cmsstor273-disk-disk1Domain 0 1 2 3 done
Processing cmsstor274.fnal.gov
Stopping gridftp-cmsstor274Domain 0 1 done
Stopping w-cmsstor274-disk-disk3Domain 0 1 2 done
Stopping w-cmsstor274-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor274-disk-disk1Domain 0 1 2 done
Processing cmsstor275.fnal.gov
Stopping gridftp-cmsstor275Domain 0 done
Stopping w-cmsstor275-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor275-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor275-disk-disk1Domain 0 1 2 3 done
Processing cmsstor276.fnal.gov
Stopping gridftp-cmsstor276Domain 0 done
Stopping w-cmsstor276-disk-disk3Domain 0 1 2 done
Stopping w-cmsstor276-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor276-disk-disk1Domain 0 1 2 done
Processing cmsstor277.fnal.gov
Stopping gridftp-cmsstor277Domain 0 done
Stopping w-cmsstor277-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor277-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor277-disk-disk1Domain 0 1 2 done
Processing cmsstor278.fnal.gov
Stopping gridftp-cmsstor278Domain 0 done
Stopping w-cmsstor278-disk-disk3Domain 0 1 2 done
Stopping w-cmsstor278-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor278-disk-disk1Domain 0 1 2 3 done
Processing cmsstor279.fnal.gov
Stopping gridftp-cmsstor279Domain 0 done
Stopping w-cmsstor279-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor279-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor279-disk-disk1Domain 0 1 2 3 done
Processing cmsstor280.fnal.gov
Stopping gridftp-cmsstor280Domain 0 1 done
Stopping w-cmsstor280-disk-disk3Domain 0 1 2 done
Stopping w-cmsstor280-disk-disk2Domain 0 1 2 done
Stopping w-cmsstor280-disk-disk1Domain 0 1 2 3 done
Processing cmsstor281.fnal.gov
Stopping gridftp-cmsstor281Domain 0 1 done
Stopping w-cmsstor281-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor281-disk-disk2Domain 0 1 2 done
Stopping w-cmsstor281-disk-disk1Domain 0 1 2 3 done
Processing cmsstor282.fnal.gov
Stopping gridftp-cmsstor282Domain 0 1 done
Stopping w-cmsstor282-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor282-disk-disk2Domain 0 1 2 done
Stopping w-cmsstor282-disk-disk1Domain 0 1 2 3 done
Processing cmsstor283.fnal.gov
Stopping gridftp-cmsstor283Domain 0 done
Stopping w-cmsstor283-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor283-disk-disk2Domain 0 1 2 done
Stopping w-cmsstor283-disk-disk1Domain 0 1 2 3 done
Processing cmsstor284.fnal.gov
Stopping gridftp-cmsstor284Domain 0 done
Stopping w-cmsstor284-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor284-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor284-disk-disk1Domain 0 1 2 3 done
Processing cmsstor285.fnal.gov
Stopping gridftp-cmsstor285Domain 0 done
Stopping w-cmsstor285-disk-disk3Domain 0 1 2 done
Stopping w-cmsstor285-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor285-disk-disk1Domain 0 1 2 done
Processing cmsstor286.fnal.gov
Stopping gridftp-cmsstor286Domain 0 done
Stopping w-cmsstor286-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor286-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor286-disk-disk1Domain 0 1 2 3 done
Processing cmsstor287.fnal.gov
Stopping gridftp-cmsstor287Domain 0 1 done
Stopping w-cmsstor287-disk-disk3Domain 0 1 2 done
Stopping w-cmsstor287-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor287-disk-disk1Domain 0 1 2 done
Processing cmsstor288.fnal.gov
Stopping gridftp-cmsstor288Domain 0 done
Stopping w-cmsstor288-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor288-disk-disk2Domain 0 1 2 done
Stopping w-cmsstor288-disk-disk1Domain 0 1 2 3 done
Processing cmsstor289.fnal.gov
Stopping gridftp-cmsstor289Domain 0 1 done
Stopping w-cmsstor289-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor289-disk-disk2Domain 0 1 2 done
Stopping w-cmsstor289-disk-disk1Domain 0 1 2 3 done
Processing cmsstor290.fnal.gov
Stopping gridftp-cmsstor290Domain 0 1 done
Stopping w-cmsstor290-disk-disk3Domain 0 1 2 3 4 done
Stopping w-cmsstor290-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor290-disk-disk1Domain 0 1 2 done
Processing cmsstor291.fnal.gov
Stopping gridftp-cmsstor291Domain 0 done
Stopping w-cmsstor291-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor291-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor291-disk-disk1Domain 0 1 2 3 done
Processing cmsstor292.fnal.gov
Stopping gridftp-cmsstor292Domain 0 1 done
Stopping w-cmsstor292-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor292-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor292-disk-disk1Domain 0 1 2 3 done
Processing cmsstor293.fnal.gov
Stopping gridftp-cmsstor293Domain 0 done
Stopping w-cmsstor293-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor293-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor293-disk-disk1Domain 0 1 2 3 done
Processing cmsstor294.fnal.gov
Stopping gridftp-cmsstor294Domain 0 1 done
Stopping w-cmsstor294-disk-disk3Domain 0 1 2 3 4 done
Stopping w-cmsstor294-disk-disk2Domain 0 1 2 done
Stopping w-cmsstor294-disk-disk1Domain 0 1 2 3 done
Processing cmsstor309.fnal.gov
Stopping gridftp-cmsstor309Domain 0 1 done
Stopping w-cmsstor309-disk-disk3Domain 0 1 2 3 4 done
Stopping w-cmsstor309-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor309-disk-disk1Domain 0 1 2 3 done
Processing cmsstor310.fnal.gov
Stopping gridftp-cmsstor310Domain 0 1 done
Stopping w-cmsstor310-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor310-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor310-disk-disk1Domain 0 1 2 3 done
Processing cmsstor311.fnal.gov
Stopping gridftp-cmsstor311Domain 0 1 done
Stopping w-cmsstor311-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor311-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor311-disk-disk1Domain 0 1 2 3 done
Processing cmsstor312.fnal.gov
Stopping gridftp-cmsstor312Domain 0 1 done
Stopping w-cmsstor312-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor312-disk-disk2Domain 0 1 2 3 4 5 6 7 8 9 [K] 10 done
Stopping w-cmsstor312-disk-disk1Domain 0 1 2 done
Processing cmsstor313.fnal.gov
Stopping gridftp-cmsstor313Domain 0 1 2 done
Stopping w-cmsstor313-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor313-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor313-disk-disk1Domain 0 1 2 3 4 done
Processing cmsstor314.fnal.gov
Stopping gridftp-cmsstor314Domain 0 1 done
Stopping w-cmsstor314-disk-disk3Domain 0 1 2 3 4 done
Stopping w-cmsstor314-disk-disk2Domain 0 1 2 done
Stopping w-cmsstor314-disk-disk1Domain 0 1 2 3 done
Processing cmsstor315.fnal.gov
Stopping gridftp-cmsstor315Domain 0 1 done
Stopping w-cmsstor315-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor315-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor315-disk-disk1Domain 0 1 2 3 done
Processing cmsstor316.fnal.gov
Stopping gridftp-cmsstor316Domain 0 1 done
Stopping w-cmsstor316-disk-disk3Domain 0 1 2 3 4 done
Stopping w-cmsstor316-disk-disk2Domain 0 1 2 done
Stopping w-cmsstor316-disk-disk1Domain 0 1 2 3 done
Processing cmsstor317.fnal.gov
Stopping gridftp-cmsstor317Domain 0 1 done
Stopping w-cmsstor317-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor317-disk-disk2Domain 0 1 2 3 4 done
Stopping w-cmsstor317-disk-disk1Domain 0 1 2 3 done
Processing cmsstor318.fnal.gov
Stopping gridftp-cmsstor318Domain 0 1 done
Stopping w-cmsstor318-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor318-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor318-disk-disk1Domain 0 1 2 3 4 5 done
Processing cmsstor319.fnal.gov
Stopping gridftp-cmsstor319Domain 0 1 done
Stopping w-cmsstor319-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor319-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor319-disk-disk1Domain 0 1 2 3 done
Processing cmsstor320.fnal.gov
Stopping gridftp-cmsstor320Domain 0 done
Stopping w-cmsstor320-disk-disk3Domain 0 1 2 3 4 done
Stopping w-cmsstor320-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor320-disk-disk1Domain 0 1 2 3 done
Processing cmsstor321.fnal.gov
Stopping gridftp-cmsstor321Domain 0 1 done
Stopping w-cmsstor321-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor321-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor321-disk-disk1Domain 0 1 2 done
Processing cmsstor322.fnal.gov
Stopping gridftp-cmsstor322Domain 0 1 done
Stopping w-cmsstor322-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor322-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor322-disk-disk1Domain 0 1 2 3 4 done
Processing cmsstor323.fnal.gov
Stopping gridftp-cmsstor323Domain 0 done
Stopping w-cmsstor323-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor323-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor323-disk-disk1Domain 0 1 2 3 done
Processing cmsstor324.fnal.gov
Stopping gridftp-cmsstor324Domain 0 done
Stopping w-cmsstor324-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor324-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor324-disk-disk1Domain 0 1 2 3 done
Processing cmsstor325.fnal.gov
Stopping gridftp-cmsstor325Domain 0 done
Stopping w-cmsstor325-disk-disk3Domain 0 1 2 done
Stopping w-cmsstor325-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor325-disk-disk1Domain 0 1 2 done
Processing cmsstor326.fnal.gov
Stopping gridftp-cmsstor326Domain 0 done
Stopping w-cmsstor326-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor326-disk-disk2Domain 0 1 2 done
Stopping w-cmsstor326-disk-disk1Domain 0 1 2 3 done
Processing cmsstor327.fnal.gov
Stopping gridftp-cmsstor327Domain 0 done
Stopping w-cmsstor327-disk-disk3Domain 0 1 2 done
Stopping w-cmsstor327-disk-disk2Domain 0 1 2 done
Stopping w-cmsstor327-disk-disk1Domain 0 1 2 3 done
Processing cmsstor328.fnal.gov
Stopping gridftp-cmsstor328Domain 0 done
Stopping w-cmsstor328-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor328-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor328-disk-disk1Domain 0 1 2 done
Processing cmsstor329.fnal.gov
Stopping gridftp-cmsstor329Domain 0 done
Stopping w-cmsstor329-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor329-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor329-disk-disk1Domain 0 1 2 3 done
Processing cmsstor330.fnal.gov
Stopping gridftp-cmsstor330Domain 0 done
Stopping w-cmsstor330-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor330-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor330-disk-disk1Domain 0 1 2 3 done
Processing cmsstor331.fnal.gov
Stopping gridftp-cmsstor331Domain 0 done
Stopping w-cmsstor331-disk-disk3Domain 0 1 2 done
Stopping w-cmsstor331-disk-disk2Domain 0 1 2 done
Stopping w-cmsstor331-disk-disk1Domain 0 1 2 done
Processing cmsstor332.fnal.gov
Stopping gridftp-cmsstor332Domain 0 done
Stopping w-cmsstor332-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor332-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor332-disk-disk1Domain 0 1 2 done
Processing cmsstor333.fnal.gov
Stopping gridftp-cmsstor333Domain 0 done
Stopping w-cmsstor333-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor333-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor333-disk-disk1Domain 0 1 2 done
Processing cmsstor334.fnal.gov
Stopping gridftp-cmsstor334Domain 0 done
Stopping w-cmsstor334-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor334-disk-disk2Domain 0 1 2 done
Stopping w-cmsstor334-disk-disk1Domain 0 1 2 3 done
Processing cmsstor335.fnal.gov
Stopping gridftp-cmsstor335Domain 0 1 done
Stopping w-cmsstor335-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor335-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor335-disk-disk1Domain 0 1 2 3 done
Processing cmsstor336.fnal.gov
Stopping gridftp-cmsstor336Domain 0 done
Stopping w-cmsstor336-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor336-disk-disk2Domain 0 1 2 done
Stopping w-cmsstor336-disk-disk1Domain 0 1 2 3 done
Processing cmsstor337.fnal.gov
Stopping gridftp-cmsstor337Domain 0 done
Stopping w-cmsstor337-disk-disk3Domain 0 1 2 done
Stopping w-cmsstor337-disk-disk2Domain 0 1 2 done
Stopping w-cmsstor337-disk-disk1Domain 0 1 2 3 done
Processing cmsstor338.fnal.gov
Stopping gridftp-cmsstor338Domain 0 1 done
Stopping w-cmsstor338-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor338-disk-disk2Domain 0 1 2 done
Stopping w-cmsstor338-disk-disk1Domain 0 1 2 3 done
Processing cmsstor339.fnal.gov
Stopping gridftp-cmsstor339Domain 0 done
Stopping w-cmsstor339-disk-disk3Domain 0 1 2 done
Stopping w-cmsstor339-disk-disk2Domain 0 1 2 done
Stopping w-cmsstor339-disk-disk1Domain 0 1 2 done
Processing cmsstor340.fnal.gov
Stopping gridftp-cmsstor340Domain 0 1 done
Stopping w-cmsstor340-disk-disk3Domain 0 1 2 done
Stopping w-cmsstor340-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor340-disk-disk1Domain 0 1 2 3 done
Processing cmsstor341.fnal.gov
Stopping gridftp-cmsstor341Domain 0 1 done
Stopping w-cmsstor341-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor341-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor341-disk-disk1Domain 0 1 2 3 done
Processing cmsstor342.fnal.gov
Stopping gridftp-cmsstor342Domain 0 done
Stopping w-cmsstor342-disk-disk3Domain 0 1 2 3 4 done
Stopping w-cmsstor342-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor342-disk-disk1Domain 0 1 2 3 done
Processing cmsstor343.fnal.gov
Stopping gridftp-cmsstor343Domain 0 done
Stopping w-cmsstor343-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor343-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor343-disk-disk1Domain 0 1 2 3 done
Processing cmsstor344.fnal.gov
Stopping gridftp-cmsstor344Domain 0 done
Stopping w-cmsstor344-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor344-disk-disk2Domain 0 1 2 done
Stopping w-cmsstor344-disk-disk1Domain 0 1 2 3 done
Processing cmsstor345.fnal.gov
Stopping gridftp-cmsstor345Domain 0 1 done
Stopping w-cmsstor345-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor345-disk-disk2Domain 0 1 2 3 4 done
Stopping w-cmsstor345-disk-disk1Domain 0 1 2 3 done
Processing cmsstor346.fnal.gov
Stopping gridftp-cmsstor346Domain 0 done
Stopping w-cmsstor346-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor346-disk-disk2Domain 0 1 2 done
Stopping w-cmsstor346-disk-disk1Domain 0 1 2 3 done
Processing cmsstor347.fnal.gov
Stopping gridftp-cmsstor347Domain 0 1 done
Stopping w-cmsstor347-disk-disk3Domain 0 1 2 3 4 done
Stopping w-cmsstor347-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor347-disk-disk1Domain 0 1 2 3 done
Processing cmsstor348.fnal.gov
Stopping gridftp-cmsstor348Domain 0 1 done
Stopping w-cmsstor348-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor348-disk-disk2Domain 0 1 2 done
Stopping w-cmsstor348-disk-disk1Domain 0 1 2 done
Processing cmsstor349.fnal.gov
Connection closed by 131.225.205.60
Stopping gridftp-cmsstor349Domain 0 done
Stopping w-cmsstor349-disk-disk3Domain 0 1 2 done
Stopping w-cmsstor349-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor349-disk-disk1Domain 0 1 2 done
Processing cmsstor350.fnal.gov
Stopping gridftp-cmsstor350Domain 0 1 done
Stopping w-cmsstor350-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor350-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor350-disk-disk1Domain 0 1 2 3 4 done
Processing cmsstor351.fnal.gov
Stopping gridftp-cmsstor351Domain 0 done
Stopping w-cmsstor351-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor351-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor351-disk-disk1Domain 0 1 2 3 done
Processing cmsstor352.fnal.gov
Stopping gridftp-cmsstor352Domain 0 1 done
Stopping w-cmsstor352-disk-disk3Domain 0 1 2 done
Stopping w-cmsstor352-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor352-disk-disk1Domain 0 1 2 3 done
Processing cmsstor353.fnal.gov
Stopping gridftp-cmsstor353Domain 0 done
Stopping w-cmsstor353-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor353-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor353-disk-disk1Domain 0 1 2 done
Processing cmsstor354.fnal.gov
Stopping gridftp-cmsstor354Domain 0 done
Stopping w-cmsstor354-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor354-disk-disk2Domain 0 1 2 done
Stopping w-cmsstor354-disk-disk1Domain 0 1 2 3 done
Processing cmsstor355.fnal.gov
Stopping gridftp-cmsstor355Domain 0 1 done
Stopping w-cmsstor355-disk-disk3Domain 0 1 2 done
Stopping w-cmsstor355-disk-disk2Domain 0 1 2 done
Stopping w-cmsstor355-disk-disk1Domain 0 1 2 3 done
Processing cmsstor356.fnal.gov
Stopping gridftp-cmsstor356Domain 0 done
Stopping w-cmsstor356-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor356-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor356-disk-disk1Domain 0 1 2 3 done
Processing cmsstor357.fnal.gov
Stopping gridftp-cmsstor357Domain 0 1 done
Stopping w-cmsstor357-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor357-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor357-disk-disk1Domain 0 1 2 3 done
Processing cmsstor358.fnal.gov
Stopping gridftp-cmsstor358Domain 0 1 done
Stopping w-cmsstor358-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor358-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor358-disk-disk1Domain 0 1 2 done
Processing cmsstor359.fnal.gov
Stopping gridftp-cmsstor359Domain 0 1 done
Stopping w-cmsstor359-disk-disk3Domain 0 1 2 done
Stopping w-cmsstor359-disk-disk2Domain 0 1 2 done
Stopping w-cmsstor359-disk-disk1Domain 0 1 2 3 done
Processing cmsstor360.fnal.gov
Stopping gridftp-cmsstor360Domain 0 1 done
Stopping w-cmsstor360-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor360-disk-disk2Domain 0 1 2 done
Stopping w-cmsstor360-disk-disk1Domain 0 1 2 3 done
Processing cmsstor361.fnal.gov
Stopping gridftp-cmsstor361Domain 0 1 done
Stopping w-cmsstor361-disk-disk3Domain 0 1 2 done
Stopping w-cmsstor361-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor361-disk-disk1Domain 0 1 2 done
Processing cmsstor362.fnal.gov
Stopping gridftp-cmsstor362Domain 0 done
Stopping w-cmsstor362-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor362-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor362-disk-disk1Domain 0 1 2 3 done
Processing cmsstor363.fnal.gov
Stopping gridftp-cmsstor363Domain 0 1 done
Stopping w-cmsstor363-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor363-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor363-disk-disk1Domain 0 1 2 3 done
Processing cmsstor364.fnal.gov
Stopping gridftp-cmsstor364Domain 0 done
Stopping w-cmsstor364-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor364-disk-disk2Domain 0 1 2 done
Stopping w-cmsstor364-disk-disk1Domain 0 1 2 3 done
Processing cmsstor365.fnal.gov
Stopping gridftp-cmsstor365Domain 0 done
Stopping w-cmsstor365-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor365-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor365-disk-disk1Domain 0 1 2 3 done
Processing cmsstor366.fnal.gov
Stopping gridftp-cmsstor366Domain 0 done
Stopping w-cmsstor366-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor366-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor366-disk-disk1Domain 0 1 2 3 done
Processing cmsstor368.fnal.gov
Stopping gridftp-cmsstor368Domain 0 1 done
Stopping w-cmsstor368-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor368-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor368-disk-disk1Domain 0 1 2 3 done
Processing cmsstor369.fnal.gov
Stopping gridftp-cmsstor369Domain 0 1 done
Stopping w-cmsstor369-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor369-disk-disk2Domain 0 1 2 done
Stopping w-cmsstor369-disk-disk1Domain 0 1 2 done
Processing cmsstor370.fnal.gov
Stopping gridftp-cmsstor370Domain 0 1 2 done
Stopping w-cmsstor370-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor370-disk-disk2Domain 0 1 2 done
Stopping w-cmsstor370-disk-disk1Domain 0 1 2 3 done
Processing cmsstor371.fnal.gov
Stopping gridftp-cmsstor371Domain 0 1 done
Stopping w-cmsstor371-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor371-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor371-disk-disk1Domain 0 1 2 done
Processing cmsstor372.fnal.gov
Stopping gridftp-cmsstor372Domain 0 1 done
Stopping w-cmsstor372-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor372-disk-disk2Domain 0 1 2 done
Stopping w-cmsstor372-disk-disk1Domain 0 1 2 3 done
Processing cmsstor373.fnal.gov
Stopping gridftp-cmsstor373Domain 0 done
Stopping w-cmsstor373-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor373-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor373-disk-disk1Domain 0 1 2 3 done
Processing cmsstor374.fnal.gov
Stopping gridftp-cmsstor374Domain 0 done
Stopping w-cmsstor374-disk-disk3Domain 0 1 2 done
Stopping w-cmsstor374-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor374-disk-disk1Domain 0 1 2 3 done
Processing cmsstor375.fnal.gov
Stopping gridftp-cmsstor375Domain 0 1 done
Stopping w-cmsstor375-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor375-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor375-disk-disk1Domain 0 1 2 3 done
Processing cmsstor376.fnal.gov
Stopping gridftp-cmsstor376Domain 0 1 done
Stopping w-cmsstor376-disk-disk3Domain 0 1 2 3 done
Stopping w-cmsstor376-disk-disk2Domain 0 1 2 done
Stopping w-cmsstor376-disk-disk1Domain 0 1 2 3 done
Processing cmsstor401.fnal.gov
Stopping gridftp-cmsstor401Domain 0 1 done
Stopping w-cmsstor401-disk-disk2Domain 0 1 2 done
Stopping w-cmsstor401-disk-disk1Domain 0 1 2 done
Processing cmsstor402.fnal.gov
Stopping gridftp-cmsstor402Domain 0 1 done
Stopping w-cmsstor402-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor402-disk-disk1Domain 0 1 2 3 done
Processing cmsstor403.fnal.gov
Stopping gridftp-cmsstor403Domain 0 1 done
Stopping w-cmsstor403-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor403-disk-disk1Domain 0 1 2 3 done
Processing cmsstor404.fnal.gov
Stopping gridftp-cmsstor404Domain 0 1 done
Stopping w-cmsstor404-disk-disk2Domain 0 1 2 done
Stopping w-cmsstor404-disk-disk1Domain 0 1 2 3 done
Processing cmsstor405.fnal.gov
Stopping gridftp-cmsstor405Domain 0 1 done
Stopping w-cmsstor405-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor405-disk-disk1Domain 0 1 2 done
Processing cmsstor406.fnal.gov
Stopping gridftp-cmsstor406Domain 0 1 done
Stopping w-cmsstor406-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor406-disk-disk1Domain 0 1 2 3 done
Processing cmsstor407.fnal.gov
Stopping gridftp-cmsstor407Domain 0 1 done
Stopping w-cmsstor407-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor407-disk-disk1Domain 0 1 2 3 done
Processing cmsstor408.fnal.gov
Stopping gridftp-cmsstor408Domain 0 1 done
Stopping w-cmsstor408-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor408-disk-disk1Domain 0 1 2 3 done
Processing cmsstor409.fnal.gov
Stopping gridftp-cmsstor409Domain 0 1 done
Stopping w-cmsstor409-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor409-disk-disk1Domain 0 1 2 done
Processing cmsstor410.fnal.gov
Stopping gridftp-cmsstor410Domain 0 1 done
Stopping w-cmsstor410-disk-disk2Domain 0 1 2 3 done
Stopping w-cmsstor410-disk-disk1Domain 0 1 2 done

real 65m15.657s
user 0m9.738s
sys 0m5.970s
[root@cmsadmin1 DOWNTIME-Aug-5-2015]#

#8 Updated by Natalia Ratnikova over 4 years ago

[root@cmsstor217 ~]# ls l /var/log/dcache/w-cmsstor217-disk-disk2Domain-oom.hprof
-rw------
1 root root 4263443655 Nov 18 2014 /var/log/dcache/w-cmsstor217-disk-disk2Domain-oom.hprof
[root@cmsstor217 ~]# rm /var/log/dcache/w-cmsstor217-disk-disk2Domain-oom.hprof
rm: remove regular file `/var/log/dcache/w-cmsstor217-disk-disk2Domain-oom.hprof'? y

Start rebooting on all cmsstor1xx nodes (25 total)

[root@cmsconsole NR]# pssh -p 25 -h cmsstor1xx.names -t 0 -i "yum clean all; yum update -y; reboot now"
[1] 13:39:30 [SUCCESS] cmsstor173
Loaded plugins: priorities, security
Cleaning repos: epel osg puppet puppet-deps slf slf-collections slf-security
: slf-source slf6x slf6x-security uscmst1
Cleaning up Everything
Loaded plugins: priorities, security
Setting up Update Process
7900 packages excluded due to repository priority protections
No Packages marked for Update
[2] 13:39:31 [SUCCESS] cmsstor189
Loaded plugins: priorities, security
Cleaning repos: epel osg puppet puppet-deps slf slf-collections slf-security
: slf-source slf6x slf6x-security uscmst1
Cleaning up Everything
Loaded plugins: priorities, security
Setting up Update Process
7900 packages excluded due to repository priority protections
No Packages marked for Update
[3] 13:39:31 [SUCCESS] cmsstor176
Loaded plugins: priorities, security
Cleaning repos: epel osg puppet puppet-deps slf slf-collections slf-security
: slf-source slf6x slf6x-security uscmst1
Cleaning up Everything
Loaded plugins: priorities, security
Setting up Update Process
7900 packages excluded due to repository priority protections
No Packages marked for Update
[4] 13:39:31 [SUCCESS] cmsstor175
Loaded plugins: priorities, security
Cleaning repos: epel osg puppet puppet-deps slf slf-collections slf-security
: slf-source slf6x slf6x-security uscmst1
Cleaning up Everything
Loaded plugins: priorities, security
Setting up Update Process
7900 packages excluded due to repository priority protections
No Packages marked for Update
[5] 13:39:31 [SUCCESS] cmsstor178
Loaded plugins: priorities, security
Cleaning repos: epel osg puppet puppet-deps slf slf-collections slf-security
: slf-source slf6x slf6x-security uscmst1
Cleaning up Everything
Loaded plugins: priorities, security
Setting up Update Process
7900 packages excluded due to repository priority protections
No Packages marked for Update
[6] 13:39:31 [SUCCESS] cmsstor183
Loaded plugins: priorities, security
Cleaning repos: epel osg puppet puppet-deps slf slf-collections slf-security
: slf-source slf6x slf6x-security uscmst1
Cleaning up Everything
Loaded plugins: priorities, security
Setting up Update Process
7900 packages excluded due to repository priority protections
No Packages marked for Update
[7] 13:39:31 [SUCCESS] cmsstor187
Loaded plugins: priorities, security
Cleaning repos: epel osg puppet puppet-deps slf slf-collections slf-security
: slf-source slf6x slf6x-security uscmst1
Cleaning up Everything
Loaded plugins: priorities, security
Setting up Update Process
7900 packages excluded due to repository priority protections
No Packages marked for Update
[8] 13:39:32 [SUCCESS] cmsstor181
Loaded plugins: priorities, security
Cleaning repos: epel osg puppet puppet-deps slf slf-collections slf-security
: slf-source slf6x slf6x-security uscmst1
Cleaning up Everything
Loaded plugins: priorities, security
Setting up Update Process
7900 packages excluded due to repository priority protections
No Packages marked for Update
[9] 13:39:32 [SUCCESS] cmsstor174
Loaded plugins: priorities, security
Cleaning repos: epel osg puppet puppet-deps slf slf-collections slf-security
: slf-source slf6x slf6x-security uscmst1
Cleaning up Everything
Loaded plugins: priorities, security
Setting up Update Process
7900 packages excluded due to repository priority protections
No Packages marked for Update
[10] 13:39:32 [SUCCESS] cmsstor179
Loaded plugins: priorities, security
Cleaning repos: epel osg puppet puppet-deps slf slf-collections slf-security
: slf-source slf6x slf6x-security uscmst1
Cleaning up Everything
Loaded plugins: priorities, security
Setting up Update Process
7900 packages excluded due to repository priority protections
No Packages marked for Update
[11] 13:39:32 [SUCCESS] cmsstor184
Loaded plugins: priorities, security
Cleaning repos: epel osg puppet puppet-deps slf slf-collections slf-security
: slf-source slf6x slf6x-security uscmst1
Cleaning up Everything
Loaded plugins: priorities, security
Setting up Update Process
7900 packages excluded due to repository priority protections
No Packages marked for Update
[12] 13:39:32 [SUCCESS] cmsstor194
Loaded plugins: priorities, security
Cleaning repos: epel osg puppet puppet-deps slf slf-collections slf-security
: slf-source slf6x slf6x-security uscmst1
Cleaning up Everything
Loaded plugins: priorities, security
Setting up Update Process
7900 packages excluded due to repository priority protections
No Packages marked for Update
[13] 13:39:32 [SUCCESS] cmsstor193
Loaded plugins: priorities, security
Cleaning repos: epel osg puppet puppet-deps slf slf-collections slf-security
: slf-source slf6x slf6x-security uscmst1
Cleaning up Everything
Loaded plugins: priorities, security
Setting up Update Process
7900 packages excluded due to repository priority protections
No Packages marked for Update
[14] 13:39:32 [SUCCESS] cmsstor188
Loaded plugins: priorities, security
Cleaning repos: epel osg puppet puppet-deps slf slf-collections slf-security
: slf-source slf6x slf6x-security uscmst1
Cleaning up Everything
Loaded plugins: priorities, security
Setting up Update Process
7900 packages excluded due to repository priority protections
No Packages marked for Update
[15] 13:39:32 [SUCCESS] cmsstor198
Loaded plugins: priorities, security
Cleaning repos: epel osg puppet puppet-deps slf slf-collections slf-security
: slf-source slf6x slf6x-security uscmst1
Cleaning up Everything
Loaded plugins: priorities, security
Setting up Update Process
7900 packages excluded due to repository priority protections
No Packages marked for Update
[16] 13:39:32 [SUCCESS] cmsstor177
Loaded plugins: priorities, security
Cleaning repos: epel osg puppet puppet-deps slf slf-collections slf-security
: slf-source slf6x slf6x-security uscmst1
Cleaning up Everything
Loaded plugins: priorities, security
Setting up Update Process
7900 packages excluded due to repository priority protections
No Packages marked for Update
[17] 13:39:32 [SUCCESS] cmsstor196
Loaded plugins: priorities, security
Cleaning repos: epel osg puppet puppet-deps slf slf-collections slf-security
: slf-source slf6x slf6x-security uscmst1
Cleaning up Everything
Loaded plugins: priorities, security
Setting up Update Process
7900 packages excluded due to repository priority protections
No Packages marked for Update
[18] 13:39:33 [SUCCESS] cmsstor197
Loaded plugins: priorities, security
Cleaning repos: epel osg puppet puppet-deps slf slf-collections slf-security
: slf-source slf6x slf6x-security uscmst1
Cleaning up Everything
Loaded plugins: priorities, security
Setting up Update Process
7900 packages excluded due to repository priority protections
No Packages marked for Update
[19] 13:39:33 [SUCCESS] cmsstor195
Loaded plugins: priorities, security
Cleaning repos: epel osg puppet puppet-deps slf slf-collections slf-security
: slf-source slf6x slf6x-security uscmst1
Cleaning up Everything
Loaded plugins: priorities, security
Setting up Update Process
7900 packages excluded due to repository priority protections
No Packages marked for Update
Stderr: Warning: No xauth data; using fake authentication data for X11 forwarding.
[20] 13:39:33 [SUCCESS] cmsstor186
Loaded plugins: priorities, security
Cleaning repos: epel osg puppet puppet-deps slf slf-collections slf-security
: slf-source slf6x slf6x-security uscmst1
Cleaning up Everything
Loaded plugins: priorities, security
Setting up Update Process
7900 packages excluded due to repository priority protections
No Packages marked for Update
Stderr: Warning: No xauth data; using fake authentication data for X11 forwarding.
[21] 13:39:33 [SUCCESS] cmsstor199
Loaded plugins: priorities, security
Cleaning repos: epel osg puppet puppet-deps slf slf-collections slf-security
: slf-source slf6x slf6x-security uscmst1
Cleaning up Everything
Loaded plugins: priorities, security
Setting up Update Process
7900 packages excluded due to repository priority protections
No Packages marked for Update
[22] 13:39:33 [SUCCESS] cmsstor182
Loaded plugins: priorities, security
Cleaning repos: epel osg puppet puppet-deps slf slf-collections slf-security
: slf-source slf6x slf6x-security uscmst1
Cleaning up Everything
Loaded plugins: priorities, security
Setting up Update Process
7900 packages excluded due to repository priority protections
No Packages marked for Update
[23] 13:39:34 [SUCCESS] cmsstor192
Loaded plugins: priorities, security
Cleaning repos: epel osg puppet puppet-deps slf slf-collections slf-security
: slf-source slf6x slf6x-security uscmst1
Cleaning up Everything
Loaded plugins: priorities, security
Setting up Update Process
7900 packages excluded due to repository priority protections
No Packages marked for Update
[24] 13:39:34 [SUCCESS] cmsstor185
Loaded plugins: priorities, security
Cleaning repos: epel osg puppet puppet-deps slf slf-collections slf-security
: slf-source slf6x slf6x-security uscmst1
Cleaning up Everything
Loaded plugins: priorities, security
Setting up Update Process
7900 packages excluded due to repository priority protections
No Packages marked for Update
[25] 13:39:34 [SUCCESS] cmsstor191
Loaded plugins: priorities, security
Cleaning repos: epel osg puppet puppet-deps slf slf-collections slf-security
: slf-source slf6x slf6x-security uscmst1
Cleaning up Everything
Loaded plugins: priorities, security
Setting up Update Process
7900 packages excluded due to repository priority protections
No Packages marked for Update
[root@cmsconsole NR]#

[root@cmsconsole NR]# for n in `cat cmsstor1xx.names`; do cmspower-cons $n; done
Connecting to node cmsstor173: ssh -x -t root:ttyS1@fcc-3-1061
Connecting to node cmsstor174: ssh -x -t root:ttyS2@fcc-3-1061
Connecting to node cmsstor175: ssh -x -t root:ttyS7@fcc-3-1062
Connecting to node cmsstor176: ssh -x -t root:ttyS6@fcc-3-1062
Connecting to node cmsstor177: ssh -x -t root:ttyS5@fcc-3-1062
Connecting to node cmsstor178: ssh -x -t root:ttyS4@fcc-3-1062
Connecting to node cmsstor179: ssh -x -t root:ttyS3@fcc-3-1062
Connecting to node cmsstor181: ssh -x -t root:ttyS1@fcc-3-1062
Connecting to node cmsstor182: ssh -x -t root:ttyS7@fcc-3-1063
Connecting to node cmsstor183: ssh -x -t root:ttyS6@fcc-3-1063
Connecting to node cmsstor184: ssh -x -t root:ttyS5@fcc-3-1063
Connecting to node cmsstor185: ssh -x -t root:ttyS4@fcc-3-1063
Connecting to node cmsstor186: ssh -x -t root:ttyS3@fcc-3-1063
Connecting to node cmsstor187: ssh -x -t root:ttyS2@fcc-3-1063
Connecting to node cmsstor188: ssh -x -t root:ttyS1@fcc-3-1063
Connecting to node cmsstor189: ssh -x -t root:ttyS7@fcc-3-1064
Connecting to node cmsstor191: ssh -x -t root:ttyS5@fcc-3-1064
Connecting to node cmsstor192: ssh -x -t root:ttyS4@fcc-3-1064
Connecting to node cmsstor193: ssh -x -t root:ttyS3@fcc-3-1064
Connecting to node cmsstor194: ssh -x -t root:ttyS2@fcc-3-1064
Connecting to node cmsstor195: ssh -x -t root:ttyS1@fcc-3-1064
Connecting to node cmsstor196: ssh -x -t root:ttyS7@fcc-3-1065
Connecting to node cmsstor197: ssh -x -t root:ttyS6@fcc-3-1065
Connecting to node cmsstor198: ssh -x -t root:ttyS5@fcc-3-1065
Connecting to node cmsstor199: ssh -x -t root:ttyS4@fcc-3-1065
[root@cmsconsole NR]#

[root@cmsconsole NR]# for n in `cat cmsstor1xx.names`; do echo ===== $n; ssh $n "uptime"; done ===== cmsstor173
13:46:54 up 3 min, 0 users, load average: 0.29, 0.31, 0.14 ===== cmsstor174
13:46:54 up 3 min, 0 users, load average: 0.10, 0.29, 0.15 ===== cmsstor175
13:46:55 up 3 min, 0 users, load average: 0.05, 0.16, 0.08 ===== cmsstor176
13:46:55 up 3 min, 0 users, load average: 0.22, 0.33, 0.16 ===== cmsstor177
13:46:56 up 3 min, 0 users, load average: 0.09, 0.24, 0.11 ===== cmsstor178
13:46:56 up 3 min, 0 users, load average: 0.28, 0.30, 0.14 ===== cmsstor179
13:46:57 up 3 min, 0 users, load average: 0.15, 0.35, 0.17 ===== cmsstor181
13:46:57 up 3 min, 0 users, load average: 0.07, 0.21, 0.10 ===== cmsstor182
13:46:58 up 3 min, 0 users, load average: 0.26, 0.30, 0.14 ===== cmsstor183
13:46:58 up 3 min, 0 users, load average: 0.12, 0.29, 0.14 ===== cmsstor184
13:46:59 up 2 min, 0 users, load average: 0.24, 0.20, 0.08 ===== cmsstor185
13:46:59 up 5 min, 0 users, load average: 0.01, 0.19, 0.12 ===== cmsstor186
ssh: connect to host cmsstor186 port 22: No route to host ===== cmsstor187
13:47:03 up 3 min, 0 users, load average: 0.16, 0.22, 0.09 ===== cmsstor188
13:47:03 up 6 min, 0 users, load average: 0.01, 0.16, 0.10 ===== cmsstor189
13:47:04 up 3 min, 0 users, load average: 0.08, 0.21, 0.10 ===== cmsstor191
13:47:04 up 3 min, 0 users, load average: 0.15, 0.25, 0.12 ===== cmsstor192
13:47:05 up 3 min, 0 users, load average: 0.18, 0.26, 0.12 ===== cmsstor193
13:47:06 up 3 min, 0 users, load average: 0.13, 0.34, 0.17 ===== cmsstor194
13:47:06 up 4 min, 0 users, load average: 0.10, 0.33, 0.16 ===== cmsstor195
13:47:07 up 4 min, 0 users, load average: 0.03, 0.21, 0.12 ===== cmsstor196
13:47:07 up 3 min, 0 users, load average: 0.10, 0.29, 0.15 ===== cmsstor197
13:47:08 up 3 min, 0 users, load average: 0.13, 0.22, 0.10 ===== cmsstor198
13:47:08 up 6 min, 0 users, load average: 0.04, 0.20, 0.13 ===== cmsstor199
13:47:09 up 3 min, 0 users, load average: 0.18, 0.40, 0.19
[root@cmsconsole NR]#

#9 Updated by Natalia Ratnikova over 4 years ago

[root@cmsdcacheadmindisk ~]# uptime
11:32:11 up 9 min, 2 users, load average: 0.00, 0.14, 0.13
[root@cmsdcacheadmindisk ~]# facter kernelrelease
2.6.32-504.30.3.el6.x86_64
[root@cmsdcacheadmindisk ~]# yum update
Loaded plugins: priorities, security
Setting up Update Process
582 packages excluded due to repository priority protections
No Packages marked for Update
[root@cmsdcacheadmindisk ~]# dcache status
DOMAIN STATUS PID USER
dirDomain stopped root
billingDomain stopped root
httpdDomain stopped root
adminDoorDomain stopped root
dCacheDomain stopped root
[root@cmsdcacheadmindisk ~]# dcache start
Starting dirDomain done
Starting billingDomain done
Starting httpdDomain done
Starting adminDoorDomain done
Starting dCacheDomain done
[root@cmsdcacheadmindisk ~]#

[root@cmssrmdisk ~]# facter kernelrelease
2.6.32-504.30.3.el6.x86_64
[root@cmssrmdisk ~]# yum update
Loaded plugins: priorities, security
Setting up Update Process
582 packages excluded due to repository priority protections
No Packages marked for Update
[root@cmssrmdisk ~]# dcache start
Starting nfsDomain.v3 done
Starting dcap-cmssrmdiskDomain done
Starting authdcap-cmssrmdiskDomain done
Starting gsidcap-cmssrmdiskDomain done
Starting xrootd-cmssrmdiskDomain done
Starting xrootdLFNs-cmssrmdiskDomain done
Starting utilityDomain done
Starting gPlazmaDomain done
Starting srm-cmssrmdiskDomain done
Starting transfermanagersDomain done
[root@cmssrmdisk ~]#

[root@cmschimeradisk ~]# uptime
11:39:58 up 1 min, 1 user, load average: 0.55, 0.30, 0.11
[root@cmschimeradisk ~]# facter kernelrelease
2.6.32-504.30.3.el6.x86_64
[root@cmschimeradisk ~]# yum update
Loaded plugins: priorities, security
Setting up Update Process
582 packages excluded due to repository priority protections
No Packages marked for Update
[root@cmschimeradisk ~]# dcache status
DOMAIN STATUS PID USER
namespaceDomain stopped root
nfsDomain stopped root
[root@cmschimeradisk ~]# dcache start
Starting namespaceDomain done
Starting nfsDomain done
[root@cmschimeradisk ~]# service postgresql-9.2 status
(pid 8144) is running...
[root@cmschimeradisk ~]# dcache status
DOMAIN STATUS PID USER
namespaceDomain running 3052 root
nfsDomain running 3149 root
[root@cmschimeradisk ~]#

#10 Updated by Natalia Ratnikova over 4 years ago

[root@cmsconsole NR]# for n in `cat cmsstor2xx.names`; do echo ===== $n; ssh $n "uptime"; done ===== cmsstor201
13:57:28 up 175 days, 2:34, 0 users, load average: 0.20, 0.07, 0.06 ===== cmsstor202
13:57:28 up 27 days, 22:10, 0 users, load average: 0.00, 0.00, 0.00 ===== cmsstor203
13:57:28 up 174 days, 23:52, 0 users, load average: 0.07, 0.03, 0.00 ===== cmsstor204
13:57:28 up 175 days, 2:33, 0 users, load average: 0.03, 0.02, 0.00 ===== cmsstor205
13:57:29 up 175 days, 2:34, 0 users, load average: 0.37, 0.13, 0.04 ===== cmsstor206
13:57:29 up 175 days, 2:36, 0 users, load average: 0.01, 0.01, 0.00 ===== cmsstor207
13:57:29 up 97 days, 20:09, 0 users, load average: 0.04, 0.03, 0.00 ===== cmsstor208
13:57:29 up 175 days, 2:34, 0 users, load average: 0.02, 0.01, 0.00 ===== cmsstor209
13:57:29 up 175 days, 2:34, 0 users, load average: 0.00, 0.03, 0.00 ===== cmsstor210
13:57:30 up 174 days, 23:54, 0 users, load average: 1.44, 1.87, 1.95 ===== cmsstor211
13:57:30 up 174 days, 23:52, 0 users, load average: 2.06, 2.02, 2.00 ===== cmsstor212
13:57:30 up 174 days, 23:51, 0 users, load average: 2.04, 2.03, 2.00 ===== cmsstor213
13:57:30 up 174 days, 23:53, 0 users, load average: 2.02, 2.01, 2.00 ===== cmsstor214
13:57:30 up 174 days, 21:43, 0 users, load average: 1.71, 1.96, 1.98 ===== cmsstor215
13:57:31 up 174 days, 23:51, 0 users, load average: 1.54, 1.78, 1.91 ===== cmsstor216
13:57:31 up 174 days, 23:53, 0 users, load average: 1.54, 1.86, 1.95 ===== cmsstor217
13:57:31 up 174 days, 23:50, 0 users, load average: 2.00, 2.00, 2.00 ===== cmsstor218
13:57:32 up 174 days, 23:50, 0 users, load average: 1.25, 1.69, 1.88 ===== cmsstor219
13:57:32 up 174 days, 23:50, 0 users, load average: 0.42, 1.38, 1.80 ===== cmsstor220
13:57:32 up 174 days, 23:50, 0 users, load average: 2.00, 2.00, 2.00 ===== cmsstor221
13:57:32 up 174 days, 23:50, 0 users, load average: 1.68, 1.93, 1.97 ===== cmsstor222
13:57:32 up 174 days, 23:49, 0 users, load average: 1.67, 1.93, 1.97 ===== cmsstor223
13:57:33 up 174 days, 21:39, 0 users, load average: 1.44, 1.86, 1.99 ===== cmsstor224
13:57:33 up 174 days, 21:43, 0 users, load average: 0.02, 0.01, 0.00 ===== cmsstor225
13:57:33 up 79 days, 5:05, 0 users, load average: 0.00, 0.00, 0.00 ===== cmsstor226
13:57:33 up 174 days, 23:48, 0 users, load average: 1.61, 1.90, 1.97 ===== cmsstor227
13:57:33 up 174 days, 23:48, 0 users, load average: 0.00, 0.02, 0.00 ===== cmsstor228
13:57:34 up 174 days, 23:48, 0 users, load average: 1.27, 1.70, 1.88 ===== cmsstor229
13:57:34 up 174 days, 21:43, 0 users, load average: 1.72, 1.94, 1.98 ===== cmsstor230
13:57:34 up 174 days, 23:48, 0 users, load average: 2.00, 2.00, 2.00 ===== cmsstor231
13:57:34 up 174 days, 21:43, 0 users, load average: 0.06, 0.06, 0.01 ===== cmsstor232
13:57:35 up 174 days, 21:42, 0 users, load average: 1.78, 1.97, 1.98 ===== cmsstor233
13:57:35 up 174 days, 23:47, 0 users, load average: 0.85, 1.60, 1.84 ===== cmsstor234
13:57:35 up 174 days, 23:47, 0 users, load average: 1.60, 1.21, 1.06 ===== cmsstor235
13:57:35 up 174 days, 23:47, 0 users, load average: 2.01, 2.02, 2.00 ===== cmsstor236
13:57:36 up 174 days, 21:43, 0 users, load average: 1.74, 1.98, 1.98 ===== cmsstor237
13:57:36 up 174 days, 21:43, 0 users, load average: 1.55, 1.91, 1.96 ===== cmsstor238
13:57:36 up 174 days, 23:46, 0 users, load average: 1.98, 2.00, 1.99 ===== cmsstor239
13:57:36 up 174 days, 23:46, 0 users, load average: 1.43, 1.84, 1.95 ===== cmsstor240
13:57:36 up 154 days, 21 min, 0 users, load average: 1.27, 1.78, 1.91 ===== cmsstor241
13:57:37 up 99 days, 16 min, 0 users, load average: 0.01, 0.02, 0.00 ===== cmsstor242
13:57:37 up 99 days, 21:53, 1 user, load average: 0.00, 0.00, 0.00 ===== cmsstor243
13:57:37 up 174 days, 23:45, 0 users, load average: 1.57, 1.88, 1.96 ===== cmsstor244
13:57:37 up 174 days, 23:45, 0 users, load average: 2.06, 2.01, 2.00 ===== cmsstor245
13:57:37 up 174 days, 21:42, 0 users, load average: 1.63, 1.94, 1.97 ===== cmsstor246
13:57:38 up 174 days, 23:45, 0 users, load average: 2.08, 2.05, 2.00 ===== cmsstor247
13:57:38 up 174 days, 23:44, 0 users, load average: 0.05, 0.05, 0.01 ===== cmsstor248
13:57:38 up 174 days, 23:44, 0 users, load average: 2.01, 2.03, 2.00 ===== cmsstor249
13:57:38 up 174 days, 23:44, 0 users, load average: 0.00, 0.00, 0.00 ===== cmsstor250
13:57:38 up 174 days, 21:42, 0 users, load average: 1.32, 1.80, 1.93 ===== cmsstor251
13:57:39 up 174 days, 21:41, 0 users, load average: 1.84, 1.97, 1.99 ===== cmsstor261
13:57:39 up 174 days, 23:42, 0 users, load average: 0.00, 0.00, 0.00 ===== cmsstor262
13:57:39 up 175 days, 2:34, 0 users, load average: 0.04, 0.08, 0.01 ===== cmsstor263
13:57:39 up 175 days, 2:34, 0 users, load average: 0.13, 0.05, 0.02 ===== cmsstor264
13:57:39 up 175 days, 2:34, 0 users, load average: 0.00, 0.00, 0.00 ===== cmsstor265
13:57:40 up 48 days, 4:27, 0 users, load average: 0.00, 0.02, 0.00 ===== cmsstor266
13:57:40 up 174 days, 23:42, 0 users, load average: 0.05, 0.01, 0.00 ===== cmsstor267
13:57:40 up 48 days, 2:03, 0 users, load average: 0.02, 0.03, 0.02 ===== cmsstor268
13:57:40 up 175 days, 2:34, 0 users, load average: 0.00, 0.02, 0.00 ===== cmsstor269
13:57:40 up 152 days, 23:05, 0 users, load average: 0.07, 0.04, 0.00 ===== cmsstor270
13:57:41 up 28 days, 6:38, 0 users, load average: 0.08, 0.06, 0.07 ===== cmsstor271
13:57:41 up 66 days, 5:24, 0 users, load average: 0.10, 0.04, 0.01 ===== cmsstor272
13:57:41 up 174 days, 23:42, 0 users, load average: 0.01, 0.03, 0.00 ===== cmsstor273
13:57:41 up 174 days, 23:41, 0 users, load average: 0.01, 0.01, 0.00 ===== cmsstor274
13:57:41 up 174 days, 23:41, 0 users, load average: 0.04, 0.03, 0.00 ===== cmsstor275
13:57:41 up 140 days, 4:10, 0 users, load average: 0.02, 0.03, 0.02 ===== cmsstor276
13:57:42 up 62 days, 2:37, 0 users, load average: 0.01, 0.02, 0.02 ===== cmsstor277
13:57:42 up 2 days, 16:19, 0 users, load average: 0.00, 0.00, 0.00 ===== cmsstor278
13:57:42 up 174 days, 23:40, 0 users, load average: 0.00, 0.02, 0.00 ===== cmsstor279
13:57:42 up 57 days, 4:02, 0 users, load average: 0.11, 0.11, 0.09 ===== cmsstor280
13:57:42 up 174 days, 23:40, 0 users, load average: 0.02, 0.05, 0.05 ===== cmsstor281
13:57:43 up 174 days, 23:39, 0 users, load average: 0.04, 0.02, 0.00 ===== cmsstor282
13:57:43 up 174 days, 23:40, 0 users, load average: 0.13, 0.09, 0.04 ===== cmsstor283
13:57:43 up 18 days, 12:19, 0 users, load average: 0.01, 0.03, 0.00 ===== cmsstor284
13:57:43 up 27 days, 14:20, 0 users, load average: 0.18, 0.14, 0.08 ===== cmsstor285
13:57:43 up 27 days, 3:50, 0 users, load average: 0.08, 0.02, 0.01 ===== cmsstor286
13:57:43 up 43 days, 23:47, 0 users, load average: 0.10, 0.06, 0.00 ===== cmsstor287
13:57:44 up 125 days, 23:36, 0 users, load average: 0.01, 0.02, 0.00 ===== cmsstor288
13:57:44 up 111 days, 23:59, 0 users, load average: 0.38, 0.17, 0.11 ===== cmsstor289
13:57:44 up 174 days, 23:38, 0 users, load average: 0.18, 0.18, 0.12 ===== cmsstor290
13:57:44 up 47 days, 3:36, 0 users, load average: 0.04, 0.03, 0.00 ===== cmsstor291
13:57:44 up 174 days, 23:38, 0 users, load average: 0.10, 0.11, 0.04 ===== cmsstor292
13:57:45 up 174 days, 23:38, 0 users, load average: 0.03, 0.05, 0.00 ===== cmsstor293
13:57:45 up 174 days, 23:37, 0 users, load average: 0.02, 0.06, 0.03 ===== cmsstor294
13:57:45 up 174 days, 23:38, 0 users, load average: 0.02, 0.05, 0.02
[root@cmsconsole NR]# wc l cmsstor2xx.names
85 cmsstor2xx.names
[root@cmsconsole NR]# date; pssh -p 30 -h cmsstor2xx.names -t 0 -i "yum clean all; yum update -y; reboot now" 2>&1 | tee cmsstor2xx.upgrades.log; date
Wed Aug 5 13:58:56 CDT 2015
[1] 14:00:09 [SUCCESS] cmsstor201
Loaded plugins: priorities, security
Cleaning repos: epel osg puppet puppet-deps slf slf-collections slf-security
: slf-source slf6x slf6x-security uscmst1
Cleaning up Everything
Loaded plugins: priorities, security
Setting up Update Process
7900 packages excluded due to repository priority protections
Resolving Dependencies
-
> Running transaction check
---> Package kernel.x86_64 0:2.6.32-504.30.3.el6 will be installed
---> Package kernel-firmware.noarch 0:2.6.32-504.23.4.el6 will be updated
---> Package kernel-firmware.noarch 0:2.6.32-504.30.3.el6 will be an update
---> Package kernel-headers.x86_64 0:2.6.32-504.8.1.el6 will be updated
---> Package kernel-headers.x86_64 0:2.6.32-504.30.3.el6 will be an update
--> Finished Dependency Resolution
--> Running transaction check
---> Package kernel.x86_64 0:2.6.32-504.1.3.el6 will be erased
--> Finished Dependency Resolution

Dependencies Resolved

================================================================================
Package Arch Version Repository Size ================================================================================
Installing:
kernel x86_64 2.6.32-504.30.3.el6 slf-security 29 M
Updating:
kernel-firmware noarch 2.6.32-504.30.3.el6 slf-security 14 M
kernel-headers x86_64 2.6.32-504.30.3.el6 slf-security 3.4 M
Removing:
kernel x86_64 2.6.32-504.1.3.el6 @anaconda-SLFSecurityUpdates-201402061501.x86_64/6
124 M

Transaction Summary ================================================================================
Install 1 Package(s)
Upgrade 2 Package(s)
Remove 1 Package(s)

.... continuous output all goes into the log and onto the screen ....

Replaced:
golang-vim.noarch 0:1.2.2-7.el6

Complete!
Wed Aug 5 14:08:10 CDT 2015

#11 Updated by Natalia Ratnikova over 4 years ago

[root@cmsconsole NR]# for n in `cat cmsstor3xx.names`; do echo ===== $n; ssh $n "uptime"; done ===== cmsstor309
14:09:20 up 175 days, 2:47, 0 users, load average: 0.09, 0.04, 0.01 ===== cmsstor310
14:09:21 up 175 days, 2:47, 0 users, load average: 0.00, 0.00, 0.00 ===== cmsstor311
14:09:21 up 175 days, 2:47, 0 users, load average: 0.00, 0.00, 0.00 ===== cmsstor312
14:09:21 up 175 days, 2:47, 0 users, load average: 0.00, 0.00, 0.00 ===== cmsstor313
14:09:21 up 174 days, 23:50, 0 users, load average: 0.02, 0.01, 0.00 ===== cmsstor314
14:09:21 up 175 days, 2:47, 0 users, load average: 0.00, 0.01, 0.00 ===== cmsstor315
14:09:22 up 175 days, 2:47, 0 users, load average: 0.18, 0.06, 0.01 ===== cmsstor316
14:09:22 up 175 days, 2:47, 0 users, load average: 0.00, 0.00, 0.00 ===== cmsstor317
14:09:22 up 156 days, 1:28, 0 users, load average: 0.13, 0.32, 0.29 ===== cmsstor318
14:09:22 up 175 days, 2:47, 0 users, load average: 0.03, 0.02, 0.00 ===== cmsstor319
14:09:22 up 175 days, 2:47, 0 users, load average: 0.00, 0.00, 0.00 ===== cmsstor320
14:09:22 up 34 days, 20 min, 0 users, load average: 0.00, 0.00, 0.00 ===== cmsstor321
14:09:23 up 175 days, 2:47, 0 users, load average: 0.05, 0.03, 0.00 ===== cmsstor322
14:09:23 up 175 days, 2:46, 0 users, load average: 0.00, 0.00, 0.00 ===== cmsstor323
14:09:23 up 9 days, 2:19, 0 users, load average: 0.00, 0.01, 0.00 ===== cmsstor324
14:09:23 up 9 days, 1:22, 0 users, load average: 0.01, 0.02, 0.00 ===== cmsstor325
14:09:23 up 9 days, 1:08, 0 users, load average: 0.00, 0.03, 0.00 ===== cmsstor326
14:09:24 up 9 days, 58 min, 0 users, load average: 0.28, 0.08, 0.02 ===== cmsstor327
14:09:24 up 9 days, 49 min, 0 users, load average: 0.16, 0.03, 0.01 ===== cmsstor328
14:09:24 up 9 days, 44 min, 0 users, load average: 0.03, 0.02, 0.00 ===== cmsstor329
14:09:24 up 9 days, 30 min, 0 users, load average: 0.00, 0.00, 0.00 ===== cmsstor330
14:09:24 up 9 days, 20 min, 0 users, load average: 0.14, 0.10, 0.03 ===== cmsstor331
14:09:25 up 9 days, 10 min, 0 users, load average: 0.03, 0.05, 0.00 ===== cmsstor332
14:09:25 up 9 days, 1 min, 0 users, load average: 0.00, 0.00, 0.00 ===== cmsstor333
14:09:25 up 8 days, 23:51, 0 users, load average: 0.00, 0.02, 0.05 ===== cmsstor334
14:09:25 up 8 days, 23:43, 0 users, load average: 0.10, 0.07, 0.01 ===== cmsstor335
14:09:25 up 8 days, 23:34, 0 users, load average: 0.09, 0.11, 0.04 ===== cmsstor336
14:09:25 up 8 days, 23:27, 0 users, load average: 0.08, 0.02, 0.01 ===== cmsstor337
14:09:26 up 8 days, 23:18, 0 users, load average: 0.00, 0.02, 0.04 ===== cmsstor338
14:09:26 up 8 days, 22:57, 0 users, load average: 0.17, 0.12, 0.04 ===== cmsstor339
14:09:26 up 8 days, 22:52, 0 users, load average: 0.10, 0.06, 0.02 ===== cmsstor340
14:09:26 up 8 days, 22:46, 0 users, load average: 0.00, 0.00, 0.00 ===== cmsstor341
14:09:27 up 11 days, 22:06, 0 users, load average: 0.18, 0.08, 0.02 ===== cmsstor342
14:09:27 up 8 days, 22:36, 0 users, load average: 0.03, 0.04, 0.00 ===== cmsstor343
14:09:27 up 154 days, 2:45, 0 users, load average: 0.07, 0.02, 0.00 ===== cmsstor344
14:09:27 up 175 days, 2:45, 0 users, load average: 0.00, 0.02, 0.00 ===== cmsstor345
14:09:27 up 175 days, 2:45, 0 users, load average: 0.00, 0.00, 0.00 ===== cmsstor346
14:09:27 up 175 days, 2:45, 0 users, load average: 0.08, 0.05, 0.01 ===== cmsstor347
14:09:28 up 135 days, 4:46, 0 users, load average: 0.09, 0.08, 0.08 ===== cmsstor348
14:09:28 up 166 days, 22:26, 0 users, load average: 0.03, 0.06, 0.06 ===== cmsstor349
14:09:28 up 5 days, 23:54, 0 users, load average: 0.18, 0.20, 0.17 ===== cmsstor350
14:09:28 up 175 days, 2:46, 0 users, load average: 0.00, 0.02, 0.00 ===== cmsstor351
14:09:28 up 160 days, 3:51, 0 users, load average: 0.02, 0.01, 0.00 ===== cmsstor352
14:09:29 up 175 days, 2:46, 0 users, load average: 0.01, 0.04, 0.05 ===== cmsstor353
14:09:29 up 29 days, 3:41, 0 users, load average: 0.12, 0.07, 0.01 ===== cmsstor354
14:09:29 up 175 days, 2:45, 0 users, load average: 0.15, 0.03, 0.01 ===== cmsstor355
14:09:29 up 175 days, 2:46, 0 users, load average: 0.02, 0.04, 0.00 ===== cmsstor356
14:09:29 up 62 days, 3:12, 0 users, load average: 0.00, 0.00, 0.00 ===== cmsstor357
14:09:30 up 175 days, 2:45, 0 users, load average: 0.23, 0.13, 0.08 ===== cmsstor358
14:09:30 up 132 days, 7:29, 0 users, load average: 0.02, 0.04, 0.00 ===== cmsstor359
14:09:30 up 175 days, 2:45, 0 users, load average: 0.00, 0.00, 0.00 ===== cmsstor360
14:09:30 up 55 days, 3:48, 0 users, load average: 0.02, 0.03, 0.01 ===== cmsstor361
14:09:30 up 175 days, 2:45, 0 users, load average: 0.02, 0.02, 0.00 ===== cmsstor362
14:09:31 up 175 days, 2:46, 0 users, load average: 0.27, 0.12, 0.03 ===== cmsstor363
14:09:31 up 85 days, 25 min, 0 users, load average: 0.07, 0.06, 0.05 ===== cmsstor364
14:09:31 up 70 days, 4:35, 0 users, load average: 0.05, 0.03, 0.00 ===== cmsstor365
14:09:31 up 161 days, 21:19, 0 users, load average: 0.03, 0.05, 0.00 ===== cmsstor366
14:09:32 up 86 days, 4:09, 0 users, load average: 0.02, 0.03, 0.00 ===== cmsstor368
14:09:32 up 175 days, 2:46, 0 users, load average: 0.04, 0.05, 0.00 ===== cmsstor369
14:09:32 up 174 days, 23:50, 0 users, load average: 0.06, 0.26, 0.25 ===== cmsstor370
14:09:32 up 174 days, 23:50, 0 users, load average: 0.05, 0.03, 0.02 ===== cmsstor371
14:09:32 up 175 days, 2:46, 0 users, load average: 0.02, 0.03, 0.00 ===== cmsstor372
14:09:33 up 175 days, 2:46, 0 users, load average: 0.10, 0.06, 0.01 ===== cmsstor373
14:09:33 up 139 days, 21:23, 0 users, load average: 0.24, 0.07, 0.06 ===== cmsstor374
14:09:33 up 26 days, 21:30, 0 users, load average: 0.00, 0.00, 0.00 ===== cmsstor375
14:09:33 up 149 days, 3:14, 0 users, load average: 0.07, 0.03, 0.00 ===== cmsstor376
14:09:34 up 175 days, 2:46, 0 users, load average: 0.13, 0.06, 0.02
[root@cmsconsole NR]# wc -l cmsstor3xx.names
67 cmsstor3xx.names
[root@cmsconsole NR]# date; pssh -p 30 -h cmsstor3xx.names -t 0 -i "yum clean all; yum update -y; reboot now" 2>&1 | tee cmsstor2xx.
cmsstor2xx.names cmsstor2xx.upgrades.log
[root@cmsconsole NR]# date; pssh -p 30 -h cmsstor3xx.names -t 0 -i "yum clean all; yum update -y; reboot now" 2>&1 | tee cmsstor3xx.upgrades.log ; date
Wed Aug 5 14:10:32 CDT 2015

[.....output in the log ... ]

kernel-firmware.noarch 0:2.6.32-504.30.3.el6                                  
kernel-headers.x86_64 0:2.6.32-504.30.3.el6

Complete!
Stderr: Warning: RPMDB altered outside of yum.
Wed Aug 5 14:14:34 CDT 2015

CHeck all 2xx and 3xx nodes are up:
[root@cmsconsole NR]# for n in `cat cmsstor2xx.names`; do echo ===== $n; ssh $n "uptime"; done ===== cmsstor201
14:13:35 up 10 min, 0 users, load average: 0.15, 0.15, 0.10 ===== cmsstor202
14:13:36 up 8 min, 0 users, load average: 0.24, 0.29, 0.18 ===== cmsstor203
14:13:37 up 9 min, 0 users, load average: 0.32, 0.30, 0.18 ===== cmsstor204
14:13:38 up 9 min, 0 users, load average: 0.14, 0.26, 0.18 ===== cmsstor205
14:13:39 up 9 min, 0 users, load average: 0.06, 0.15, 0.11 ===== cmsstor206
14:13:40 up 11 min, 0 users, load average: 0.12, 0.19, 0.13 ===== cmsstor207
14:13:41 up 9 min, 0 users, load average: 0.04, 0.16, 0.13 ===== cmsstor208
14:13:42 up 9 min, 0 users, load average: 0.27, 0.27, 0.18 ===== cmsstor209
14:13:42 up 9 min, 0 users, load average: 0.15, 0.21, 0.16 ===== cmsstor210
14:13:43 up 10 min, 0 users, load average: 0.19, 0.20, 0.15 ===== cmsstor211
14:13:44 up 2 min, 0 users, load average: 1.25, 0.62, 0.23 ===== cmsstor212
14:13:44 up 6 min, 0 users, load average: 0.13, 0.24, 0.12 ===== cmsstor213
14:13:45 up 8 min, 0 users, load average: 0.04, 0.15, 0.11 ===== cmsstor214
14:13:46 up 10 min, 0 users, load average: 0.20, 0.22, 0.16 ===== cmsstor215
14:13:46 up 7 min, 0 users, load average: 0.01, 0.12, 0.08 ===== cmsstor216
14:13:47 up 10 min, 0 users, load average: 0.16, 0.25, 0.18 ===== cmsstor217
14:13:48 up 6 min, 0 users, load average: 0.07, 0.22, 0.13 ===== cmsstor218
14:13:49 up 1 min, 0 users, load average: 1.66, 0.57, 0.20 ===== cmsstor219
14:13:50 up 8 min, 0 users, load average: 0.20, 0.23, 0.15 ===== cmsstor220
14:13:50 up 6 min, 0 users, load average: 0.03, 0.18, 0.10 ===== cmsstor221
14:13:51 up 7 min, 0 users, load average: 0.04, 0.17, 0.10 ===== cmsstor222
14:13:52 up 7 min, 0 users, load average: 0.15, 0.20, 0.12 ===== cmsstor223
14:13:52 up 8 min, 0 users, load average: 0.01, 0.14, 0.12 ===== cmsstor224
14:13:53 up 12 min, 0 users, load average: 0.16, 0.14, 0.12 ===== cmsstor225
14:13:54 up 9 min, 0 users, load average: 0.01, 0.15, 0.13 ===== cmsstor226
14:13:55 up 7 min, 0 users, load average: 0.08, 0.20, 0.12 ===== cmsstor227
14:13:55 up 10 min, 0 users, load average: 0.17, 0.18, 0.14 ===== cmsstor228
14:13:57 up 7 min, 0 users, load average: 0.07, 0.24, 0.16 ===== cmsstor229
14:13:57 up 9 min, 0 users, load average: 0.23, 0.22, 0.16 ===== cmsstor230
14:13:58 up 6 min, 0 users, load average: 0.04, 0.16, 0.10 ===== cmsstor231
14:13:59 up 11 min, 0 users, load average: 0.07, 0.12, 0.10 ===== cmsstor232
14:14:00 up 9 min, 0 users, load average: 0.07, 0.19, 0.15 ===== cmsstor233
14:14:00 up 8 min, 0 users, load average: 0.07, 0.21, 0.16 ===== cmsstor234
14:14:01 up 6 min, 0 users, load average: 0.18, 0.35, 0.20 ===== cmsstor235
14:14:01 up 4 min, 0 users, load average: 0.09, 0.30, 0.15 ===== cmsstor236
14:14:02 up 9 min, 0 users, load average: 0.05, 0.16, 0.12 ===== cmsstor237
14:14:03 up 5 min, 0 users, load average: 0.05, 0.30, 0.18 ===== cmsstor238
14:14:04 up 6 min, 0 users, load average: 0.09, 0.23, 0.14 ===== cmsstor239
14:14:04 up 6 min, 0 users, load average: 0.04, 0.31, 0.20 ===== cmsstor240
14:14:05 up 6 min, 0 users, load average: 0.04, 0.24, 0.15 ===== cmsstor241
14:14:06 up 7 min, 0 users, load average: 0.18, 0.29, 0.18 ===== cmsstor242
14:14:06 up 9 min, 0 users, load average: 0.12, 0.14, 0.09 ===== cmsstor243
14:14:07 up 5 min, 0 users, load average: 0.08, 0.25, 0.15 ===== cmsstor244
14:14:07 up 5 min, 0 users, load average: 0.16, 0.30, 0.17 ===== cmsstor245
14:14:08 up 7 min, 0 users, load average: 0.04, 0.18, 0.13 ===== cmsstor246
14:14:08 up 5 min, 0 users, load average: 0.13, 0.28, 0.16 ===== cmsstor247
14:14:09 up 6 min, 0 users, load average: 0.09, 0.22, 0.15 ===== cmsstor248
14:14:09 up 4 min, 0 users, load average: 0.15, 0.18, 0.09 ===== cmsstor249
14:14:10 up 6 min, 0 users, load average: 0.38, 0.36, 0.20 ===== cmsstor250
14:14:10 up 7 min, 0 users, load average: 0.06, 0.17, 0.11 ===== cmsstor251
14:14:11 up 7 min, 0 users, load average: 0.06, 0.18, 0.12 ===== cmsstor261
14:14:12 up 7 min, 0 users, load average: 0.37, 0.58, 0.35 ===== cmsstor262
14:14:12 up 6 min, 0 users, load average: 0.21, 0.41, 0.25 ===== cmsstor263
14:14:13 up 7 min, 0 users, load average: 0.33, 0.48, 0.29 ===== cmsstor264
14:14:13 up 6 min, 0 users, load average: 0.31, 0.47, 0.27 ===== cmsstor265
14:14:13 up 6 min, 0 users, load average: 0.53, 0.61, 0.33 ===== cmsstor266
14:14:14 up 7 min, 0 users, load average: 0.43, 0.64, 0.37 ===== cmsstor267
14:14:14 up 7 min, 0 users, load average: 0.35, 0.64, 0.38 ===== cmsstor268
14:14:15 up 6 min, 0 users, load average: 0.82, 0.85, 0.43 ===== cmsstor269
14:14:15 up 6 min, 0 users, load average: 0.07, 0.34, 0.21 ===== cmsstor270
14:14:16 up 6 min, 0 users, load average: 0.60, 0.71, 0.39 ===== cmsstor271
14:14:16 up 6 min, 0 users, load average: 0.67, 0.81, 0.44 ===== cmsstor272
14:14:17 up 7 min, 0 users, load average: 0.27, 0.50, 0.30 ===== cmsstor273
14:14:17 up 6 min, 0 users, load average: 0.52, 0.58, 0.31 ===== cmsstor274
14:14:17 up 6 min, 0 users, load average: 0.42, 0.49, 0.27 ===== cmsstor275
14:14:18 up 7 min, 0 users, load average: 0.76, 0.73, 0.39 ===== cmsstor276
14:14:18 up 6 min, 0 users, load average: 0.18, 0.42, 0.26 ===== cmsstor277
14:14:19 up 6 min, 0 users, load average: 0.49, 0.57, 0.31 ===== cmsstor278
14:14:19 up 6 min, 0 users, load average: 0.31, 0.41, 0.23 ===== cmsstor279
14:14:19 up 5 min, 0 users, load average: 0.47, 0.53, 0.27 ===== cmsstor280
14:14:20 up 6 min, 0 users, load average: 0.34, 0.79, 0.45 ===== cmsstor281
14:14:20 up 6 min, 0 users, load average: 1.02, 0.85, 0.42 ===== cmsstor282
14:14:21 up 6 min, 0 users, load average: 0.57, 0.57, 0.31 ===== cmsstor283
14:14:21 up 6 min, 0 users, load average: 0.32, 0.45, 0.25 ===== cmsstor284
14:14:22 up 6 min, 0 users, load average: 0.44, 0.55, 0.31 ===== cmsstor285
14:14:22 up 6 min, 0 users, load average: 0.49, 0.59, 0.35 ===== cmsstor286
14:14:22 up 7 min, 0 users, load average: 0.35, 0.58, 0.35 ===== cmsstor287
14:14:23 up 6 min, 0 users, load average: 0.80, 0.66, 0.35 ===== cmsstor288
14:14:23 up 6 min, 0 users, load average: 0.20, 0.28, 0.17 ===== cmsstor289
14:14:24 up 6 min, 0 users, load average: 0.26, 0.34, 0.19 ===== cmsstor290
14:14:24 up 6 min, 0 users, load average: 0.45, 0.62, 0.34 ===== cmsstor291
14:14:25 up 5 min, 0 users, load average: 0.48, 0.66, 0.35 ===== cmsstor292
14:14:25 up 6 min, 0 users, load average: 0.30, 0.30, 0.18 ===== cmsstor293
14:14:26 up 6 min, 0 users, load average: 0.40, 0.69, 0.36 ===== cmsstor294
14:14:26 up 6 min, 0 users, load average: 0.71, 0.72, 0.38
[root@cmsconsole NR]#

[root@cmsconsole NR]# for n in `cat cmsstor3xx.names`; do echo ===== $n; ssh $n "uptime"; done ===== cmsstor309
14:16:10 up 1 min, 0 users, load average: 1.06, 0.40, 0.14 ===== cmsstor310
14:16:10 up 1 min, 0 users, load average: 1.80, 0.63, 0.22 ===== cmsstor311
14:16:11 up 1 min, 0 users, load average: 1.22, 0.44, 0.16 ===== cmsstor312
14:16:11 up 1 min, 0 users, load average: 1.05, 0.39, 0.14 ===== cmsstor313
14:16:12 up 1 min, 0 users, load average: 1.01, 0.38, 0.13 ===== cmsstor314
14:16:12 up 1 min, 0 users, load average: 0.65, 0.26, 0.09 ===== cmsstor315
14:16:13 up 1 min, 0 users, load average: 1.37, 0.52, 0.18 ===== cmsstor316
14:16:14 up 1 min, 0 users, load average: 1.44, 0.49, 0.17 ===== cmsstor317
ssh: connect to host cmsstor317 port 22: Connection timed out ===== cmsstor318
14:17:17 up 2 min, 0 users, load average: 0.48, 0.33, 0.13 ===== cmsstor319
14:17:17 up 2 min, 0 users, load average: 0.46, 0.39, 0.16 ===== cmsstor320
14:17:18 up 2 min, 0 users, load average: 0.45, 0.32, 0.12 ===== cmsstor321
14:17:18 up 2 min, 0 users, load average: 0.68, 0.55, 0.22 ===== cmsstor322
14:17:19 up 1 min, 0 users, load average: 1.29, 0.59, 0.21 ===== cmsstor323
14:17:19 up 3 min, 0 users, load average: 0.73, 0.68, 0.29 ===== cmsstor324
14:17:20 up 2 min, 0 users, load average: 1.19, 0.75, 0.29 ===== cmsstor325
14:17:20 up 3 min, 0 users, load average: 1.21, 0.91, 0.37 ===== cmsstor326
14:17:20 up 2 min, 0 users, load average: 1.45, 0.98, 0.39 ===== cmsstor327
14:17:21 up 2 min, 0 users, load average: 1.14, 0.77, 0.31 ===== cmsstor328
14:17:21 up 2 min, 0 users, load average: 1.53, 0.91, 0.36 ===== cmsstor329
14:17:22 up 2 min, 0 users, load average: 1.21, 0.77, 0.30 ===== cmsstor330
14:17:22 up 2 min, 0 users, load average: 2.11, 1.10, 0.42 ===== cmsstor331
14:17:23 up 2 min, 0 users, load average: 2.70, 1.38, 0.53 ===== cmsstor332
14:17:23 up 2 min, 0 users, load average: 1.30, 0.89, 0.37 ===== cmsstor333
14:17:23 up 3 min, 0 users, load average: 1.34, 1.28, 0.55 ===== cmsstor334
14:17:24 up 3 min, 0 users, load average: 1.39, 0.80, 0.32 ===== cmsstor335
14:17:24 up 3 min, 0 users, load average: 0.99, 0.65, 0.27 ===== cmsstor336
14:17:25 up 2 min, 0 users, load average: 1.31, 0.99, 0.40 ===== cmsstor337
14:17:25 up 2 min, 0 users, load average: 1.23, 0.78, 0.31 ===== cmsstor338
14:17:26 up 3 min, 0 users, load average: 0.68, 0.65, 0.27 ===== cmsstor339
14:17:26 up 1 min, 0 users, load average: 1.99, 0.61, 0.21 ===== cmsstor340
14:17:26 up 2 min, 0 users, load average: 1.49, 0.89, 0.35 ===== cmsstor341
14:17:28 up 2 min, 0 users, load average: 0.41, 0.50, 0.22 ===== cmsstor342
14:17:27 up 2 min, 0 users, load average: 1.12, 0.74, 0.30 ===== cmsstor343
14:17:28 up 2 min, 0 users, load average: 1.53, 0.76, 0.29 ===== cmsstor344
14:17:28 up 2 min, 0 users, load average: 2.03, 0.88, 0.33 ===== cmsstor345
14:17:29 up 2 min, 0 users, load average: 2.05, 0.94, 0.35 ===== cmsstor346
14:17:29 up 2 min, 0 users, load average: 1.90, 0.90, 0.34 ===== cmsstor347
14:17:29 up 1 min, 0 users, load average: 2.27, 0.75, 0.26 ===== cmsstor348
14:17:30 up 1 min, 0 users, load average: 2.65, 0.81, 0.28 ===== cmsstor349
14:17:30 up 1 min, 0 users, load average: 0.93, 0.51, 0.19 ===== cmsstor350
14:17:31 up 2 min, 0 users, load average: 1.08, 0.67, 0.27 ===== cmsstor351
14:17:31 up 2 min, 0 users, load average: 2.75, 1.27, 0.48 ===== cmsstor352
14:17:32 up 1 min, 0 users, load average: 2.32, 0.86, 0.31 ===== cmsstor353
ssh: connect to host cmsstor353 port 22: No route to host ===== cmsstor354
14:17:50 up 2 min, 0 users, load average: 1.32, 0.67, 0.26 ===== cmsstor355
14:17:51 up 1 min, 0 users, load average: 2.36, 0.89, 0.32 ===== cmsstor356
14:17:51 up 2 min, 0 users, load average: 1.80, 1.06, 0.42 ===== cmsstor357
14:17:52 up 1 min, 0 users, load average: 3.01, 0.92, 0.32 ===== cmsstor358
14:17:52 up 2 min, 0 users, load average: 2.02, 0.93, 0.35 ===== cmsstor359
14:17:53 up 1 min, 0 users, load average: 2.44, 0.97, 0.35 ===== cmsstor360
14:17:53 up 1 min, 0 users, load average: 2.94, 0.80, 0.27 ===== cmsstor361
14:17:53 up 1 min, 0 users, load average: 2.13, 0.79, 0.29 ===== cmsstor362
14:17:54 up 2 min, 0 users, load average: 2.36, 1.21, 0.46 ===== cmsstor363
14:17:54 up 1 min, 0 users, load average: 2.50, 0.90, 0.32 ===== cmsstor364
14:17:55 up 1 min, 0 users, load average: 2.40, 0.75, 0.26 ===== cmsstor365
14:17:55 up 1 min, 0 users, load average: 3.20, 0.97, 0.34 ===== cmsstor366
14:17:56 up 1 min, 0 users, load average: 3.29, 1.00, 0.35 ===== cmsstor368
14:17:56 up 1 min, 0 users, load average: 2.04, 0.69, 0.24 ===== cmsstor369
ssh: connect to host cmsstor369 port 22: Connection timed out ===== cmsstor370
14:19:00 up 2 min, 0 users, load average: 0.84, 0.54, 0.21 ===== cmsstor371
14:19:00 up 2 min, 0 users, load average: 1.20, 0.84, 0.34 ===== cmsstor372
14:19:01 up 3 min, 0 users, load average: 0.86, 0.85, 0.36 ===== cmsstor373
14:19:01 up 3 min, 0 users, load average: 1.57, 1.10, 0.45 ===== cmsstor374
14:19:02 up 3 min, 0 users, load average: 1.32, 1.02, 0.44 ===== cmsstor375
14:19:02 up 3 min, 0 users, load average: 1.86, 1.17, 0.48 ===== cmsstor376
14:19:03 up 2 min, 0 users, load average: 2.00, 1.08, 0.42
[root@cmsconsole NR]#

THree nodes are not up yet:

cmsstor317
cmsstor353
cmsstor369

Proceeding with ten 4xx nodes :
[root@cmsconsole NR]# wc -l cmsstor4xx.names
10 cmsstor4xx.names
[root@cmsconsole NR]# for n in `cat cmsstor4xx.names`; do echo ===== $n; ssh $n "uptime"; done ===== cmsstor401
14:17:16 up 175 days, 18:57, 0 users, load average: 0.15, 0.14, 0.15 ===== cmsstor402
14:17:16 up 175 days, 19:03, 0 users, load average: 0.00, 0.00, 0.00 ===== cmsstor403
14:17:16 up 175 days, 19:03, 0 users, load average: 0.11, 0.16, 0.17 ===== cmsstor404
14:17:17 up 175 days, 19:03, 0 users, load average: 0.00, 0.00, 0.00 ===== cmsstor405
14:17:17 up 175 days, 19:03, 0 users, load average: 0.12, 0.03, 0.01 ===== cmsstor406
14:17:17 up 175 days, 19:03, 0 users, load average: 0.00, 0.00, 0.00 ===== cmsstor407
14:17:17 up 175 days, 19:03, 0 users, load average: 0.23, 0.24, 0.12 ===== cmsstor408
14:17:18 up 175 days, 19:03, 0 users, load average: 0.00, 0.00, 0.00 ===== cmsstor409
14:17:18 up 26 days, 18:05, 0 users, load average: 0.06, 0.02, 0.00 ===== cmsstor410
14:17:18 up 26 days, 17:59, 0 users, load average: 0.09, 0.04, 0.00
[root@cmsconsole NR]# date; pssh -p 10 -h cmsstor4xx.names -t 0 -i "yum clean all; yum update -y; reboot now" 2>&1 | tee cmsstor4xx.upgrades.log ; date
Wed Aug 5 14:18:01 CDT 2015
[1] 14:19:11 [SUCCESS] cmsstor409
Loaded plugins: priorities, security
[... ]

kernel-headers.x86_64 0:2.6.32-504.30.3.el6

Complete!
Wed Aug 5 14:19:43 CDT 2015

#12 Updated by Natalia Ratnikova over 4 years ago

[root@cmsconsole NR]# for n in cmsstor317 cmsstor369 cmsstor401 cmsstor403; do cmspower-powerit -a cycle -c "Reboot for kernel upgrade " $n; done === cmsstor317 ===
connecting to APC APCCMS1055-2, outlet 22
Outlet state: OFF
connecting to APC APCCMS1055-2, outlet 22
Outlet state: ON === cmsstor369 ===
connecting to APC APCCMS1057-2, outlet 2
Outlet state: OFF
connecting to APC APCCMS1057-2, outlet 2
Outlet state: ON === cmsstor401 ===
connecting to APC apccms1570-1, outlet 1
Outlet state: OFF
connecting to APC apccms1570-1, outlet 1
Outlet state: ON === cmsstor403 ===
connecting to APC apccms1570-1, outlet 6
Outlet state: OFF
connecting to APC apccms1570-1, outlet 6
Outlet state: ON

#13 Updated by Natalia Ratnikova over 4 years ago

Gerard did:

[root@cmsadmin1 ~]# pssh -p 30 -t 600 -l root -h /root/natalia/DOWNTIME-Aug-5-2015/all_dcache_disk_pools -i "yum remove -y kernel-2.6.32-549.el6.x86_64; yum update -y"

Natalia rebooted all old-kernel-nodes. cmsstor221 did not come up. rest is OK
enabled puppet on all store nodes.

[root@cmsadmin1 DOWNTIME-Aug-5-2015]# pssh -p 30 -h old_kernel_nodes -t 0 -i "hostname; facter kernelrelease" | grep 2.6.32 | sort -u
2.6.32-504.30.3.el6.x86_64
[root@cmsadmin1 DOWNTIME-Aug-5-2015]# pssh -p 30 -h all_dcache_disk_pools -t 0 -i "hostname; puppet agent --enable "

Start agents:

cmsphedex-disk.fnal.gov - phedexdisk/production (SLF 6.6)
8-core Xeon E5430 @ 2.66GHz (PowerEdge 1950); 31.35 GB RAM, 16.00 GB swap
[root@cmsphedex-disk ~]# su - cmsprod
-bash-4.1$ cd siteconfs/siteconf-fnaldisk.4.1.3-comp3-1.0
-bash-4.1$ ./pmgr Dev start
Testing your DB connection
2015-08-05 20:55:38: CheckDBConnection20088: (re)connecting to database
2015-08-05 20:55:40: CheckDBConnection20088: disconnected from database
Connection to TMDB with /home/cmsprod/phedex/info/DBParam:Dev/T1USFNAL succeeded
FileExport: removing old stop flag /home/cmsprod/phedex/Dev_T1_US_FNAL_Disk/state/exp-pfn//stop
FileExport: pid 20097 started in /home/cmsprod/phedex/Dev_T1_US_FNAL_Disk/state/exp-pfn/
FileRemove: removing old stop flag /home/cmsprod/phedex/Dev_T1_US_FNAL_Disk/state/download-remove//stop
FileRemove: pid 20104 started in /home/cmsprod/phedex/Dev_T1_US_FNAL_Disk/state/download-remove/
Unknown option: jobs
FileDownload: removing old stop flag /home/cmsprod/phedex/Dev_T1_US_FNAL_Disk/state/download-cern//stop
FileDownload: pid 20112 started in /home/cmsprod/phedex/Dev_T1_US_FNAL_Disk/state/download-cern/
Unknown option: jobs
FileDownload: removing old stop flag /home/cmsprod/phedex/Dev_T1_US_FNAL_Disk/state/download-tier1//stop
FileDownload: pid 20120 started in /home/cmsprod/phedex/Dev_T1_US_FNAL_Disk/state/download-tier1/
Unknown option: jobs
FileDownload: removing old stop flag /home/cmsprod/phedex/Dev_T1_US_FNAL_Disk/state/download-tier2//stop
FileDownload: pid 20129 started in /home/cmsprod/phedex/Dev_T1_US_FNAL_Disk/state/download-tier2/
Unknown option: jobs
FileDownload: removing old stop flag /home/cmsprod/phedex/Dev_T1_US_FNAL_Disk/state/download-tier3//stop
FileDownload: pid 20137 started in /home/cmsprod/phedex/Dev_T1_US_FNAL_Disk/state/download-tier3/
BlockDownloadVerify: removing old stop flag /home/cmsprod/phedex/Dev_T1_US_FNAL_Disk/state/blockverify//stop
BlockDownloadVerify: pid 20145 started in /home/cmsprod/phedex/Dev_T1_US_FNAL_Disk/state/blockverify/
Watchdog: removing old stop flag /home/cmsprod/phedex/Dev_T1_US_FNAL_Disk/state/watchdog//stop
2015-08-05 20:55:44: Watchdog20151: starting Agent session (id=2)
2015-08-05 20:55:44: Watchdog20151: STATISTICS: Reporting every 43200 seconds, detail=0
2015-08-05 20:55:44: Watchdog20151: has successfully initialised
2015-08-05 20:55:44: Watchdog20151: debug: Agent was loaded DB=>1, CYCLE=1, DROPBOX=1
2015-08-05 20:55:44: Watchdog20151: Lookup agent "download-remove"
2015-08-05 20:55:44: Watchdog20151: Lookup agent "download-cern"
2015-08-05 20:55:44: Watchdog20151: Lookup agent "download-tier1"
2015-08-05 20:55:44: Watchdog20151: Lookup agent "download-tier2"
2015-08-05 20:55:44: Watchdog20151: Lookup agent "download-tier3"
2015-08-05 20:55:44: Watchdog20151: Lookup agent "exp-pfn"
2015-08-05 20:55:44: Watchdog20151: Lookup agent "blockverify"
2015-08-05 20:55:44: Watchdog20151: I am running these agents: Watchdog, blockverify, download-cern, download-remove, download-tier1, download-tier2, download-tier3, exp-pfn
2015-08-05 20:55:44: Watchdog20151: (re)connecting to database
2015-08-05 20:55:44: Watchdog20151: Creating new private DBH
Watchdog: pid 20536 started in /home/cmsprod/phedex/Dev_T1_US_FNAL_Disk/state/watchdog/
T1 disk pmgr: start Dev
-bash-4.1$ ./pmgr Debug start
Testing your DB connection
2015-08-05 20:55:51: CheckDBConnection20540: (re)connecting to database
2015-08-05 20:55:53: CheckDBConnection20540: disconnected from database
Connection to TMDB with /home/cmsprod/phedex/info/DBParam:Debug/T1USFNAL succeeded
FileExport: removing old stop flag /home/cmsprod/phedex/Debug_T1_US_FNAL_Disk/state/exp-pfn//stop
FileExport: pid 20550 started in /home/cmsprod/phedex/Debug_T1_US_FNAL_Disk/state/exp-pfn/
FileRemove: removing old stop flag /home/cmsprod/phedex/Debug_T1_US_FNAL_Disk/state/download-remove//stop
FileRemove: pid 20557 started in /home/cmsprod/phedex/Debug_T1_US_FNAL_Disk/state/download-remove/
Unknown option: jobs
FileDownload: removing old stop flag /home/cmsprod/phedex/Debug_T1_US_FNAL_Disk/state/download-cern//stop
FileDownload: pid 20565 started in /home/cmsprod/phedex/Debug_T1_US_FNAL_Disk/state/download-cern/
Unknown option: jobs
FileDownload: removing old stop flag /home/cmsprod/phedex/Debug_T1_US_FNAL_Disk/state/download-tier1//stop
FileDownload: pid 20572 started in /home/cmsprod/phedex/Debug_T1_US_FNAL_Disk/state/download-tier1/
Unknown option: jobs
FileDownload: removing old stop flag /home/cmsprod/phedex/Debug_T1_US_FNAL_Disk/state/download-tier2//stop
FileDownload: pid 20580 started in /home/cmsprod/phedex/Debug_T1_US_FNAL_Disk/state/download-tier2/
Unknown option: jobs
FileDownload: removing old stop flag /home/cmsprod/phedex/Debug_T1_US_FNAL_Disk/state/download-tier3//stop
FileDownload: pid 20587 started in /home/cmsprod/phedex/Debug_T1_US_FNAL_Disk/state/download-tier3/
BlockDownloadVerify: removing old stop flag /home/cmsprod/phedex/Debug_T1_US_FNAL_Disk/state/blockverify//stop
BlockDownloadVerify: pid 20596 started in /home/cmsprod/phedex/Debug_T1_US_FNAL_Disk/state/blockverify/
Watchdog: removing old stop flag /home/cmsprod/phedex/Debug_T1_US_FNAL_Disk/state/watchdog//stop
2015-08-05 20:55:56: Watchdog20602: starting Agent session (id=2)
2015-08-05 20:55:56: Watchdog20602: STATISTICS: Reporting every 43200 seconds, detail=0
2015-08-05 20:55:56: Watchdog20602: has successfully initialised
2015-08-05 20:55:56: Watchdog20602: Lookup agent "download-remove"
2015-08-05 20:55:56: Watchdog20602: Lookup agent "download-cern"
2015-08-05 20:55:56: Watchdog20602: Lookup agent "download-tier1"
2015-08-05 20:55:56: Watchdog20602: Lookup agent "download-tier2"
2015-08-05 20:55:56: Watchdog20602: Lookup agent "download-tier3"
2015-08-05 20:55:56: Watchdog20602: Lookup agent "exp-pfn"
2015-08-05 20:55:56: Watchdog20602: Lookup agent "blockverify"
2015-08-05 20:55:56: Watchdog20602: I am running these agents: Watchdog, blockverify, download-cern, download-remove, download-tier1, download-tier2, download-tier3, exp-pfn
2015-08-05 20:55:56: Watchdog20602: (re)connecting to database
Watchdog: pid 20991 started in /home/cmsprod/phedex/Debug_T1_US_FNAL_Disk/state/watchdog/
T1 disk pmgr: start Debug
-bash-4.1$ ./pmgr Prod start
Testing your DB connection
2015-08-05 20:56:09: CheckDBConnection21311: (re)connecting to database
2015-08-05 20:56:11: CheckDBConnection21311: disconnected from database
Connection to TMDB with /home/cmsprod/phedex/info/DBParam:Prod/T1USFNAL succeeded
FileExport: removing old stop flag /home/cmsprod/phedex/Prod_T1_US_FNAL_Disk/state/exp-pfn//stop
FileExport: pid 21345 started in /home/cmsprod/phedex/Prod_T1_US_FNAL_Disk/state/exp-pfn/
FileRemove: removing old stop flag /home/cmsprod/phedex/Prod_T1_US_FNAL_Disk/state/download-remove//stop
FileRemove: pid 21356 started in /home/cmsprod/phedex/Prod_T1_US_FNAL_Disk/state/download-remove/
Unknown option: jobs
FileDownload: removing old stop flag /home/cmsprod/phedex/Prod_T1_US_FNAL_Disk/state/download-cern//stop
FileDownload: pid 21364 started in /home/cmsprod/phedex/Prod_T1_US_FNAL_Disk/state/download-cern/
Unknown option: jobs
FileDownload: removing old stop flag /home/cmsprod/phedex/Prod_T1_US_FNAL_Disk/state/download-tier1//stop
FileDownload: pid 21372 started in /home/cmsprod/phedex/Prod_T1_US_FNAL_Disk/state/download-tier1/
Unknown option: jobs
FileDownload: removing old stop flag /home/cmsprod/phedex/Prod_T1_US_FNAL_Disk/state/download-tier2//stop
FileDownload: pid 21380 started in /home/cmsprod/phedex/Prod_T1_US_FNAL_Disk/state/download-tier2/
Unknown option: jobs
FileDownload: removing old stop flag /home/cmsprod/phedex/Prod_T1_US_FNAL_Disk/state/download-tier3//stop
FileDownload: pid 21388 started in /home/cmsprod/phedex/Prod_T1_US_FNAL_Disk/state/download-tier3/
BlockDownloadVerify: removing old stop flag /home/cmsprod/phedex/Prod_T1_US_FNAL_Disk/state/blockverify//stop
BlockDownloadVerify: pid 21396 started in /home/cmsprod/phedex/Prod_T1_US_FNAL_Disk/state/blockverify/
Watchdog: removing old stop flag /home/cmsprod/phedex/Prod_T1_US_FNAL_Disk/state/watchdog//stop
2015-08-05 20:56:14: Watchdog21402: starting Agent session (id=2)
2015-08-05 20:56:14: Watchdog21402: STATISTICS: Reporting every 43200 seconds, detail=0
2015-08-05 20:56:14: Watchdog21402: has successfully initialised
2015-08-05 20:56:14: Watchdog21402: Lookup agent "download-remove"
2015-08-05 20:56:14: Watchdog21402: Lookup agent "download-cern"
2015-08-05 20:56:14: Watchdog21402: Lookup agent "download-tier1"
2015-08-05 20:56:14: Watchdog21402: Lookup agent "download-tier2"
2015-08-05 20:56:14: Watchdog21402: Lookup agent "download-tier3"
2015-08-05 20:56:14: Watchdog21402: Lookup agent "exp-pfn"
2015-08-05 20:56:14: Watchdog21402: Lookup agent "blockverify"
2015-08-05 20:56:14: Watchdog21402: I am running these agents: Watchdog, blockverify, download-cern, download-remove, download-tier1, download-tier2, download-tier3, exp-pfn
2015-08-05 20:56:14: Watchdog21402: (re)connecting to database
Watchdog: pid 23911 started in /home/cmsprod/phedex/Prod_T1_US_FNAL_Disk/state/watchdog/
T1 disk pmgr: start Prod
-bash-4.1$

#14 Updated by Natalia Ratnikova over 4 years ago

[root@cmsstor403 ~]# service puppet start
Starting puppet agent: [ OK ]
root@cmsstor403 ~]# puppet agent -t
Warning: Local environment: "production" doesn't match server specified node environment "itb", switching agent to "itb".
Info: Retrieving pluginfacts
Info: Retrieving plugin
Info: Loading facts in /var/lib/puppet/lib/facter/iptables_version.rb
Info: Loading facts in /var/lib/puppet/lib/facter/puppet_vardir.rb
Info: Loading facts in /var/lib/puppet/lib/facter/facter_dot_d.rb
Info: Loading facts in /var/lib/puppet/lib/facter/cvmfsversion.rb
Info: Loading facts in /var/lib/puppet/lib/facter/afs_cache_size.rb
Info: Loading facts in /var/lib/puppet/lib/facter/ip6tables_version.rb
Info: Loading facts in /var/lib/puppet/lib/facter/pe_version.rb
Info: Loading facts in /var/lib/puppet/lib/facter/iptables_persistent_version.rb
Info: Loading facts in /var/lib/puppet/lib/facter/rsyslog_version.rb
Info: Loading facts in /var/lib/puppet/lib/facter/root_home.rb
Info: Loading facts in /var/lib/puppet/lib/facter/os_maj_version.rb
Info: Loading facts in /var/lib/puppet/lib/facter/code_server.rb
Info: Loading facts in /var/lib/puppet/lib/facter/certificate_facts.rb
Info: Loading facts in /var/lib/puppet/lib/facter/cvmfspartsize.rb
Info: Loading facts in /var/lib/puppet/lib/facter/postgres_default_version.rb
Info: Loading facts in /var/lib/puppet/lib/facter/concat_basedir.rb
Info: Caching catalog for cmsstor403.fnal.gov
Info: Applying configuration version '1438809728'
Notice: /Stage[main]/Swatch::Service/Service[swatch]/ensure: ensure changed 'stopped' to 'running'
Info: /Stage[main]/Swatch::Service/Service[swatch]: Unscheduling refresh on Service[swatch]
Notice: Finished catalog run in 17.69 seconds
[root@cmsstor403 ~]#

zabbix had an alarm about lack of swapping space.
Gerard fixed it (AIM):

this is because UUID=e63d1ef4-19a5-4c1d-a394-c1ce8aa1d6b3swapswapdefaults00
must not befound
fixed with [root@cmsstor403 ~]# swapon -a

the setup in /etc/fstab looks good to me
not sure why it didn’t mount the swap on boot

Unset downtime on dCache disk group
and on cmssrv228 cmsphedex-disk and cmsphedex-lpc servers

Several alarms about kernel errors in the system logs popped up in check_mk two hour later after the happening, Gerrard said this is a feature of check_mk implementation which needs to be fixed properly on another level.

The only remaining problem so far is cmsstor221, which is turning down by itself after it comes up on reboot (trieaged by Gerard) Will place a service call for it

#15 Updated by Natalia Ratnikova over 4 years ago

Finished kernel upgrade on the nodes we've missed or postponed to after the downtime.

cmschimeradiskbackup.fnal.gov Done
cmsphedex-disk.fnal.gov Done

cmspnfs1.fnal.gov Done - disk mount problem -> missing label, see notes in the next log.
cmsstor112.fnal.gov Done
cmsstor151.fnal.gov Done
cmsstor152.fnal.gov Done - same disk mount problems
cmsstor153.fnal.gov disk mount problems
cmsstor154.fnal.gov disk mount problems
cmsstor155.fnal.gov Done

cmssrv240.fnal.gov Done
cmssrv94.fnal.gov Done

Catalin's old test nodes, currently in unconfigured state

cmsstor252.fnal.gov 2.6.32-504.8.1.el6.x86_64
cmsstor253.fnal.gov 2.6.32-504.8.1.el6.x86_64
cmsstor254.fnal.gov 2.6.32-504.8.1.el6.x86_64
cmsstor255.fnal.gov 2.6.32-504.8.1.el6.x86_64
cmsstor256.fnal.gov 2.6.32-504.8.1.el6.x86_64

cmsphedex-tape.fnal.gov - old SL5 phedex server - reinstalled with SL6 as unconfigured node,
replaced by cmssrv228.

#16 Updated by Natalia Ratnikova over 4 years ago

After reboot several nodes would come into maintenance mode:

Give root password for maintenance
(or type Control-D to continue):

On Continue node will reboot and come to the same state again.
If root pwd is given, login into the system succeeds, but system disk is mounted readonly.

DBdisk label was found missing on cmspnfs1

Tim helped to repair this , his commands history:
457 fdisk -l
458 vi /etc/fstab
459 mount /dev/sdb3 /var/lib/pgsql-puppet
460 ls -la
461 mount
462 mount /dev/sdb3 /mnt
463 mount | grep sdb
464 fdisk -l
465 mount /dev/sda1 /mnt
466 cd /mnt
467 ls
468 ls -la
469 pwd
470 df -h .
471 mount .
472 mount
473 cd
474 umount /mnt
475 ls
476 cd /mnt
477 ls
478 ls -la
479 mkfs.xfs
480 mklab
481 mkfs.xfs -L DBdisk /dev/sda1
482 xfs_admin -L DBdisk /dev/sda1
483 ls -la
484 ls -la
485 df -h
486 cd
487 mount /dev/sda1 /mnt
488 ls
489 ls -la
490 cd /mnt
491 ls
492 ls -la
493 du
494 df -h .
495 ls -la
496 fg
497 mount -o remount,rw /
498 vi /etc/fstab
499 exit

After that checked that puppet runs cleanly, and dcache is started.
[root@cmspnfs1 ~]# df
Filesystem 1K-blocks Used Available Use% Mounted on
/dev/sdb3 553860088 40727612 484991372 8% /
tmpfs 24707184 0 24707184 0% /dev/shm
/dev/sdb1 999320 90016 856876 10% /boot
/dev/sda1 287701384 64424 273015932 1% /storage/local/data1

Tim's commands on cmsstor152:

493  vi /etc/fstab
494 mount -o remount,rw /
495 vi /etc/fstab
496 fdisk -l
497 mount
498 less /etc/fstab
499 exit

To test - login into amin interface on cmsstor152
cd PoolManager
..
cd PnfsManager
..
cd cd w-cmsstor112-disk_itb-disk1
..

and run a few commands (see help) like list files PNFS ids with rep ls and then find their path with pathfinder, check info, pinboard , etc. Things work fine.

#17 Updated by Natalia Ratnikova over 4 years ago

Fixing cmsstor153 (srm server):

login on the console with a root pwd (no remote login possible).

[root@cmsstor153 ~]# cat /etc/fstab
  1. HEADER: This file was autogenerated at Thu Feb 12 11:21:51 -0600 2015
  2. HEADER: by puppet. While it can still be managed manually, it
  3. HEADER: is definitely not recommended.
#
  1. /etc/fstab
  2. Created by anaconda on Wed May 21 21:41:27 2014 #
  3. Accessible filesystems, by reference, are maintained under '/dev/disk'
  4. See man pages fstab(5), findfs(8), mount(8) and/or blkid(8) for more info #
    UUID=5506b6b3-44d7-4bab-9ed3-5d8990be8adc / ext4 defaults 11
    UUID=3280a196-4b2b-40ed-8715-e5f0b6afe8a3 /boot ext4 defaults 12
    UUID=601e4e2d-a62c-41dd-bf78-a43780599ee1 swap swap defaults 00
    tmpfs /dev/shm tmpfs defaults 0 0
    devpts /dev/pts devpts gid=5,mode=620 0 0
    sysfs /sys sysfs defaults 0 0
    proc /proc proc defaults 0 0
    #LABEL=DBdisk /var/lib/pgsql-puppet ext4 defaults 1 1
    LABEL=DBdisk /var/lib/pgsql-puppet ext4 defaults 1 1

[root@cmsstor153 ~]# fdisk -l

Disk /dev/sda: 250.0 GB, 250000000000 bytes
255 heads, 63 sectors/track, 30394 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk identifier: 0x00000080

Device Boot      Start         End      Blocks   Id  System
/dev/sda1 * 1 131 1048576 83 Linux
Partition 1 does not end on cylinder boundary.
/dev/sda2 131 2220 16777216 82 Linux swap / Solaris
/dev/sda3 2220 30395 226313216 83 Linux

Disk /dev/sdb: 12001.7 GB, 12001652244480 bytes
255 heads, 63 sectors/track, 1459117 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 131072 bytes / 402653184 bytes
Disk identifier: 0x00000000

Disk /dev/sdc: 12001.7 GB, 12001652244480 bytes
255 heads, 63 sectors/track, 1459117 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 131072 bytes / 402653184 bytes
Disk identifier: 0x00000000

Disk /dev/sdd: 12001.7 GB, 12001652244480 bytes
255 heads, 63 sectors/track, 1459117 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 131072 bytes / 402653184 bytes
Disk identifier: 0x00000000

[root@cmsstor153 ~]# lsblk -o name,mountpoint,label,size,uuid
NAME MOUNTPOINT LABEL SIZE UUID
sda 232.9G
├─sda1 1G 3280a196-4b2b-40ed-8715-e5f0b6afe8a3
├─sda2 16G 601e4e2d-a62c-41dd-bf78-a43780599ee1
└─sda3 / 215.8G 5506b6b3-44d7-4bab-9ed3-5d8990be8adc
sr0 1024M
sdb 10.9T 08e06d98-432d-4dd8-a1b8-027390127893
sdc 10.9T 6e3cd6c7-dac1-497c-8416-5d4a967b32c0
sdd 10.9T c88e29f6-cdcf-447b-aa10-6121741703b6

HINT from CHih-Hao : how to figure out the postgres db location:

chekc arguments to postmaster process on the node:

[root@cmspnfs1 ~]# ps aux | grep master
postgres 7392 0.0 0.0 260184 35168 ? S 11:04 0:00 /usr/pgsql-9.2/bin/postmaster -p 5432 -D /var/lib/pgsql/9.2/data

So for cmspnfs1 we have db running on the system disk.
hieradata defined database disk mount with ensure unmounted. What's the purpose of this?

Checked blame - commit belongs to Gerard with no comments.

oncmspnfs1:
Moved /etc/fstab and run puppet to see what it creates.

It actually creates the mount for DBdisk based on label.
Label was not existing, Tim put it on some random disk, and no puppet makes sure that disk is not mounted?
What all this is needed for???
Asking Gerard.

The same situation is on all _itb head nodes/servers, and no labels on the disks... so the system does NOT come up properly..

#18 Updated by Natalia Ratnikova over 4 years ago

Reimplement mounts for testbed DB disks to fix mount problems introduced with inheritance.

Actions:
1) Create new branch off of itb :
natalia_fix_dbdisk_mounts_on_testbed

2) Modify hiera yaml files for each role:
to ensure DBdisk mount point is 'absent' instead of 'unmounted'

3) commit changes

4) In ENC - move one of the servers onto natalia_fix_dbdisk_mounts_on_testbed branch

5) Run puppet manually, check that node and dcache service come up properly.

6) Move all remaining servers into natalia_fix_dbdisk_mounts_on_testbed branch

7) Run puppet manually, check that all nodes come up properly and testbed instance is running and functioning well.

8) Start puppet service everytwhere.

9) Let branch to be reviewed and merge into itb.

10) Move servers in ENC to itb branch.

#19 Updated by Natalia Ratnikova over 4 years ago

The previous plan does not succeed, because we have another dependency missing:

Info: Loading facts in /var/lib/puppet/lib/facter/ip6tables_version.rb
Error: Could not retrieve catalog from remote server: Error 400 on SERVER: Invalid relationship: Class[Dcache::Db] { require => File[/var/lib/pgsql-puppet/data1] }, because File[/var/lib/pgsql-puppet/data1] doesn't seem to be in the catalog
Warning: Not using cache on failed catalog
Error: Could not retrieve catalog; skipping run

Next two options are:

A) make it how it was before i.e. label a spare disk and require it to be unmounted.
This is kind of conceptual flaw ,as we are bound to be wasting a disk.
B) Label and mount the DBdisk and move the databases from the system disk onto DBdisk. 
This way we will have the same setup as in the production instance, which is good.

- Consulted with Chih-Hao about moving the database: this can be done while db is stopped.

The steps for plan B:
B-1 stop puppet
B-2 stop the database
B-3 rename the database location
B-4 label and mount the DBdisk
B-5 make sure the same location is created and resides on the mounted disk
B-6 copy the database files onto the new disk old location.
B-7 start the database and check that it works
B-8 Make changes in the yaml files 'absent' -> 'mounted', push to the branch
B-9 Follow steps 5-10 from the previous plan:

5) Run puppet manually, check that node and dcache service come up properly.

6) Move all remaining servers into natalia_fix_dbdisk_mounts_on_testbed branch

7) Run puppet manually, check that all nodes come up properly and testbed instance is running and functioning well.

8) Start puppet service everytwhere.

9) Let branch to be reviewed and merge into itb.

10) Move servers in ENC to itb branch.

#20 Updated by Natalia Ratnikova over 4 years ago

cmsstor152.fnal.gov - dcacheadmindisk_itb/natalia_dcache_java_upgrade_test (SLF 6.6)
8-core Xeon E5430 @ 2.66GHz (PowerEdge 1950); 15.57 GB RAM, 16.00 GB swap
[root@cmsstor152 ~]# who
root ttyS0 2015-08-18 11:42
root pts/0 2015-08-18 11:41 (cmsadmin1.fnal.gov)
root pts/1 2015-08-18 13:03 (cmsadmin1.fnal.gov)
[root@cmsstor152 ~]# mount
/dev/sda3 on / type ext4 (rw)
proc on /proc type proc (rw)
sysfs on /sys type sysfs (rw)
devpts on /dev/pts type devpts (rw,gid=5,mode=620)
tmpfs on /dev/shm type tmpfs (rw)
/dev/sda1 on /boot type ext4 (rw)
none on /proc/sys/fs/binfmt_misc type binfmt_misc (rw)
[root@cmsstor152 ~]# cat /etc/fstab
  1. HEADER: This file was autogenerated at Wed Aug 12 11:07:17 -0500 2015
  2. HEADER: by puppet. While it can still be managed manually, it
  3. HEADER: is definitely not recommended.
#
  1. /etc/fstab
  2. Created by anaconda on Wed May 21 21:40:30 2014 #
  3. Accessible filesystems, by reference, are maintained under '/dev/disk'
  4. See man pages fstab(5), findfs(8), mount(8) and/or blkid(8) for more info #
    UUID=9b25cf54-b1ed-4f47-9a5e-d38caaa94acf / ext4 defaults 1
    UUID=a10c8181-b945-4a5f-85c4-8d4edde92e67 /boot ext4 defaults 2
    UUID=2c68d637-911f-4652-82e1-a70add7338ca swap swap defaults 0
    tmpfs /dev/shm tmpfs defaults 0 0
    devpts /dev/pts devpts gid=5,mode=620 0 0
    sysfs /sys sysfs defaults 0 0
    proc /proc proc defaults 0 0
    #LABEL=DBdisk /var/lib/pgsql-puppet ext4 defaults 1 1
    #LABEL=DBdisk /var/lib/pgsql-puppet ext4 defaults 1 1
    #LABEL=DBdisk /var/lib/pgsql-puppet ext4 defaults 1 1
    LABEL=DBdisk /var/lib/pgsql-puppet ext4 defaults 1 1
    [root@cmsstor152 ~]# service puppet status
    puppet (pid 16982) is running...
    [root@cmsstor152 ~]# service puppet stop
    Stopping puppet agent: [ OK ]
    [root@cmsstor152 ~]# service dcache-server status
    DOMAIN STATUS PID USER
    dirDomain running 8252 root
    billingDomain running 8317 root
    httpdDomain running 8352 root
    adminDoorDomain running 8406 root
    dCacheDomain running 8469 root
    [root@cmsstor152 ~]# service dcache-server stop
    Stopping dCacheDomain 0 done
    Stopping adminDoorDomain 0 done
    Stopping httpdDomain 0 1 done
    Stopping billingDomain 0 done
    Stopping dirDomain 0 done
    [root@cmsstor152 ~]#

[root@cmsstor152 ~]# service postgresql-9.2 status
(pid 7734) is running...
[root@cmsstor152 ~]# service postgresql-9.2 stop
Stopping postgresql-9.2 service: [ OK ]
[root@cmsstor152 ~]#

#21 Updated by Natalia Ratnikova over 4 years ago

For cmsstor152 there is no actually a spare local disk, there is data disk (Satabeast?) as it used to be a pool node. It would be an overkill to have just admin node dbs there.

So we go for plan A for now, until a better solution is found, on the puppet side.

[root@cmsstor152 PG_9.2_201204301]# lsblk -o name,mountpoint,label,size,uuid
NAME MOUNTPOINT LABEL SIZE UUID
sda 232.9G
├─sda1 /boot 1G a10c8181-b945-4a5f-85c4-8d4edde92e67
├─sda2 [SWAP] 16G 2c68d637-911f-4652-82e1-a70add7338ca
└─sda3 / 215.8G 9b25cf54-b1ed-4f47-9a5e-d38caaa94acf
sr0 1024M
sdc 10.9T 6267cb12-7051-42e4-909b-7b948aecb569
sdd 10.9T b22e1143-92c5-4a16-b40d-be3566b832dc
sdb 10.9T a397ce45-5bf2-4dc2-97b1-2c7ebf6f652d

[root@cmsstor152 PG_9.2_201204301]# xfs_admin -L DBdisk /dev/sdc
writing all SBs
new label = "DBdisk"
[root@cmsstor152 PG_9.2_201204301]# lsblk -o name,mountpoint,label,size,uuid
NAME MOUNTPOINT LABEL SIZE UUID
sda 232.9G
├─sda1 /boot 1G a10c8181-b945-4a5f-85c4-8d4edde92e67
├─sda2 [SWAP] 16G 2c68d637-911f-4652-82e1-a70add7338ca
└─sda3 / 215.8G 9b25cf54-b1ed-4f47-9a5e-d38caaa94acf
sr0 1024M
sdc DBdisk 10.9T 6267cb12-7051-42e4-909b-7b948aecb569
sdd 10.9T b22e1143-92c5-4a16-b40d-be3566b832dc
sdb 10.9T a397ce45-5bf2-4dc2-97b1-2c7ebf6f652d
[root@cmsstor152 PG_9.2_201204301]#

#22 Updated by Natalia Ratnikova over 4 years ago

Revert git commit with mount absent for all servers.

[cmsdev33 11:04] git show3f747cf108395216e8b3677550dc81d9b9f853c9
git: 'show3f747cf108395216e8b3677550dc81d9b9f853c9' is not a git command. See 'git --help'.
[cmsdev33 11:04] git show 3f747cf108395216e8b3677550dc81d9b9f853c9
commit 3f747cf108395216e8b3677550dc81d9b9f853c9
Author: Natalia Ratnikova <>
Date: Tue Aug 18 11:29:42 2015 -0500

Trying to fix a conceptual flaw in DBdisk mount.

diff --git a/hieradata/roles/chimeradisk_itb.yaml b/hieradata/roles/chimeradisk_itb.yaml
index 8ec9563..a229780 100644
--- a/hieradata/roles/chimeradisk_itb.yaml
+++ b/hieradata/roles/chimeradisk_itb.yaml
@ -10,7 +10,7 @ dcache::packages:

dcache::dbmount:
dbdisk:
- ensure: 'unmounted'
+ ensure: 'absent'
path: /var/lib/pgsql-puppet
device: 'LABEL=DBdisk'
fstype: 'ext4'
diff --git a/hieradata/roles/chimeradiskbackup_itb.yaml b/hieradata/roles/chimeradiskbackup_itb.yaml
index 568cb62..8f1131e 100644
--- a/hieradata/roles/chimeradiskbackup_itb.yaml
+++ b/hieradata/roles/chimeradiskbackup_itb.yaml
@ -14,13 +14,13 @ dcache::packages:
dcache::dbmount:
dbdisk:
- ensure: 'unmounted'
+ ensure: 'absent'
path: /var/lib/pgsql-puppet
device: 'LABEL=DBdisk'
fstype: 'ext4'
subdir: 'data1'
dbdisk-backup:
- ensure: 'unmounted'
+ ensure: 'absent'
path: /var/lib/pgsql
device: 'LABEL=DBdisk'
fstype: 'ext4'
diff --git a/hieradata/roles/dcacheadmindisk_itb.yaml b/hieradata/roles/dcacheadmindisk_itb.yaml
index ee53a3d..c2c1038 100644
--- a/hieradata/roles/dcacheadmindisk_itb.yaml
+++ b/hieradata/roles/dcacheadmindisk_itb.yaml
@ -11,7 +11,7 @ dcache::packages:
dcache::dbmount:
dbdisk:
- ensure: 'unmounted'
+ ensure: 'absent'
device: 'LABEL=DBdisk'
path: /var/lib/pgsql-puppet
fstype: 'ext4'
diff --git a/hieradata/roles/srmdisk_itb.yaml b/hieradata/roles/srmdisk_itb.yaml
index 8f3868e..2382c2e 100644
--- a/hieradata/roles/srmdisk_itb.yaml
+++ b/hieradata/roles/srmdisk_itb.yaml
@ -16,7 +16,7 @ dcache::packages:
dcache::dbmount:
dbdisk:
- ensure: 'unmounted'
+ ensure: 'absent'
path: /var/lib/pgsql-puppet
device: 'LABEL=DBdisk'
subdir: 'data1'
[cmsdev33 11:04] git revert 3f747cf108395216e8b3677550dc81d9b9f853c9
[natalia_fix_dbdisk_mounts_on_testbed 5b7329e] Revert "Trying to fix a conceptual flaw in DBdisk mount." Just requiring mount absent leaved other dependencies around. See details in redmine #9789 This reverts commit 3f747cf108395216e8b3677550dc81d9b9f853c9.
4 files changed, 5 insertions(+), 5 deletions(-)
[cmsdev33 11:04]

#23 Updated by Natalia Ratnikova over 4 years ago

note there are errors in trying to reload the poolmanager.
Restart the dcache and pg services and rerun puppet twice to see if that helps.
Second run is clean .

[root@cmsstor152 ~]# puppet agent t
Warning: Local environment: "production" doesn't match server specified node environment "natalia_fix_dbdisk_mounts_on_testbed", switching agent to "natalia_fix_dbdisk_mounts_on_testbed".
Info: Retrieving pluginfacts
Info: Retrieving plugin
Info: Loading facts in /var/lib/puppet/lib/facter/iptables_version.rb
Info: Loading facts in /var/lib/puppet/lib/facter/cvmfspartsize.rb
Info: Loading facts in /var/lib/puppet/lib/facter/pe_version.rb
Info: Loading facts in /var/lib/puppet/lib/facter/afs_cache_size.rb
Info: Loading facts in /var/lib/puppet/lib/facter/os_maj_version.rb
Info: Loading facts in /var/lib/puppet/lib/facter/code_server.rb
Info: Loading facts in /var/lib/puppet/lib/facter/cvmfsversion.rb
Info: Loading facts in /var/lib/puppet/lib/facter/facter_dot_d.rb
Info: Loading facts in /var/lib/puppet/lib/facter/postgres_default_version.rb
Info: Loading facts in /var/lib/puppet/lib/facter/certificate_facts.rb
Info: Loading facts in /var/lib/puppet/lib/facter/root_home.rb
Info: Loading facts in /var/lib/puppet/lib/facter/concat_basedir.rb
Info: Loading facts in /var/lib/puppet/lib/facter/iptables_persistent_version.rb
Info: Loading facts in /var/lib/puppet/lib/facter/rsyslog_version.rb
Info: Loading facts in /var/lib/puppet/lib/facter/puppet_vardir.rb
Info: Loading facts in /var/lib/puppet/lib/facter/ip6tables_version.rb
Info: Caching catalog for cmsstor152.fnal.gov
Info: Applying configuration version '1439924099'
Notice: /Stage[main]/Vulcan::Dcache/File[/etc/dcache/dcache.kpwd]/content:
--
/etc/dcache/dcache.kpwd 2015-08-18 12:46:39.745156147 -0500
++ /tmp/puppet-file20150818-7093-1cs3z7n-0 2015-08-18 13:55:31.991913737 -0500
@ -3572,7 +3572,7 @
mapping "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=fanzago/CN=610896/CN=Federica Fanzago" uscms706
mapping "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=farida/CN=518177/CN=Farida Fassi" atlas
mapping "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=fassoul/CN=390926/CN=Dimitris Fassouliotis" atlas
-mapping "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=fatsai/CN=711765/CN=Fang-Ying Tsai" uscms6225
+mapping "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=fatsai/CN=711765/CN=Fang-Ying Tsai" uscms6232
mapping "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=favaro/CN=698706/CN=Carlotta Favaro" uscms2711
mapping "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=fay/CN=391098/CN=Jean Fay" uscms2026
mapping "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=fball/CN=751136/CN=Fionn Amhairghen Ball" uscms5572
@ -4042,7 +4042,7 @
mapping "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=jmejiagu/CN=738553/CN=Jhovanny Andres Mejia Guisao" uscms4599
mapping "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=jmhernan/CN=763451/CN=Jonathan Matthew Hernandez" uscms6206
mapping "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=jmhogan/CN=676292/CN=Julie Hogan" jmanagan
-mapping "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=jmhogan/CN=676292/CN=Julie Managan Hogan" uscms6230
+mapping "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=jmhogan/CN=676292/CN=Julie Managan Hogan" uscms6223
mapping "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=jmolinai/CN=671301/CN=Jorge Andres Molina Insfran" uscms5608
mapping "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=jmonk/CN=664162/CN=James William Monk" atlas
mapping "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=jmonkevi/CN=759961/CN=Jonas Monkevicius" uscms5876
@ -4484,7 +4484,7 @
mapping "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=mendez/CN=490587/CN=Hector Mendez" mendez
mapping "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=mendo/CN=711096/CN=Masaki Endo" atlas
mapping "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=meneguz/CN=394388/CN=Anna Meneguzzo" uscms2810
-mapping "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=meola/CN=665823/CN=Sabino Meola" uscms6228
+mapping "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=meola/CN=665823/CN=Sabino Meola" uscms6224
mapping "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=meridian/CN=555986/CN=Paolo Meridiani" meridian
mapping "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=merina/CN=673555/CN=Merina Albert" uscms5650
mapping "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=merkelp/CN=631815/CN=Petra Merkel" uscms1570
@ -4938,7 +4938,7 @
mapping "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=rharris/CN=514003/CN=Robert Harris" uscms5602
mapping "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=rhawking/CN=399511/CN=Richard Hawkings" atlas
mapping "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=rheller/CN=743534/CN=Ryan Edward Heller" uscms5977
-mapping "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=riahi/CN=672488/CN=Hassen Riahi" uscms6229
+mapping "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=riahi/CN=672488/CN=Hassen Riahi" uscms6225
mapping "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=ribon/CN=571572/CN=Alberto Ribon" geant4
mapping "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=ric/CN=477544/CN=Riccardo Paramatti" uscms646
mapping "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=ricab/CN=683129/CN=Ricardo Lince Amaral Farto E Abreu" atlas
@ -7355,6 +7355,7 @
mapping "/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Arkadios Manousakis-katsikakis/CN=UID:amanou" cdf
mapping "/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Arkadiy L. Klebaner/CN=UID:klebaner" fermilab
mapping "/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Armand J. Bianchi/CN=UID:bianchiaj" fermilab
+mapping "/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Armando Barboza/CN=UID:abarboza" fermilab
mapping "/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Arnab Purohit/CN=UID:apurohit" fermilab
mapping "/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Arnau Vallribera/CN=UID:apujol" fermilab
mapping "/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Arnaud Duperrin/CN=UID:duperrin" fermilab
@ -7376,6 +7377,7 @
mapping "/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Artur M. Ankowski/CN=UID:aankowsk" fermilab
mapping "/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Arturo Gonzales/CN=UID:gonzales" fermilab
mapping "/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Arun Saini/CN=UID:asaini" fermilab
+mapping "/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Arun Srivastava/CN=UID:asrivast" fermilab
mapping "/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Arun Tadepalli/CN=UID:arunts" fermilab
mapping "/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Arya Farahi/CN=UID:afarahi" fermilab
mapping "/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Aseet Mukherjee/CN=UID:aseet" fermilab
@ -8234,6 +8236,7 @
mapping "/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Debarati Roy/CN=UID:debarati" fermilab
mapping "/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Deborah H. Griffin/CN=UID:dgriffin" fermilab
mapping "/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Deborah Harris/CN=UID:dharris" fermilab
+mapping "/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Deborah J. Gulledge/CN=UID:dgulledg" fermilab
mapping "/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Deborah L. Bonifas/CN=UID:debbieb" fermilab
mapping "/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Deborah Sebastian/CN=UID:debseb" fermilab
mapping "/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Debra L. Cobb/CN=UID:cobb" fermilab
@ -14180,6 +14183,7 @
mapping "/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Arkadios Manousakis-katsikakis/CN=UID:amanou" fermilab
mapping "/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Arkadiy L. Klebaner/CN=UID:klebaner" fermilab
mapping "/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Armand J. Bianchi/CN=UID:bianchiaj" fermilab
+mapping "/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Armando Barboza/CN=UID:abarboza" fermilab
mapping "/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Arnab Purohit/CN=UID:apurohit" fermilab
mapping "/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Arnau Vallribera/CN=UID:apujol" fermilab
mapping "/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Arnaud Duperrin/CN=UID:duperrin" fermilab
@ -14202,6 +14206,7 @
mapping "/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Artur M. Ankowski/CN=UID:aankowsk" fermilab
mapping "/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Arturo Gonzales/CN=UID:gonzales" fermilab
mapping "/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Arun Saini/CN=UID:asaini" fermilab
+mapping "/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Arun Srivastava/CN=UID:asrivast" fermilab
mapping "/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Arun Tadepalli/CN=UID:arunts" fermilab
mapping "/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Arvin Samali/CN=UID:asamali" fermilab
mapping "/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Arya Farahi/CN=UID:afarahi" fermilab
@ -15170,6 +15175,7 @
mapping "/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Debarati Roy/CN=UID:debarati" fermilab
mapping "/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Deborah H. Griffin/CN=UID:dgriffin" fermilab
mapping "/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Deborah Harris/CN=UID:dharris" fermilab
+mapping "/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Deborah J. Gulledge/CN=UID:dgulledg" fermilab
mapping "/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Deborah L. Bonifas/CN=UID:debbieb" fermilab
mapping "/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Deborah Sebastian/CN=UID:debseb" fermilab
mapping "/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Debra L. Cobb/CN=UID:cobb" fermilab
@ -37024,6 +37030,7 @
/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Arka Santra/CN=UID:asantra4
/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Arkadiy L. Klebaner/CN=UID:klebaner
/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Armand J. Bianchi/CN=UID:bianchiaj
/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Armando Barboza/CN=UID:abarboza
/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Arnab Purohit/CN=UID:apurohit
/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Arnau Vallribera/CN=UID:apujol
/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Arnaud Duperrin/CN=UID:duperrin
@ -37043,6 +37050,7 @
/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Artur Galt/CN=UID:agalt
/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Arturo Gonzales/CN=UID:gonzales
/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Arun Saini/CN=UID:asaini
+ /DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Arun Srivastava/CN=UID:asrivast
/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Arya Farahi/CN=UID:afarahi
/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Aseet Mukherjee/CN=UID:aseet
/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Ashish Kumar/CN=UID:ashishk
@ -37787,6 +37795,7 @
/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Deanne S. Slapa/CN=UID:dslapa
/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Debarati Roy/CN=UID:debarati
/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Deborah H. Griffin/CN=UID:dgriffin
+ /DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Deborah J. Gulledge/CN=UID:dgulledg
/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Deborah L. Bonifas/CN=UID:debbieb
/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Deborah Sebastian/CN=UID:debseb
/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Debra L. Cobb/CN=UID:cobb
@ -42272,6 +42281,7 @
/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Arkadios Manousakis-katsikakis/CN=UID:amanou
/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Arkadiy L. Klebaner/CN=UID:klebaner
/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Armand J. Bianchi/CN=UID:bianchiaj
+ /DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Armando Barboza/CN=UID:abarboza
/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Arnab Purohit/CN=UID:apurohit
/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Arnau Vallribera/CN=UID:apujol
/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Arnaud Duperrin/CN=UID:duperrin
@ -42292,6 +42302,7 @
/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Artur Galt/CN=UID:agalt
/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Arturo Gonzales/CN=UID:gonzales
/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Arun Saini/CN=UID:asaini
+ /DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Arun Srivastava/CN=UID:asrivast
/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Arvin Samali/CN=UID:asamali
/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Arya Farahi/CN=UID:afarahi
/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Aseet Mukherjee/CN=UID:aseet
@ -43132,6 +43143,7 @
/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Deanne S. Slapa/CN=UID:dslapa
/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Debarati Roy/CN=UID:debarati
/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Deborah H. Griffin/CN=UID:dgriffin
+ /DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Deborah J. Gulledge/CN=UID:dgulledg
/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Deborah L. Bonifas/CN=UID:debbieb
/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Deborah Sebastian/CN=UID:debseb
/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Debra L. Cobb/CN=UID:cobb
@ -55155,8 +55167,14 @
login uscms6222 read-only 1123 1999 / /dcache /dcache
/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=sicheng/CN=748935/CN=Sicheng Wang

login uscms6223 read-only 1123 1999 / /dcache /dcache
/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=jmhogan/CN=676292/CN=Julie Managan Hogan

+login uscms6224 read-only 1123 1999 / /dcache /dcache
/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=meola/CN=665823/CN=Sabino Meola

login uscms6225 read-only 1123 1999 / /dcache /dcache
- /DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=fatsai/CN=711765/CN=Fang-Ying Tsai
/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=riahi/CN=672488/CN=Hassen Riahi

login uscms6226 read-only 1123 1999 / /dcache /dcache
/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=strolog/CN=679949/CN=John Strologas
@ -55164,21 +55182,15 @
login uscms6227 read-only 1123 1999 / /dcache /dcache
/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=bbatool/CN=766833/CN=Binish Batool

login uscms6228 read-only 1123 1999 / /dcache /dcache
/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=meola/CN=665823/CN=Sabino Meola

-login uscms6229 read-only 1123 1999 / /dcache /dcache
/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=riahi/CN=672488/CN=Hassen Riahi
-
login uscms623 read-only 1123 1999 / /dcache /dcache
/DC=es/DC=irisgrid/O=ciemat/CN=isabel.josa

login uscms6230 read-only 1123 1999 / /dcache /dcache
/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=jmhogan/CN=676292/CN=Julie Managan Hogan
-
login uscms6231 read-only 1123 1999 / /dcache /dcache
/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=ahazi/CN=676282/CN=Andras Hazi

login uscms6232 read-only 1123 1999 / /dcache /dcache
/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=fatsai/CN=711765/CN=Fang-Ying Tsai
+
login uscms6234 read-only 1123 1999 / /dcache /dcache
/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Gerard Bernabeu Altayo/CN=UID:gerard1

Info: /Stage[main]/Vulcan::Dcache/File[/etc/dcache/dcache.kpwd]: Filebucketed /etc/dcache/dcache.kpwd to main with sum a1f6ca23c3f7ebaf51f1cac345edbdf8
Notice: /Stage[main]/Vulcan::Dcache/File[/etc/dcache/dcache.kpwd]/content: content changed '{md5}a1f6ca23c3f7ebaf51f1cac345edbdf8' to '{md5}188b803cce55de406e4c91033fc1897d'
Info: /Stage[main]/P_fact::Puppet/P_fact::String[puppetenvironment]/File[/etc/facter/facts.d/puppetenvironment.txt]: Filebucketed /etc/facter/facts.d/puppetenvironment.txt to main with sum 430af18c5e7188929f99b954233d1840
Notice: /Stage[main]/P_fact::Puppet/P_fact::String[puppetenvironment]/File[/etc/facter/facts.d/puppetenvironment.txt]/content: content changed '{md5}430af18c5e7188929f99b954233d1840' to '{md5}96a11947f7b5c2fb94c0e52291a849ff'
Info: /Stage[main]/Motd::Warning/Motd[/etc/motd]/File[/etc/motd]: Filebucketed /etc/motd to main with sum c2071a07891f3d8d8b00e3f0f2acd9d2
Notice: /Stage[main]/Motd::Warning/Motd[/etc/motd]/File[/etc/motd]/content: content changed '{md5}c2071a07891f3d8d8b00e3f0f2acd9d2' to '{md5}97b5ed153cbd7c3803eefb6f6f5841de'
Notice: /Stage[main]/Vulcan::Dcache/File[/etc/grid-security/grid-vorolemap]/content:
--- /etc/grid-security/grid-vorolemap 2015-08-18 12:46:59.138028915 0500
++ /tmp/puppet-file20150818-7093-k1d7mq-0 2015-08-18 13:55:45.373515498 -0500
@ -8761,8 +8761,8 @
"/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=rheller/CN=743534/CN=Ryan Edward Heller" "/cms/Role=cmsuser" uscms5977
"/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=rheller/CN=743534/CN=Ryan Edward Heller" "/cms/uscms/Role=cmsuser" uscms5977
"/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=rheller/CN=743534/CN=Ryan Edward Heller" uscms5977
"/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=riahi/CN=672488/CN=Hassen Riahi" "/cms" uscms6225
-"/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=riahi/CN=672488/CN=Hassen Riahi" uscms6225
"/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=riahi/CN=672488/CN=Hassen Riahi" "/cms" uscms6229
"/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=riahi/CN=672488/CN=Hassen Riahi" uscms6229
"/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=ribon/CN=571572/CN=Alberto Ribon" "/geant4" geant4
"/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=ribon/CN=571572/CN=Alberto Ribon" "/geant4/Role=lcgadmin" geant4
"/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=ric/CN=477544/CN=Riccardo Paramatti" "/cms" uscms646
@ -15565,6 +15565,8 @
"/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Arkadiy L. Klebaner/CN=UID:klebaner" "/fermilab/Role=Analysis" fermilab
"/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Armand J. Bianchi/CN=UID:bianchiaj" "/fermilab" fermilab
"/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Armand J. Bianchi/CN=UID:bianchiaj" "/fermilab/Role=Analysis" fermilab
"/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Armando Barboza/CN=UID:abarboza" "/fermilab" fermilab
"/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Armando Barboza/CN=UID:abarboza" "/fermilab/Role=Analysis" fermilab
"/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Arnab Purohit/CN=UID:apurohit" "/fermilab" fermilab
"/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Arnab Purohit/CN=UID:apurohit" "/fermilab/Role=Analysis" fermilab
"/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Arnau Vallribera/CN=UID:apujol" "/fermilab" fermilab
@ -15640,6 +15642,8 @
"/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Arturo Gonzales/CN=UID:gonzales" "/fermilab/Role=Analysis" fermilab
"/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Arun Saini/CN=UID:asaini" "/fermilab" fermilab
"/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Arun Saini/CN=UID:asaini" "/fermilab/Role=Analysis" fermilab
"/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Arun Srivastava/CN=UID:asrivast" "/fermilab" fermilab
"/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Arun Srivastava/CN=UID:asrivast" "/fermilab/Role=Analysis" fermilab
"/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Arun Tadepalli/CN=UID:arunts" "/fermilab" fermilab
"/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Arun Tadepalli/CN=UID:arunts" "/fermilab/Role=Analysis" fermilab
"/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Arun Tadepalli/CN=UID:arunts" "/fermilab/seaquest" fermilab
@ -17779,6 +17783,8 @
"/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Deborah Harris/CN=UID:dharris" "/fermilab/Role=Analysis" fermilab
"/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Deborah Harris/CN=UID:dharris" "/fermilab/minos" fermilab
"/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Deborah Harris/CN=UID:dharris" "/fermilab/minos/Role=Analysis" fermilab
"/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Deborah J. Gulledge/CN=UID:dgulledg" "/fermilab" fermilab
"/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Deborah J. Gulledge/CN=UID:dgulledg" "/fermilab/Role=Analysis" fermilab
"/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Deborah L. Bonifas/CN=UID:debbieb" "/fermilab" fermilab
"/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Deborah L. Bonifas/CN=UID:debbieb" "/fermilab/Role=Analysis" fermilab
"/DC=gov/DC=fnal/O=Fermilab/OU=People/CN=Deborah Sebastian/CN=UID:debseb" "/fermilab" fermilab
@ -32661,6 +32667,8 @
"/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Arkadiy L. Klebaner/CN=UID:klebaner" "/fermilab/Role=Analysis" fermilab
"/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Armand J. Bianchi/CN=UID:bianchiaj" "/fermilab" fermilab
"/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Armand J. Bianchi/CN=UID:bianchiaj" "/fermilab/Role=Analysis" fermilab
"/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Armando Barboza/CN=UID:abarboza" "/fermilab" fermilab
"/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Armando Barboza/CN=UID:abarboza" "/fermilab/Role=Analysis" fermilab
"/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Arnab Purohit/CN=UID:apurohit" "/fermilab" fermilab
"/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Arnab Purohit/CN=UID:apurohit" "/fermilab/Role=Analysis" fermilab
"/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Arnau Vallribera/CN=UID:apujol" "/fermilab" fermilab
@ -32738,6 +32746,8 @
"/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Arturo Gonzales/CN=UID:gonzales" "/fermilab/Role=Analysis" fermilab
"/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Arun Saini/CN=UID:asaini" "/fermilab" fermilab
"/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Arun Saini/CN=UID:asaini" "/fermilab/Role=Analysis" fermilab
"/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Arun Srivastava/CN=UID:asrivast" "/fermilab" fermilab
"/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Arun Srivastava/CN=UID:asrivast" "/fermilab/Role=Analysis" fermilab
"/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Arun Tadepalli/CN=UID:arunts" "/fermilab" fermilab
"/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Arun Tadepalli/CN=UID:arunts" "/fermilab/Role=Analysis" fermilab
"/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Arun Tadepalli/CN=UID:arunts" "/fermilab/seaquest" fermilab
@ -35145,6 +35155,8 @
"/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Deborah Harris/CN=UID:dharris" "/fermilab/Role=Analysis" fermilab
"/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Deborah Harris/CN=UID:dharris" "/fermilab/minos" fermilab
"/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Deborah Harris/CN=UID:dharris" "/fermilab/minos/Role=Analysis" fermilab
"/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Deborah J. Gulledge/CN=UID:dgulledg" "/fermilab" fermilab
+"/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Deborah J. Gulledge/CN=UID:dgulledg" "/fermilab/Role=Analysis" fermilab
"/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Deborah L. Bonifas/CN=UID:debbieb" "/fermilab" fermilab
"/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Deborah L. Bonifas/CN=UID:debbieb" "/fermilab/Role=Analysis" fermilab
"/DC=gov/DC=fnal/O=Fermilab/OU=Robots/CN=fifegrid/CN=batch/CN=Deborah Sebastian/CN=UID:debseb" "/fermilab" fermilab

Info: /Stage[main]/Vulcan::Dcache/File[/etc/grid-security/grid-vorolemap]: Filebucketed /etc/grid-security/grid-vorolemap to main with sum 762ed131ed3690e3fa84fba70474682a
Notice: /Stage[main]/Vulcan::Dcache/File[/etc/grid-security/grid-vorolemap]/content: content changed '{md5}762ed131ed3690e3fa84fba70474682a' to '{md5}9ebc478d91d42bf29c15ab31c42547c3'
Notice: /Stage[main]/Dcache::Dcachedomain/File[/etc/dcache/poolmanager.conf]/content:
--- /etc/dcache/poolmanager.conf 2015-08-04 18:15:44.567984883 -0500
+++ /tmp/puppet-file20150818-7093-uz7ma6-0 2015-08-18 13:55:51.033770032 -0500
@ -27,6 +27,8 @
psu addto ugroup LFSOnly cms.lfsonly@enstore
psu create ugroup T1DiskEval
psu addto ugroup T1DiskEval cms.T1DiskEval@enstore
+psu create ugroup Resilient
+psu addto ugroup Resilient cms.resilient@enstore
psu create ugroup world-net
psu addto ugroup world-net 0.0.0.0/0.0.0.0
psu create ugroup any-protocol
@ -36,40 +38,29 @ # The pools ... #

-psu create pool w-cmsstor112-disk_itb-disk1
-psu create pool w-cmsstor112-disk_itb-disk2
-psu create pool w-cmsstor112-disk_itb-disk3
-psu create pool w-cmsstor151-disk_itb-disk1
-psu create pool w-cmsstor151-disk_itb-disk2
-psu create pool w-cmsstor151-disk_itb-disk3
psu create pool w-cmsstor155-disk_itb-disk1
psu create pool w-cmsstor155-disk_itb-disk2
psu create pool w-cmsstor155-disk_itb-disk3

+ # # The pool groups ... #
psu create pgroup flushPools
psu create pgroup burningPools
-psu create pgroup readonlyPools

-psu addto pgroup burningPools w-cmsstor112-disk_itb-disk1
-psu addto pgroup burningPools w-cmsstor112-disk_itb-disk2
-psu addto pgroup burningPools w-cmsstor112-disk_itb-disk3

psu addto pgroup flushPools w-cmsstor155-disk_itb-disk1
psu addto pgroup flushPools w-cmsstor155-disk_itb-disk2
psu addto pgroup flushPools w-cmsstor155-disk_itb-disk3

-psu addto pgroup readonlyPools w-cmsstor151-disk_itb-disk1
-psu addto pgroup readonlyPools w-cmsstor151-disk_itb-disk2
-psu addto pgroup readonlyPools w-cmsstor151-disk_itb-disk3

#
 # Sections
 #
pm create flush-section
pm create Resilient-section
pm set flush-section -p2p=0.33 -spacecostfactor=0.0 -cpucostfactor=0.0 -max-pnfs-copies=3 -stage-oncost=yes
+pm set Resilient-section -p2p=4.0 -spacecostfactor=0.0 -cpucostfactor=0.0 -max-pnfs-copies=3 -stage-allowed=no -stage-oncost=no
#
 # The links ...
@ -77,12 +68,8 @
psu create link flush-link any-store world-net any-protocol
psu set link flush-link readpref=10 -writepref=10 -cachepref=0 -p2ppref=10 -section=flush-section
psu add link flush-link flushPools

psu create link readonly-link any-store world-net any-protocol
-psu set link readonly-link -readpref=5 -writepref=0 -cachepref=0 -p2ppref=0 -section=flush-section
-psu add link readonly-link readonlyPools

-
+psu create link Resilient-link Resilient world-net any-protocol
+psu set link Resilient-link -readpref=20 -writepref=20 -cachepref=0 -p2ppref=20 -section=Resilient-section
psu set allpoolsactive on

@ -103,3 +90,4 @
#rc set max copies 3
rc set max restore unlimited
rc set sameHostCopy besteffort
+

Info: /Stage[main]/Dcache::Dcachedomain/File[/etc/dcache/poolmanager.conf]: Filebucketed /etc/dcache/poolmanager.conf to main with sum 82b575e6e02bab899e5b68c5317815c1
Notice: /Stage[main]/Dcache::Dcachedomain/File[/etc/dcache/poolmanager.conf]/content: content changed '{md5}82b575e6e02bab899e5b68c5317815c1' to '{md5}b0d1176cbc4be4633d800f8a2fd579f7'
Info: /Stage[main]/Dcache::Dcachedomain/File[/etc/dcache/poolmanager.conf]: Scheduling refresh of Exec[poolmanager reload]
Notice: /Stage[main]/Dcache::Dcachedomain/Exec[poolmanager reload]/returns: Pseudo-terminal will not be allocated because stdin is not a terminal.
Notice: /Stage[main]/Dcache::Dcachedomain/Exec[poolmanager reload]/returns: ssh: connect to host cmsstor152.fnal.gov port 22223: Connection refused
Error: /Stage[main]/Dcache::Dcachedomain/Exec[poolmanager reload]: Failed to call refresh: /bin/echo -e 'cd PoolManager
reload -yes
..
logoff
' | ssh -1 -i /etc/dcache/admin/server_key -c blowfish -p 22223 cmsstor152.fnal.gov -l admin returned 255 instead of one of [0]
Error: /Stage[main]/Dcache::Dcachedomain/Exec[poolmanager reload]: /bin/echo -e 'cd PoolManager
reload -yes
..
logoff
' | ssh -1 -i /etc/dcache/admin/server_key -c blowfish -p 22223 cmsstor152.fnal.gov -l admin returned 255 instead of one of [0]
Notice: /Stage[main]/Dcache/Service[dcache-server]/ensure: ensure changed 'stopped' to 'running'
Info: /Stage[main]/Dcache/Service[dcache-server]: Unscheduling refresh on Service[dcache-server]
Notice: /Stage[main]/Postgresql::Server/Service[postgresqld]/ensure: ensure changed 'stopped' to 'running'
Info: /Stage[main]/Postgresql::Server/Service[postgresqld]: Unscheduling refresh on Service[postgresqld]
Notice: Finished catalog run in 38.23 seconds
[root@cmsstor152 ~]# service dcache-server stop
Stopping dCacheDomain 0 1 done
Stopping adminDoorDomain 0 done
Stopping httpdDomain 0 1 done
Stopping billingDomain 0 1 done
Stopping dirDomain 0 done
[root@cmsstor152 ~]# service postgresql-9.2 stop
Stopping postgresql-9.2 service: [ OK ]
[root@cmsstor152 ~]# service postgresql-9.2 start
Starting postgresql-9.2 service: [ OK ]
[root@cmsstor152 ~]# service dcache-server start
Starting dirDomain done
Starting billingDomain done
Starting httpdDomain done
Starting adminDoorDomain done
Starting dCacheDomain done
[root@cmsstor152 ~]#
[root@cmsstor152 ~]# service dcache-server stop
Stopping dCacheDomain 0 1 done
Stopping adminDoorDomain 0 done
Stopping httpdDomain 0 1 done
Stopping billingDomain 0 1 done
Stopping dirDomain 0 done
[root@cmsstor152 ~]# service postgresql-9.2 stop
Stopping postgresql-9.2 service: [ OK ]
[root@cmsstor152 ~]# service postgresql-9.2 start
Starting postgresql-9.2 service: [ OK ]
[root@cmsstor152 ~]# service dcache-server start
Starting dirDomain done
Starting billingDomain done
Starting httpdDomain done
Starting adminDoorDomain done
Starting dCacheDomain done
[root@cmsstor152 ~]# puppet agent -t
Warning: Local environment: "production" doesn't match server specified node environment "natalia_fix_dbdisk_mounts_on_testbed", switching agent to "natalia_fix_dbdisk_mounts_on_testbed".
Info: Retrieving pluginfacts
Info: Retrieving plugin
Info: Loading facts in /var/lib/puppet/lib/facter/iptables_version.rb
Info: Loading facts in /var/lib/puppet/lib/facter/cvmfspartsize.rb
Info: Loading facts in /var/lib/puppet/lib/facter/pe_version.rb
Info: Loading facts in /var/lib/puppet/lib/facter/afs_cache_size.rb
Info: Loading facts in /var/lib/puppet/lib/facter/os_maj_version.rb
Info: Loading facts in /var/lib/puppet/lib/facter/code_server.rb
Info: Loading facts in /var/lib/puppet/lib/facter/cvmfsversion.rb
Info: Loading facts in /var/lib/puppet/lib/facter/facter_dot_d.rb
Info: Loading facts in /var/lib/puppet/lib/facter/postgres_default_version.rb
Info: Loading facts in /var/lib/puppet/lib/facter/certificate_facts.rb
Info: Loading facts in /var/lib/puppet/lib/facter/root_home.rb
Info: Loading facts in /var/lib/puppet/lib/facter/concat_basedir.rb
Info: Loading facts in /var/lib/puppet/lib/facter/iptables_persistent_version.rb
Info: Loading facts in /var/lib/puppet/lib/facter/rsyslog_version.rb
Info: Loading facts in /var/lib/puppet/lib/facter/puppet_vardir.rb
Info: Loading facts in /var/lib/puppet/lib/facter/ip6tables_version.rb
Info: Caching catalog for cmsstor152.fnal.gov
Info: Applying configuration version '1439924646'
Notice: Finished catalog run in 26.16 seconds
[root@cmsstor152 ~]#

#24 Updated by Natalia Ratnikova over 4 years ago

ssh login is not possible.
Login from console

[root@cmsstor153 ~]# df
Filesystem 1K-blocks Used Available Use% Mounted on
/dev/sda3 222630532 26038560 185276312 13% /
[root@cmsstor153 ~]# cat /etc/fstab
  1. HEADER: This file was autogenerated at Thu Feb 12 11:21:51 -0600 2015
  2. HEADER: by puppet. While it can still be managed manually, it
  3. HEADER: is definitely not recommended.
#
  1. /etc/fstab
  2. Created by anaconda on Wed May 21 21:41:27 2014 #
  3. Accessible filesystems, by reference, are maintained under '/dev/disk'
  4. See man pages fstab(5), findfs(8), mount(8) and/or blkid(8) for more info #
    UUID=5506b6b3-44d7-4bab-9ed3-5d8990be8adc / ext4 defaults 11
    UUID=3280a196-4b2b-40ed-8715-e5f0b6afe8a3 /boot ext4 defaults 12
    UUID=601e4e2d-a62c-41dd-bf78-a43780599ee1 swap swap defaults 00
    tmpfs /dev/shm tmpfs defaults 0 0
    devpts /dev/pts devpts gid=5,mode=620 0 0
    sysfs /sys sysfs defaults 0 0
    proc /proc proc defaults 0 0
    #LABEL=DBdisk /var/lib/pgsql-puppet ext4 defaults 1 1
    LABEL=DBdisk /var/lib/pgsql-puppet ext4 defaults 1 1

[root@cmsstor153 ~]# lsblk -o name,mountpoint,label,size,uuid
NAME MOUNTPOINT LABEL SIZE UUID
sda 232.9G
├─sda1 /boot 1G 3280a196-4b2b-40ed-8715-e5f0b6afe8a3
├─sda2 16G 601e4e2d-a62c-41dd-bf78-a43780599ee1
└─sda3 / 215.8G 5506b6b3-44d7-4bab-9ed3-5d8990be8adc
sr0 1024M
sdb 10.9T 08e06d98-432d-4dd8-a1b8-027390127893
sdc 10.9T 6e3cd6c7-dac1-497c-8416-5d4a967b32c0
sdd 10.9T c88e29f6-cdcf-447b-aa10-6121741703b6

[root@cmsstor153 ~]# xfs_admin -L DBdisk /dev/sdb
writing all SBs
new label = "DBdisk"
[root@cmsstor153 ~]# lsblk -o name,mountpoint,label,size,uuid
NAME MOUNTPOINT LABEL SIZE UUID
sda 232.9G
├─sda1 /boot 1G 3280a196-4b2b-40ed-8715-e5f0b6afe8a3
├─sda2 16G 601e4e2d-a62c-41dd-bf78-a43780599ee1
└─sda3 / 215.8G 5506b6b3-44d7-4bab-9ed3-5d8990be8adc
sr0 1024M
sdb DBdisk 10.9T 08e06d98-432d-4dd8-a1b8-027390127893
sdc 10.9T 6e3cd6c7-dac1-497c-8416-5d4a967b32c0
sdd 10.9T c88e29f6-cdcf-447b-aa10-6121741703b6

puppet does not run because the system is read-only mounted.

Reboot the node => it should work now that DBdisk label exists, no ENC change should be necessary.

#25 Updated by Natalia Ratnikova over 4 years ago

[root@cmspnfs1 ~]# lsblk -o name,mountpoint,label,size,uuid
NAME MOUNTPOINT LABEL SIZE UUID
sdb 557.8G
├─sdb1 /boot 1G d50f8aaf-9505-432b-a941-3158b6f57504
├─sdb2 [SWAP] 20G 995f4e67-5c6c-450f-802f-185fa93bbc5d
└─sdb3 / 536.8G 8768b22d-bd96-4acd-b267-dfdbe924f411
sda 278.9G
└─sda1 /storage/local/data1 278.9G d82c49e1-3115-4ac0-ab23-f4d94eab8486
sr0 1024M
[root@cmspnfs1 ~]# cat /etc/fstab
  1. HEADER: This file was autogenerated at Wed Aug 12 11:05:42 -0500 2015
  2. HEADER: by puppet. While it can still be managed manually, it
  3. HEADER: is definitely not recommended.
#
  1. /etc/fstab
  2. Created by anaconda on Wed Nov 26 19:57:49 2014 #
  3. Accessible filesystems, by reference, are maintained under '/dev/disk'
  4. See man pages fstab(5), findfs(8), mount(8) and/or blkid(8) for more info #
    UUID=8768b22d-bd96-4acd-b267-dfdbe924f411 / ext4 defaults 1
    UUID=d50f8aaf-9505-432b-a941-3158b6f57504 /boot ext4 defaults 2
    UUID=d82c49e1-3115-4ac0-ab23-f4d94eab8486 /storage/local/data1 ext4 defaults 1 2
    UUID=995f4e67-5c6c-450f-802f-185fa93bbc5d swap swap defaults 0
    tmpfs /dev/shm tmpfs defaults 0 0
    devpts /dev/pts devpts gid=5,mode=620 0 0
    sysfs /sys sysfs defaults 0 0
    proc /proc proc defaults 0 0
    #LABEL=DBdisk /var/lib/pgsql-puppet ext4 defaults 1 1
    #LABEL=DBdisk /var/lib/pgsql-puppet ext4 defaults 1 1
    #LABEL=DBdisk /var/lib/pgsql-puppet ext4 defaults 1 1
    LABEL=DBdisk /var/lib/pgsql-puppet ext4 defaults 1 1
    [root@cmspnfs1 ~]#
    [root@cmspnfs1 ~]# mount
    /dev/sdb3 on / type ext4 (rw)
    proc on /proc type proc (rw)
    sysfs on /sys type sysfs (rw)
    devpts on /dev/pts type devpts (rw,gid=5,mode=620)
    tmpfs on /dev/shm type tmpfs (rw)
    /dev/sdb1 on /boot type ext4 (rw)
    /dev/sda1 on /storage/local/data1 type ext4 (rw)
    none on /proc/sys/fs/binfmt_misc type binfmt_misc (rw)
    [root@cmspnfs1 ~]# df
    Filesystem 1K-blocks Used Available Use% Mounted on
    /dev/sdb3 553860088 42085904 483633080 9% /
    tmpfs 24707184 0 24707184 0% /dev/shm
    /dev/sdb1 999320 90016 856876 10% /boot
    /dev/sda1 287701384 64424 273015932 1% /storage/local/data1

Check if puppet runs cleanly, first with noop, as we do not want to break the node again.

Runs cleanly. Although DBdisk label does not exist:
[root@cmspnfs1 ~]# lsblk -o name,mountpoint,label,size,uuid
NAME MOUNTPOINT LABEL SIZE UUID
sdb 557.8G
├─sdb1 /boot 1G d50f8aaf-9505-432b-a941-3158b6f57504
├─sdb2 [SWAP] 20G 995f4e67-5c6c-450f-802f-185fa93bbc5d
└─sdb3 / 536.8G 8768b22d-bd96-4acd-b267-dfdbe924f411
sda 278.9G
└─sda1 /storage/local/data1 278.9G d82c49e1-3115-4ac0-ab23-f4d94eab8486
sr0 1024M

Try if persists the reboot:

[root@cmspnfs1 ~]# reboot

Broadcast message from
(/dev/pts/0) at 14:57 ...

The system is going down for reboot NOW!
[root@cmspnfs1 ~]# Connection to cmspnfs1 closed by remote host.
Connection to cmspnfs1 closed.

Node comes up in a maintenance mode again on reboot with readonly system.
  1. HEADER: This file was autogenerated at Wed Aug 12 11:05:42 -0500 2015
  2. HEADER: by puppet. While it can still be managed manually, it
  3. HEADER: is definitely not recommended.
[root@cmspnfs1 ~]# touch nnnn 
touch: cannot touch `nnnn': Read-only file system
[root@cmspnfs1 ~]# cat /etc/fstab
#
  1. /etc/fstab
  2. Created by anaconda on Wed Nov 26 19:57:49 2014 #
  3. Accessible filesystems, by reference, are maintained under '/dev/disk'
  4. See man pages fstab(5), findfs(8), mount(8) and/or blkid(8) for more info #
    UUID=8768b22d-bd96-4acd-b267-dfdbe924f411 / ext4 defaults 11
    UUID=d50f8aaf-9505-432b-a941-3158b6f57504 /boot ext4 defaults 12
    UUID=d82c49e1-3115-4ac0-ab23-f4d94eab8486 /storage/local/data1 ext4 defaults 1 2
    UUID=995f4e67-5c6c-450f-802f-185fa93bbc5d swap swap defaults 00
    tmpfs /dev/shm tmpfs defaults 0 0
    devpts /dev/pts devpts gid=5,mode=620 0 0
    sysfs /sys sysfs defaults 0 0
    proc /proc proc defaults 0 0
    #LABEL=DBdisk /var/lib/pgsql-puppet ext4 defaults 1 1
    #LABEL=DBdisk /var/lib/pgsql-puppet ext4 defaults 1 1
    #LABEL=DBdisk /var/lib/pgsql-puppet ext4 defaults 1 1
    LABEL=DBdisk /var/lib/pgsql-puppet ext4 defaults 1 1
    [root@cmspnfs1 ~]# df
    Filesystem 1K-blocks Used Available Use% Mounted on
    /dev/sdb3 553860088 42107304 483611680 9% /
    [root@cmspnfs1 ~]# lsblk o name,mountpoint,label,size,uuid
    NAME MOUNTPOINT LABEL SIZE UUID
    sdb 557.8G
    ├─sdb1 1G d50f8aaf-9505-432b-a941-3158b6f57504
    ├─sdb2 20G 995f4e67-5c6c-450f-802f-185fa93bbc5d
    └─sdb3 / 536.8G 8768b22d-bd96-4acd-b267-dfdbe924f411
    sda 278.9G
    └─sda1 278.9G d82c49e1-3115-4ac0-ab23-f4d94eab8486
    sr0 1024M
    [root@cmspnfs1 ~]# mount /dev/sda1 /mnt
    EXT4-fs (sda1): mounted filesystem with ordered data mode. Opts:
    [root@cmspnfs1 ~]# cd /mnt
    [root@cmspnfs1 mnt]# ls
    copyOnceWalls-disk.sh delNewWalls.sh lost+found
    copyWalls-disk.sh delOnceNewWalls.sh
    [root@cmspnfs1 mnt]# ls -altr
    total 40
    -rwxr-xr-x 1 cmsprod us_cms 418 Jul 2 2014 delOnceNewWalls.sh
    -rwxr-xr-x 1 cmsprod us_cms 452 Jul 2 2014 delNewWalls.sh
    -rwxr-xr-x 1 cmsprod us_cms 677 Jul 2 2014 copyWalls-disk.sh
    -rwxr-xr-x 1 cmsprod us_cms 627 Jul 2 2014 copyOnceWalls-disk.sh
    drwx-----
    . 2 root root 16384 Nov 26 2014 lost+found
    drwxr-xr-x. 3 root root 4096 Dec 11 2014 .
    dr-xr-xr-x. 25 root root 4096 Aug 18 14:57 ..


Also available in: Atom PDF