Project

General

Profile

grid.sh

grid wrapper example - Marcus O'flaherty, 02/11/2017 02:04 PM

 
1
#!/bin/bash 
2
#PROCESSOFFSET=10	# use this to offset PROCESS number, if you need to, for some reason...
3
unset PROCESSOFFSET
4

    
5
echo "setting up software base"
6
export CODE_BASE=/grid/fermiapp/products
7
#export CODE_BASE=/cvmfs/fermilab.opensciencegrid.org/products
8
source ${CODE_BASE}/common/etc/setup
9
export PRODUCTS=${PRODUCTS}:${CODE_BASE}/larsoft
10

    
11
setup geant4 v4_10_1_p02a -q e9:debug:qt
12

    
13
setup genie v2_12_0a -q debug:e10:r6
14
setup genie_xsec v2_12_0 -q DefaultPlusMECWithNC
15
setup genie_phyopt v2_12_0 -q dkcharmtau
16
setup -q debug:e10 xerces_c v3_1_3
17

    
18
setup -q debug:e10:nu root v6_06_08
19
source ${CODE_BASE}/larsoft/root/v6_06_08/Linux64bit+2.6-2.12-e10-nu-debug/bin/thisroot.sh
20

    
21
setup -q debug:e9 clhep v2_2_0_8
22
setup cmake     #sets up 2_8_8
23
setup fife_utils
24

    
25
export XERCESROOT=${CODE_BASE}/larsoft/xerces_c/v3_1_3/Linux64bit+2.6-2.12-e10-debug
26
export G4SYSTEM=Linux-g++
27
export ROOT_PATH=${CODE_BASE}/larsoft/root/v6_06_08/Linux64bit+2.6-2.12-e10-nu-debug/cmake
28
export GEANT4_PATH=${GEANT4_FQ_DIR}/lib64/Geant4-10.1.2
29
export GEANT4_MAKEFULL_PATH=${GEANT4_DIR}/${GEANT4_VERSION}/source/geant4.10.01.p02
30
export ROOT_INCLUDE_PATH=${ROOT_INCLUDE_PATH}:${GENIE}/../include/GENIE
31
export ROOT_LIBRARY_PATH=${ROOT_LIBRARY_PATH}:${GENIE}/../lib
32
#export LD_LIBRARY_PATH=/annie/app/users/moflaher/wcsim/wcsim:$LD_LIBRARY_PATH
33
#export ROOT_INCLUDE_PATH=/annie/app/users/moflaher/wcsim/wcsim/include:$ROOT_INCLUDE_PATH
34

    
35

    
36
# copy the source files
37
SOURCEFILEDIR=/pnfs/annie/scratch/users/moflaher
38
SOURCEFILEZIP=wcsim.tar.gz
39
echo "searching for source files in ${SOURCEFILEDIR}/${SOURCEFILEZIP}"
40
echo "ifdh ls ${SOURCEFILEDIR}"
41
ifdh ls ${SOURCEFILEDIR}
42
ifdh ls ${SOURCEFILEDIR}/${SOURCEFILEZIP} 1>/dev/null 2>&1
43
if [ $? -eq 0 ]; then
44
  echo "copying source files"
45
  ifdh cp -D ${SOURCEFILEDIR}/${SOURCEFILEZIP} .
46
else 
47
  echo "source file zip not found in ${SOURCEFILEDIR}!"
48
fi
49
if [ ! -f ${SOURCEFILEZIP} ]; then
50
  echo "could not copy source file zip from ${SOURCEFILEDIR}, trying alternatives"
51
  for TARLOC in /pnfs/annie/persistent/users/moflaher \
52
                /annie/app/users/moflaher/wcsim
53
  do
54
    echo "try copy from ${TARLOC}"
55
    ifdh cp -D ${TARLOC}/${SOURCEFILEZIP} .
56
    if [ ! -f ${SOURCEFILEZIP} ]; then continue; fi
57
  done
58
  if [ ! -f ${SOURCEFILEZIP} ]; then
59
    echo "source file zip not found in any accessible locations!!!"
60
    exit 11
61
  fi
62
fi
63

    
64
# extract and compile the application
65
echo "unzipping source files"
66
tar zxvf ${SOURCEFILEZIP}
67

    
68
echo "compiling application"
69
mkdir build
70
cd wcsim
71
make rootcint
72
make 
73
cp src/WCSimRootDict_rdict.pcm ./
74
cd ../build
75
cmake ../wcsim
76
make
77
rm libWCSimRootDict.rootmap
78
if [ ! -x ./WCSim ]; then
79
    if [ -a ./WCSim ]; then
80
        chmod +x ./WCSim
81
        hash -r
82
    fi
83
fi
84
if [ ! -x ./WCSim ]; then
85
    echo "something failed in compilation?! WCSim not found!"
86
    exit 12
87
fi
88

    
89
# copy the list of input files
90
INPUTFILELISTDIR=/pnfs/annie/scratch/users/moflaher
91
INPUTFILELISTNAME=filenums.txt
92
echo "searching for input file list in ${INPUTFILELISTDIR}/${INPUTFILELISTNAME}"
93
echo "ifdh ls ${INPUTFILELISTDIR}"
94
ifdh ls ${INPUTFILELISTDIR}
95
ifdh ls ${INPUTFILELISTDIR}/${INPUTFILELISTNAME} 1>/dev/null 2>&1
96
if [ $? -eq 0 ]; then
97
    echo "copying input file list"
98
    ifdh cp -D ${INPUTFILELISTDIR}/${INPUTFILELISTNAME} .   # list of input files
99
else
100
    echo "${INPUTFILELISTNAME} not found in ${INPUTFILELISTDIR}! Trying alternatives..."
101
    for INLISTLOC in /pnfs/annie/persistent/users/moflaher \
102
                     /annie/app/users/moflaher/wcsim
103
    do
104
      echo "try copy from ${INLISTLOC}"
105
      ifdh cp -D ${INLISTLOC}/${INPUTFILELISTNAME} .
106
      if [ ! -f ${INPUTFILELISTNAME} ]; then continue; fi
107
    done
108
    if [ ! -f ${INPUTFILELISTNAME} ]; then
109
      echo "input file list not found in any accessible locations!!!"
110
      exit 13
111
    fi
112
fi
113

    
114
# calculate the input file to use
115
let THECOUNTER=${PROCESS}+${PROCESSOFFSET}+1
116
THENUM=`less filenums.txt | sed -n ${THECOUNTER},${THECOUNTER}p`
117
echo "this job has process ${PROCESS}, and will use file num ${THENUM}"
118

    
119
# copy the input files
120
#DIRTDIR=/pnfs/annie/persistent/users/rhatcher/g4dirt
121
DIRTDIR=/pnfs/annie/persistent/users/moflaher/g4dirt
122
DIRTFILE=annie_tank_flux.${THENUM}.root
123
GENIEDIR=/pnfs/annie/persistent/users/rhatcher/genie
124
GENIEFILE=gntp.${THENUM}.ghep.root
125

    
126
echo "copying the input files ${DIRTDIR}/${DIRTFILE} and ${GENIEDIR}/${GENIEFILE}"
127
ifdh cp -D ${DIRTDIR}/${DIRTFILE} .
128
ifdh cp -D ${GENIEDIR}/${GENIEFILE} .
129
if [ ! -f ${DIRTFILE} ]; then echo "dirt file not found!!!"; exit 14; fi
130
if [ ! -f ${GENIEFILE} ]; then echo "genie file not found!!!"; exit 15; fi
131

    
132
echo "writing primaries_directory.mac"
133
echo "/mygen/neutrinosdirectory ${PWD}/gntp.*.ghep.root" >  primaries_directory.mac
134
echo "/mygen/primariesdirectory ${PWD}/annie_tank_flux.*.root" >>  primaries_directory.mac
135
echo "/run/beamOn 10000" >> WCSim.mac   # will end the run as rqd if there are fewer events in the input file
136

    
137
# run executable here, rename the output file
138
NOWS=`date "+%s"`
139
DATES=`date "+%Y-%m-%d %H:%M:%S"`
140
echo "checkpoint start @ ${DATES} s=${NOWS}"
141
echo " "
142

    
143
./WCSim WCSim.mac
144

    
145
echo " "
146
NOWF=`date "+%s"`
147
DATEF=`date "+%Y-%m-%d %H:%M:%S"`
148
let DS=${NOWF}-${NOWS}
149
echo "checkpoint finish @ ${DATEF} s=${NOWF}  ds=${DS}"
150
echo " "
151

    
152
OUTDIR=/pnfs/annie/persistent/users/moflaher/wcsim
153
echo "copying the output files to ${OUTDIR}"
154

    
155
# copy back the output files
156
DATESTRING=$(date)      # contains a bunch of spaces, dont use in filenames
157
for file in wcsim_*; do
158
        tmp=${file%.*}  # strip .root extension
159
        OUTFILE=${tmp}.${THENUM}.root
160
        echo "moving ${file} to ${OUTFILE}"
161
        mv ${file} ${OUTFILE}
162
        echo "copying ${OUTFILE} to ${OUTDIR}"
163
        ifdh cp -D ${OUTFILE} ${OUTDIR}
164
        if [ $? -ne 0 ]; then echo "something went wrong with the copy?!"; fi
165
done
166

    
167
# clean things up
168
cd ..
169
rm -rf wcsim
170
rm -rf build
171
rm -rf ${SOURCEFILEZIP}