-
Notifications
You must be signed in to change notification settings - Fork 1
/
Copy pathACCESS_Archiver.sh
executable file
·257 lines (232 loc) · 6.69 KB
/
ACCESS_Archiver.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
#!/bin/bash
source /etc/profile.d/modules.sh
module purge
module load pbs
set -a
#########################
#
# This is the ACCESS Archiver, v1.1
# 21/01/2022
#
# Developed by Chloe Mackallah, CSIRO Aspendale
#
#########################
# NON-STANDARD USER OPTIONS
#split zonal/non-zonal files - CM2 DAMIP and CM2-Chem runs only
zonal=false
#reduce daily plev19 data to plev8 - ESM CMIP6 runs only
plev8=false
#convert_unknown = [true, false]; convert unrecognised pp files that are found?
convert_unknown=false
#########################
# DO NOT EDIT - FIXED TASKS
#check wrapper is used
if [ -z $arch_dir ]; then
echo "no experiment settings"; exit
fi
if [ -z $here ]; then
here=$( pwd )
fi
mkdir -p $here/tmp/$loc_exp
rm -f $here/tmp/$loc_exp/*
#identify NCI project of base_dir
IFS='/'
read -a base_dir_split <<< $base_dir
for bdse in ${base_dir_split[@]}; do
if [[ $bdse == '' ]] || [[ $bdse == ' ' ]] || [[ $bdse == 'g' ]] || \
[[ $bdse == 'data' ]] || [[ $bdse == 'scratch' ]]; then
continue
fi
base_grp=$bdse
break
done
#identify NCI project of arch_dir
read -a arch_dir_split <<< $arch_dir
for adse in ${arch_dir_split[@]}; do
if [[ $adse == '' ]] || [[ $adse == ' ' ]] || [[ $adse == 'g' ]] || \
[[ $adse == 'data' ]] || [[ $adse == 'scratch' ]]; then
continue
fi
arch_grp=$adse
break
done
IFS=' '
mkdir -p $arch_dir/$loc_exp/{history/atm/netCDF,restart/atm}
chgrp -R $arch_grp $arch_dir/$loc_exp
if [[ $access_version == *chem ]]; then
zonal=true
fi
#
#########################
# PRINT DETAILS TO SCREEN
echo -e "\n==== ACCESS_Archiver ===="
echo "here: $here"
echo "compute project: $comp_proj"
echo "base directory: $base_dir"
echo "archive directory: $arch_dir"
echo "local experiment: $loc_exp"
echo "access version: $access_version"
echo "subdaily atm data: $subdaily"
#########################
# RUN SUBROUTINES
if [[ $access_version == om2 ]]; then
${here}/subroutines/find_files_om2.sh
elif [[ $access_version == esmpayu ]]; then
${here}/subroutines/find_files_payu.sh
else
${here}/subroutines/find_files.sh
fi
if [[ -n $first_year || -n $last_year ]]; then
first_year=${first_year:-1}
last_year=${last_year:-9999}
echo "Archiving restricted to years" $first_year ":" $last_year
for file in $here/tmp/$loc_exp/*csv; do
python3 $here/subroutines/restrict_years.py $first_year $last_year $file
if [[ $? != 0 ]]; then
echo "Error subsetting csv file"
exit 1
fi
done
fi
echo -e "\n---- Setting up jobs ----"
#---- um2nc parallel job ----#
cp $here/subroutines/run_um2nc.py $here/tmp/$loc_exp/run_um2nc.py
#
cat << EOF > $here/tmp/$loc_exp/job_um2nc.qsub.sh
#!/bin/bash
#PBS -P ${comp_proj}
#PBS -l walltime=48:00:00,ncpus=24,mem=190Gb
#PBS -l wd
#PBS -l storage=scratch/${base_grp}+gdata/${base_grp}+scratch/${arch_grp}+gdata/${arch_grp}+gdata/hh5+gdata/access
#PBS -q normal
#PBS -j oe
#PBS -N ${loc_exp}_um2nc
module purge
module use /g/data/hh5/public/modules
module use ~access/modules
module load cdo
module load nco
module load pythonlib/um2netcdf4/2.0
set -a
ncpus=\$PBS_NCPUS
here=$here
base_dir=$base_dir
arch_dir=$arch_dir
arch_grp=$arch_grp
loc_exp=$loc_exp
zonal=$zonal
access_version=$access_version
plev8=$plev8
ncexists=$ncexists
subdaily=$subdaily
convert_unknown=$convert_unknown
UMDIR=/projects/access/umdir
echo -e "\n==== ACCESS_Archiver -- um2netcdf_iris ===="
echo "base dir: $base_dir"
echo "arch dir: $arch_dir"
echo "local exp: $loc_exp"
echo "access version: $access_version"
python -W ignore $here/tmp/$loc_exp/run_um2nc.py
EOF
if [[ $access_version != om2 ]]; then
ls $here/tmp/$loc_exp/job_um2nc.qsub.sh
chmod +x $here/tmp/$loc_exp/job_um2nc.qsub.sh
qsub $here/tmp/$loc_exp/job_um2nc.qsub.sh
fi
#----------------------------#
#exit
#---- mppnccombine job --------------#
cp $here/subroutines/mppnccomb_check.sh $here/tmp/$loc_exp/mppnccomb_check.sh
#
cat << EOF > $here/tmp/$loc_exp/job_mppnc.qsub.sh
#!/bin/bash
#PBS -P ${comp_proj}
#PBS -l walltime=48:00:00,ncpus=1,mem=12Gb
#PBS -l wd
#PBS -l storage=scratch/${base_grp}+gdata/${base_grp}+scratch/${arch_grp}+gdata/${arch_grp}+gdata/hh5+gdata/access
#PBS -q normal
#PBS -j oe
#PBS -N ${loc_exp}_mppnc
module purge
module use /g/data/hh5/public/modules
module use ~access/modules
module load cdo
module load nco
module load conda/analysis3
set -a
here=$here
base_dir=$base_dir
arch_dir=$arch_dir
arch_grp=$arch_grp
loc_exp=$loc_exp
access_version=$access_version
echo -e "\n==== ACCESS_Archiver -- mppnccombine ===="
echo "base dir: $base_dir"
echo "arch dir: $arch_dir"
echo "local exp: $loc_exp"
echo "access version: $access_version"
$here/tmp/$loc_exp/mppnccomb_check.sh
EOF
if [[ $access_version != om2 ]] || [[ $access_version != *amip ]]; then #|| [[ $access_version != *chem ]]; then
ls $here/tmp/$loc_exp/job_mppnc.qsub.sh
chmod +x $here/tmp/$loc_exp/job_mppnc.qsub.sh
#qsub $here/tmp/$loc_exp/job_mppnc.qsub.sh
fi
#----------------------------#
#exit
#---- copy job --------------#
if [[ $access_version == *payu* ]]; then
cp $here/subroutines/cp_hist_payu.sh $here/tmp/$loc_exp/cp_hist.sh
cp $here/subroutines/cp_rest_payu.sh $here/tmp/$loc_exp/cp_rest.sh
elif [[ $access_version == om2 ]]; then
cp $here/subroutines/link_arch_om2.sh $here/tmp/$loc_exp/link_arch_om2.sh
elif [[ $access_version == *amip ]]; then #|| [[ $access_version == *chem ]]; then
cp $here/subroutines/cp_rest.sh $here/tmp/$loc_exp/cp_rest.sh
else
cp $here/subroutines/cp_hist.sh $here/tmp/$loc_exp/cp_hist.sh
cp $here/subroutines/cp_rest.sh $here/tmp/$loc_exp/cp_rest.sh
fi
#
cat << EOF > $here/tmp/$loc_exp/job_arch.qsub.sh
#!/bin/bash
#PBS -P ${comp_proj}
#PBS -l walltime=48:00:00,ncpus=1,mem=8Gb
#PBS -l wd
#PBS -l storage=scratch/${base_grp}+gdata/${base_grp}+scratch/${arch_grp}+gdata/${arch_grp}+gdata/hh5+gdata/access
#PBS -q normal
#PBS -j oe
#PBS -N ${loc_exp}_arch
##PBS -W depend=beforeany:$mppn
module purge
module use /g/data/hh5/public/modules
module use ~access/modules
module load cdo
module load nco
module load conda/analysis3
set -a
here=$here
base_dir=$base_dir
arch_dir=$arch_dir
arch_grp=$arch_grp
loc_exp=$loc_exp
access_version=$access_version
echo -e "\n==== ACCESS_Archiver -- copy_job ===="
echo "base dir: $base_dir"
echo "arch dir: $arch_dir"
echo "local exp: $loc_exp"
echo "access version: $access_version"
if [[ $access_version == om2 ]]; then
$here/tmp/$loc_exp/link_arch_om2.sh
elif [[ $access_version == *amip ]]; then #|| [[ $access_version == *chem ]]; then
$here/tmp/$loc_exp/cp_rest.sh
else
$here/tmp/$loc_exp/cp_hist.sh
$here/tmp/$loc_exp/cp_rest.sh
qsub $here/tmp/$loc_exp/job_mppnc.qsub.sh
fi
EOF
ls $here/tmp/$loc_exp/job_arch.qsub.sh
chmod +x $here/tmp/$loc_exp/job_arch.qsub.sh
qsub $here/tmp/$loc_exp/job_arch.qsub.sh
#----------------------------#
echo -e "\n---- DONE: $loc_exp ----"