[Dart-dev] [6872] DART/trunk/models: Adding support for the round-robin task assignment.
nancy at ucar.edu
nancy at ucar.edu
Fri Apr 4 14:11:25 MDT 2014
Revision: 6872
Author: thoar
Date: 2014-04-04 14:11:25 -0600 (Fri, 04 Apr 2014)
Log Message:
-----------
Adding support for the round-robin task assignment.
CLM also handles inflation files with the initial cookie strategy now.
Modified Paths:
--------------
DART/trunk/models/CESM/shell_scripts/cam_assimilate.csh
DART/trunk/models/CESM/shell_scripts/clm_assimilate.csh
DART/trunk/models/CESM/shell_scripts/pop_assimilate.csh
DART/trunk/models/POP/shell_scripts/assimilate.csh
DART/trunk/models/cam/shell_scripts/assimilate.csh
DART/trunk/models/clm/shell_scripts/assimilate.csh
-------------- next part --------------
Modified: DART/trunk/models/CESM/shell_scripts/cam_assimilate.csh
===================================================================
--- DART/trunk/models/CESM/shell_scripts/cam_assimilate.csh 2014-04-04 18:23:35 UTC (rev 6871)
+++ DART/trunk/models/CESM/shell_scripts/cam_assimilate.csh 2014-04-04 20:11:25 UTC (rev 6872)
@@ -37,6 +37,8 @@
set COPY = 'cp -fv --preserve=timestamps'
set LINK = 'ln -fvs'
set REMOVE = 'rm -fr'
+ set TASKS_PER_NODE = `echo $LSB_SUB_RES_REQ | sed -ne '/ptile/s#.*\[ptile=\([0-9][0-9]*\)]#\1#p'`
+ setenv MP_DEBUG_NOTIMEOUT yes
set BASEOBSDIR = /glade/p/image/Observations/ACARS
set LAUNCHCMD = mpirun.lsf
@@ -64,7 +66,6 @@
#-------------------------------------------------------------------------
set FILE = `head -n 1 rpointer.atm_0001`
-set FILE = $FILE:t
set FILE = $FILE:r
set ATM_DATE_EXT = `echo $FILE:e`
set ATM_DATE = `echo $FILE:e | sed -e "s#-# #g"`
@@ -138,6 +139,19 @@
echo "`date` -- END COPY BLOCK"
+# If possible, use the round-robin approach to deal out the tasks.
+# Since the ensemble manager is not used by cam_to_dart or dart_to_cam,
+# it is OK to set it here and have it used by all routines.
+
+if ($?TASKS_PER_NODE) then
+ if ($#TASKS_PER_NODE > 0) then
+ ${COPY} input.nml input.nml.$$
+ sed -e "s#layout.*#layout = 2#" \
+ -e "s#tasks_per_node.*#tasks_per_node = $TASKS_PER_NODE#" input.nml.$$ >! input.nml
+ ${REMOVE} input.nml.$$
+ endif
+endif
+
#=========================================================================
# Block 2: Stage the files needed for SAMPLING ERROR CORRECTION
#
@@ -191,19 +205,23 @@
# so they should be named something like prior_inflate_restart.YYYY-MM-DD-SSSSS
#
# NOTICE: inf_initial_from_restart and inf_sd_initial_from_restart are somewhat
-# problematic. During the bulk of an experiment, these should be FALSE, since
+# problematic. During the bulk of an experiment, these should be TRUE, since
# we want to read existing inflation files. However, the first assimilation
-# might need these to be TRUE and then subsequently be set to FALSE.
-# There are two ways to handle this.
-# 1) Create the initial files offline with values of unity by using
-# 'fill_inflation_restart' and stage them with the appropriate names
-# in the RUNDIR.
+# might need these to be FALSE and then subsequently be set to TRUE.
+# There are two ways to handle this:
+#
+# 1) Create the initial files offline (perhaps with 'fill_inflation_restart')
+# and stage them with the appropriate names in the RUNDIR.
+# You must manually remove the cam_inflation_cookie file
+# from the RUNDIR in this case.
+# - OR -
# 2) create a cookie file called RUNDIR/cam_inflation_cookie
# The existence of this file will cause this script to set the
# namelist appropriately. This script will 'eat' the cookie file
# to prevent this from happening for subsequent executions. If the
# inflation file does not exist for them, and it needs to, this script
-# should die.
+# should die. The CESM_DART_config script automatically creates a cookie
+# file to support this option.
#
# The strategy is to use the LATEST inflation file from the CESM 'rundir'.
# After an assimilation, the new inflation values/files will be moved to
@@ -331,7 +349,7 @@
${REMOVE} ../cam_inflation_cookie
#=========================================================================
-# Block 4: Convert N CAM restart files to DART initial condition files.
+# Block 4: Convert N CAM initial files to DART initial condition files.
# cam_to_dart is serial code, we can do all of these at the same time
# as long as we can have unique namelists for each of them.
#
@@ -359,22 +377,19 @@
mkdir -p $MYTEMPDIR
cd $MYTEMPDIR
- # Turns out the .h0. files are timestamped with the START of the
- # run, which is *not* ATM_DATE_EXT ... I just link to a whatever
- # is convenient (since the info is static).
# make sure there are no old output logs hanging around
$REMOVE output.${member}.cam_to_dart
- set ATM_INITIAL_FILENAME = `printf ../../${CASE}.cam_%04d.i.${ATM_DATE_EXT}.nc ${member}`
- set ATM_HISTORY_FILENAME = `ls -1t ../../${CASE}.cam*.h0.* | head -n 1`
+ set ATM_INITIAL_FILENAME = `printf ${CASE}.cam_%04d.i.${ATM_DATE_EXT}.nc ${member}`
+ set ATM_HISTORY_FILENAME = `printf ${CASE}.cam_%04d.h0.${ATM_DATE_EXT}.nc ${member}`
set DART_IC_FILENAME = `printf filter_ics.%04d ${member}`
set DART_RESTART_FILE = `printf filter_restart.%04d ${member}`
sed -e "s#dart_ics#../${DART_IC_FILENAME}#" \
-e "s#dart_restart#../${DART_RESTART_FILE}#" < ../input.nml >! input.nml
- ${LINK} $ATM_INITIAL_FILENAME caminput.nc
- ${LINK} $ATM_HISTORY_FILENAME cam_phis.nc
+ ${LINK} ../../$ATM_INITIAL_FILENAME caminput.nc
+ ${LINK} ../../$ATM_HISTORY_FILENAME cam_phis.nc
echo "starting cam_to_dart for member ${member} at "`date`
${EXEROOT}/cam_to_dart >! output.${member}.cam_to_dart &
@@ -400,48 +415,36 @@
# Will result in a set of files : 'filter_restart.xxxx'
#
# DART namelist settings required:
-# &filter_nml: async = 0,
-# &filter_nml: adv_ens_command = "./no_model_advance.csh",
-# &filter_nml: restart_in_file_name = 'filter_ics'
-# &filter_nml: restart_out_file_name = 'filter_restart'
-# &filter_nml: obs_sequence_in_name = 'obs_seq.out'
-# &filter_nml: obs_sequence_out_name = 'obs_seq.final'
-# &filter_nml: init_time_days = -1,
-# &filter_nml: init_time_seconds = -1,
-# &filter_nml: first_obs_days = -1,
-# &filter_nml: first_obs_seconds = -1,
-# &filter_nml: last_obs_days = -1,
-# &filter_nml: last_obs_seconds = -1,
-# &ensemble_manager_nml: single_restart_file_in = .false.
+# &filter_nml: async = 0,
+# &filter_nml: adv_ens_command = "no_advance_script",
+# &filter_nml: restart_in_file_name = 'filter_ics'
+# &filter_nml: restart_out_file_name = 'filter_restart'
+# &filter_nml: obs_sequence_in_name = 'obs_seq.out'
+# &filter_nml: obs_sequence_out_name = 'obs_seq.final'
+# &filter_nml: init_time_days = -1,
+# &filter_nml: init_time_seconds = -1,
+# &filter_nml: first_obs_days = -1,
+# &filter_nml: first_obs_seconds = -1,
+# &filter_nml: last_obs_days = -1,
+# &filter_nml: last_obs_seconds = -1,
+# &ensemble_manager_nml: single_restart_file_in = .false.
+# &ensemble_manager_nml: single_restart_file_out = .false.
#
#=========================================================================
# CAM:static_init_model() always needs a caminput.nc and a cam_phis.nc
# for geometry information, etc.
-set ATM_INITIAL_FILENAME = ../${CASE}.cam_0001.i.${ATM_DATE_EXT}.nc
-set ATM_HISTORY_FILENAME = `ls -1t ../${CASE}.cam*.h0.* | head -n 1`
+set ATM_INITIAL_FILENAME = ${CASE}.cam_0001.i.${ATM_DATE_EXT}.nc
+set ATM_HISTORY_FILENAME = ${CASE}.cam_0001.h0.${ATM_DATE_EXT}.nc
-${LINK} $ATM_INITIAL_FILENAME caminput.nc
-${LINK} $ATM_HISTORY_FILENAME cam_phis.nc
+${LINK} ../$ATM_INITIAL_FILENAME caminput.nc
+${LINK} ../$ATM_HISTORY_FILENAME cam_phis.nc
-# On yellowstone, you can explore task layouts with the following:
-if ( $?LSB_PJL_TASK_GEOMETRY ) then
- setenv ORIGINAL_LAYOUT "${LSB_PJL_TASK_GEOMETRY}"
-
- # setenv GEOMETRY_32_1NODE \
- # "{(0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31)}";
- # setenv LSB_PJL_TASK_GEOMETRY "${GEOMETRY_32_1NODE}"
-endif
-
echo "`date` -- BEGIN FILTER"
${LAUNCHCMD} ${EXEROOT}/filter_cam || exit -7
echo "`date` -- END FILTER"
-if ( $?LSB_PJL_TASK_GEOMETRY ) then
- setenv LSB_PJL_TASK_GEOMETRY "${ORIGINAL_LAYOUT}"
-endif
-
${MOVE} Prior_Diag.nc ../cam_Prior_Diag.${ATM_DATE_EXT}.nc
${MOVE} Posterior_Diag.nc ../cam_Posterior_Diag.${ATM_DATE_EXT}.nc
${MOVE} obs_seq.final ../cam_obs_seq.${ATM_DATE_EXT}.final
@@ -449,7 +452,7 @@
# Accomodate any possible inflation files
# 1) rename file to reflect current date
-# 2) move to CENTRALDIR so the DART INFLATION BLOCK works next time and
+# 2) move to RUNDIR so the DART INFLATION BLOCK works next time and
# that they can get archived.
foreach FILE ( ${PRIOR_INF_OFNAME} ${POSTE_INF_OFNAME} ${PRIOR_INF_DIAG} ${POSTE_INF_DIAG} )
@@ -523,14 +526,8 @@
#-------------------------------------------------------------------------
# Cleanup
-# we (DART) do not need these files, and CESM does not need them either
-# to continue a run. if we remove them here they do not get moved to
-# the short-term archiver.
#-------------------------------------------------------------------------
-# ${REMOVE} ${RUNDIR}/*.rh0.*
-# ${REMOVE} ${RUNDIR}/*.rs1.*
-
echo "`date` -- END CAM_ASSIMILATE"
exit 0
Modified: DART/trunk/models/CESM/shell_scripts/clm_assimilate.csh
===================================================================
--- DART/trunk/models/CESM/shell_scripts/clm_assimilate.csh 2014-04-04 18:23:35 UTC (rev 6871)
+++ DART/trunk/models/CESM/shell_scripts/clm_assimilate.csh 2014-04-04 20:11:25 UTC (rev 6872)
@@ -37,11 +37,36 @@
set COPY = 'cp -fv --preserve=timestamps'
set LINK = 'ln -fvs'
set REMOVE = 'rm -fr'
+ set TASKS_PER_NODE = `echo $LSB_SUB_RES_REQ | sed -ne '/ptile/s#.*\[ptile=\([0-9][0-9]*\)]#\1#p'`
+ setenv MP_DEBUG_NOTIMEOUT yes
set BASEOBSDIR = /glade/p/image/Observations/land
set LAUNCHCMD = mpirun.lsf
breaksw
+ case lone*:
+ # UT lonestar
+ set MOVE = '/bin/mv -fv'
+ set COPY = '/bin/cp -fv --preserve=timestamps'
+ set LINK = '/bin/ln -fvs'
+ set REMOVE = '/bin/rm -fr'
+
+ set BASEOBSDIR = ${WORK}/DART/observations/snow/work/obs_seqs
+ set LAUNCHCMD = mpirun.lsf
+ breaksw
+
+ case la*:
+ # LBNL "lawrencium"
+ set MOVE = 'mv -fv'
+ set COPY = 'cp -fv --preserve=timestamps'
+ set LINK = 'ln -fvs'
+ set REMOVE = 'rm -fr'
+ set TASKS_PER_NODE = $MAX_TASKS_PER_NODE
+
+ set BASEOBSDIR = /your/observation/directory/here
+ set LAUNCHCMD = "mpiexec -n $NTASKS"
+ breaksw
+
default:
# NERSC "hopper"
set MOVE = 'mv -fv'
@@ -64,7 +89,6 @@
#-------------------------------------------------------------------------
set FILE = `head -n 1 rpointer.lnd_0001`
-set FILE = $FILE:t
set FILE = $FILE:r
set LND_DATE_EXT = `echo $FILE:e`
set LND_DATE = `echo $FILE:e | sed -e "s#-# #g"`
@@ -159,6 +183,19 @@
echo "`date` -- END COPY BLOCK"
+# If possible, use the round-robin approach to deal out the tasks.
+# Since the ensemble manager is not used by clm_to_dart or dart_to_clm,
+# it is OK to set it here and have it used by all routines.
+
+if ($?TASKS_PER_NODE) then
+ if ($#TASKS_PER_NODE > 0) then
+ ${COPY} input.nml input.nml.$$
+ sed -e "s#layout.*#layout = 2#" \
+ -e "s#tasks_per_node.*#tasks_per_node = $TASKS_PER_NODE#" input.nml.$$ >! input.nml
+ ${REMOVE} input.nml.$$
+ endif
+endif
+
#=========================================================================
# Block 2: Stage the files needed for SAMPLING ERROR CORRECTION
#
@@ -207,15 +244,29 @@
# files to be as listed above. When being archived, the filenames get a
# unique extension (describing the assimilation time) appended to them.
#
-# The inflation file is essentially a duplicate of the model state ...
-# it is slaved to a specific geometry. The initial files are created
-# offline with values of unity. For the purpose of this script, they are
-# thought to be the output of a previous assimilation, so they should be
-# named something like prior_inflate_restart.YYYY-MM-DD-SSSSS
+# The inflation file is essentially a duplicate of the DART model state ...
+# For the purpose of this script, they are the output of a previous assimilation,
+# so they should be named something like prior_inflate_restart.YYYY-MM-DD-SSSSS
#
-# The first inflation file can be created with 'fill_inflation_restart'
-# which can be built in the usual DART manner.
+# NOTICE: inf_initial_from_restart and inf_sd_initial_from_restart are somewhat
+# problematic. During the bulk of an experiment, these should be TRUE, since
+# we want to read existing inflation files. However, the first assimilation
+# might need these to be FALSE and then subsequently be set to TRUE.
+# There are two ways to handle this:
#
+# 1) Create the initial files offline (perhaps with 'fill_inflation_restart')
+# and stage them with the appropriate names in the RUNDIR.
+# You must manually remove the clm_inflation_cookie file
+# from the RUNDIR in this case.
+# - OR -
+# 2) create a cookie file called RUNDIR/clm_inflation_cookie
+# The existence of this file will cause this script to set the
+# namelist appropriately. This script will 'eat' the cookie file
+# to prevent this from happening for subsequent executions. If the
+# inflation file does not exist for them, and it needs to, this script
+# should die. The CESM_DART_config script automatically creates a cookie
+# file to support this option.
+#
# The strategy is to use the LATEST inflation file from the CESM 'rundir'.
# After an assimilation, the new inflation values/files will be moved to
# the CESM rundir to be used for subsequent assimilations. If the short-term
@@ -258,7 +309,24 @@
if ( $PRIOR_INF > 0 ) then
if ($PRIOR_TF == false) then
+ # we are not using an existing inflation file.
echo "inf_flavor(1) = $PRIOR_INF, using namelist values."
+
+ else if ( -e ../clm_inflation_cookie ) then
+ # We want to use an existing inflation file, but this is
+ # the first assimilation so there is no existing inflation
+ # file. This is the signal we need to to coerce the namelist
+ # to have different values for this execution ONLY.
+ # Since the local namelist comes from CASEROOT each time, we're golden.
+
+ set PRIOR_TF = FALSE
+
+ex input.nml <<ex_end
+g;inf_initial_from_restart ;s;= .*;= .${PRIOR_TF}., .${POSTE_TF}.,;
+g;inf_sd_initial_from_restart ;s;= .*;= .${PRIOR_TF}., .${POSTE_TF}.,;
+wq
+ex_end
+
else
# Look for the output from the previous assimilation
(ls -rt1 ../clm_${PRIOR_INF_OFNAME}.* | tail -n 1 >! latestfile) > & /dev/null
@@ -284,9 +352,25 @@
if ( $POSTE_INF > 0 ) then
if ($POSTE_TF == false) then
+ # we are not using an existing inflation file.
echo "inf_flavor(2) = $POSTE_INF, using namelist values."
+
+ else if ( -e ../clm_inflation_cookie ) then
+ # We want to use an existing inflation file, but this is
+ # the first assimilation so there is no existing inflation
+ # file. This is the signal we need to to coerce the namelist
+ # to have different values for this execution ONLY.
+ # Since the local namelist comes from CASEROOT each time, we're golden.
+
+ set POSTE_TF = FALSE
+
+ex input.nml <<ex_end
+g;inf_initial_from_restart ;s;= .*;= .${PRIOR_TF}., .${POSTE_TF}.,;
+g;inf_sd_initial_from_restart ;s;= .*;= .${PRIOR_TF}., .${POSTE_TF}.,;
+wq
+ex_end
+
else
-
# Look for the output from the previous assimilation
(ls -rt1 ../clm_${POSTE_INF_OFNAME}.* | tail -n 1 >! latestfile) > & /dev/null
set nfiles = `cat latestfile | wc -l`
@@ -305,6 +389,9 @@
echo "Posterior Inflation not requested for this assimilation."
endif
+# Eat the cookie regardless
+${REMOVE} ../clm_inflation_cookie
+
#=========================================================================
# Block 4: Convert N CLM restart files to DART initial condition files.
# clm_to_dart is serial code, we can do all of these at the same time
@@ -348,19 +435,6 @@
${LINK} ../../$LND_RESTART_FILENAME clm_restart.nc
${LINK} ../../$LND_HISTORY_FILENAME clm_history.nc
- # patch the CLM restart files to ensure they have the proper
- # _FillValue and missing_value attributes.
-# ncatted -O -a _FillValue,frac_sno,o,d,1.0e+36 clm_restart.nc
-# ncatted -O -a missing_value,frac_sno,o,d,1.0e+36 clm_restart.nc
-# ncatted -O -a _FillValue,DZSNO,o,d,1.0e+36 clm_restart.nc
-# ncatted -O -a missing_value,DZSNO,o,d,1.0e+36 clm_restart.nc
-# ncatted -O -a _FillValue,H2OSOI_LIQ,o,d,1.0e+36 clm_restart.nc
-# ncatted -O -a missing_value,H2OSOI_LIQ,o,d,1.0e+36 clm_restart.nc
-# ncatted -O -a _FillValue,H2OSOI_ICE,o,d,1.0e+36 clm_restart.nc
-# ncatted -O -a missing_value,H2OSOI_ICE,o,d,1.0e+36 clm_restart.nc
-# ncatted -O -a _FillValue,T_SOISNO,o,d,1.0e+36 clm_restart.nc
-# ncatted -O -a missing_value,T_SOISNO,o,d,1.0e+36 clm_restart.nc
-
echo "starting clm_to_dart for member ${member} at "`date`
${EXEROOT}/clm_to_dart >! output.${member}.clm_to_dart &
@@ -385,19 +459,20 @@
# Will result in a set of files : 'filter_restart.xxxx'
#
# DART namelist settings required:
-# &filter_nml: async = 0,
-# &filter_nml: adv_ens_command = "./no_model_advance.csh",
-# &filter_nml: restart_in_file_name = 'filter_ics'
-# &filter_nml: restart_out_file_name = 'filter_restart'
-# &filter_nml: obs_sequence_in_name = 'obs_seq.out'
-# &filter_nml: obs_sequence_out_name = 'obs_seq.final'
-# &filter_nml: init_time_days = -1,
-# &filter_nml: init_time_seconds = -1,
-# &filter_nml: first_obs_days = -1,
-# &filter_nml: first_obs_seconds = -1,
-# &filter_nml: last_obs_days = -1,
-# &filter_nml: last_obs_seconds = -1,
-# &ensemble_manager_nml: single_restart_file_in = .false.
+# &filter_nml: async = 0,
+# &filter_nml: adv_ens_command = "no_advance_script",
+# &filter_nml: restart_in_file_name = 'filter_ics'
+# &filter_nml: restart_out_file_name = 'filter_restart'
+# &filter_nml: obs_sequence_in_name = 'obs_seq.out'
+# &filter_nml: obs_sequence_out_name = 'obs_seq.final'
+# &filter_nml: init_time_days = -1,
+# &filter_nml: init_time_seconds = -1,
+# &filter_nml: first_obs_days = -1,
+# &filter_nml: first_obs_seconds = -1,
+# &filter_nml: last_obs_days = -1,
+# &filter_nml: last_obs_seconds = -1,
+# &ensemble_manager_nml: single_restart_file_in = .false.
+# &ensemble_manager_nml: single_restart_file_out = .false.
#
#=========================================================================
@@ -433,7 +508,7 @@
# Accomodate any possible inflation files
# 1) rename file to reflect current date
-# 2) move to CENTRALDIR so the DART INFLATION BLOCK works next time and
+# 2) move to RUNDIR so the DART INFLATION BLOCK works next time and
# that they can get archived.
foreach FILE ( ${PRIOR_INF_OFNAME} ${POSTE_INF_OFNAME} ${PRIOR_INF_DIAG} ${POSTE_INF_DIAG} )
@@ -445,7 +520,7 @@
end
#=========================================================================
-# Block 6: Update the clm restart files.
+# Block 6: Update the clm restart files ... simultaneously ...
#
# Each member will do its job in its own directory, which already exists
# and has the required input files remaining from 'Block 4'
Modified: DART/trunk/models/CESM/shell_scripts/pop_assimilate.csh
===================================================================
--- DART/trunk/models/CESM/shell_scripts/pop_assimilate.csh 2014-04-04 18:23:35 UTC (rev 6871)
+++ DART/trunk/models/CESM/shell_scripts/pop_assimilate.csh 2014-04-04 20:11:25 UTC (rev 6872)
@@ -37,6 +37,8 @@
set COPY = 'cp -fv --preserve=timestamps'
set LINK = 'ln -fvs'
set REMOVE = 'rm -fr'
+ set TASKS_PER_NODE = `echo $LSB_SUB_RES_REQ | sed -ne '/ptile/s#.*\[ptile=\([0-9][0-9]*\)]#\1#p'`
+ setenv MP_DEBUG_NOTIMEOUT yes
set BASEOBSDIR = /glade/p/image/Observations/WOD09
set LAUNCHCMD = mpirun.lsf
@@ -64,7 +66,6 @@
#-------------------------------------------------------------------------
set FILE = `head -n 1 rpointer.ocn_0001.restart`
-set FILE = $FILE:t
set FILE = $FILE:r
set OCN_DATE_EXT = `echo $FILE:e`
set OCN_DATE = `echo $FILE:e | sed -e "s#-# #g"`
@@ -74,8 +75,8 @@
set OCN_SECONDS = `echo $OCN_DATE[4] | bc`
set OCN_HOUR = `echo $OCN_DATE[4] / 3600 | bc`
-echo "valid time of model is $OCN_YEAR $OCN_MONTH $OCN_DAY $OCN_SECONDS (seconds)"
-echo "valid time of model is $OCN_YEAR $OCN_MONTH $OCN_DAY $OCN_HOUR (hours)"
+echo "valid time of ocean is $OCN_YEAR $OCN_MONTH $OCN_DAY $OCN_SECONDS (seconds)"
+echo "valid time of ocean is $OCN_YEAR $OCN_MONTH $OCN_DAY $OCN_HOUR (hours)"
#-------------------------------------------------------------------------
# Determine if current time is 0Z - if so, assimilate.
@@ -150,6 +151,19 @@
echo "`date` -- END COPY BLOCK"
+# If possible, use the round-robin approach to deal out the tasks.
+# Since the ensemble manager is not used by pop_to_dart or dart_to_pop,
+# it is OK to set it here and have it used by all routines.
+
+if ($?TASKS_PER_NODE) then
+ if ($#TASKS_PER_NODE > 0) then
+ ${COPY} input.nml input.nml.$$
+ sed -e "s#layout.*#layout = 2#" \
+ -e "s#tasks_per_node.*#tasks_per_node = $TASKS_PER_NODE#" input.nml.$$ >! input.nml
+ ${REMOVE} input.nml.$$
+ endif
+endif
+
#=========================================================================
# Block 2: Stage the files needed for SAMPLING ERROR CORRECTION
#
@@ -203,19 +217,23 @@
# so they should be named something like prior_inflate_restart.YYYY-MM-DD-SSSSS
#
# NOTICE: inf_initial_from_restart and inf_sd_initial_from_restart are somewhat
-# problematic. During the bulk of an experiment, these should be FALSE, since
+# problematic. During the bulk of an experiment, these should be TRUE, since
# we want to read existing inflation files. However, the first assimilation
-# might need these to be TRUE and then subsequently be set to FALSE.
-# There are two ways to handle this.
-# 1) Create the initial files offline with values of unity by using
-# 'fill_inflation_restart' and stage them with the appropriate names
-# in the RUNDIR.
+# might need these to be FALSE and then subsequently be set to TRUE.
+# There are two ways to handle this:
+#
+# 1) Create the initial files offline (perhaps with 'fill_inflation_restart')
+# and stage them with the appropriate names in the RUNDIR.
+# You must manually remove the pop_inflation_cookie file
+# from the RUNDIR in this case.
+# - OR -
# 2) create a cookie file called RUNDIR/pop_inflation_cookie
# The existence of this file will cause this script to set the
# namelist appropriately. This script will 'eat' the cookie file
# to prevent this from happening for subsequent executions. If the
# inflation file does not exist for them, and it needs to, this script
-# should die.
+# should die. The CESM_DART_config script automatically creates a cookie
+# file to support this option.
#
# The strategy is to use the LATEST inflation file from the CESM 'rundir'.
# After an assimilation, the new inflation values/files will be moved to
@@ -409,19 +427,20 @@
# Will result in a set of files : 'filter_restart.xxxx'
#
# DART namelist settings required:
-# &filter_nml: async = 0,
-# &filter_nml: adv_ens_command = "./no_model_advance.csh",
-# &filter_nml: restart_in_file_name = 'filter_ics'
-# &filter_nml: restart_out_file_name = 'filter_restart'
-# &filter_nml: obs_sequence_in_name = 'obs_seq.out'
-# &filter_nml: obs_sequence_out_name = 'obs_seq.final'
-# &filter_nml: init_time_days = -1,
-# &filter_nml: init_time_seconds = -1,
-# &filter_nml: first_obs_days = -1,
-# &filter_nml: first_obs_seconds = -1,
-# &filter_nml: last_obs_days = -1,
-# &filter_nml: last_obs_seconds = -1,
-# &ensemble_manager_nml: single_restart_file_in = .false.
+# &filter_nml: async = 0,
+# &filter_nml: adv_ens_command = "no_CESM_advance_script",
+# &filter_nml: restart_in_file_name = 'filter_ics'
+# &filter_nml: restart_out_file_name = 'filter_restart'
+# &filter_nml: obs_sequence_in_name = 'obs_seq.out'
+# &filter_nml: obs_sequence_out_name = 'obs_seq.final'
+# &filter_nml: init_time_days = -1,
+# &filter_nml: init_time_seconds = -1,
+# &filter_nml: first_obs_days = -1,
+# &filter_nml: first_obs_seconds = -1,
+# &filter_nml: last_obs_days = -1,
+# &filter_nml: last_obs_seconds = -1,
+# &ensemble_manager_nml: single_restart_file_in = .false.
+# &ensemble_manager_nml: single_restart_file_out = .false.
#
#=========================================================================
@@ -457,7 +476,7 @@
# Accomodate any possible inflation files
# 1) rename file to reflect current date
-# 2) move to CENTRALDIR so the DART INFLATION BLOCK works next time and
+# 2) move to RUNDIR so the DART INFLATION BLOCK works next time and
# that they can get archived.
foreach FILE ( ${PRIOR_INF_OFNAME} ${POSTE_INF_OFNAME} ${PRIOR_INF_DIAG} ${POSTE_INF_DIAG} )
Modified: DART/trunk/models/POP/shell_scripts/assimilate.csh
===================================================================
--- DART/trunk/models/POP/shell_scripts/assimilate.csh 2014-04-04 18:23:35 UTC (rev 6871)
+++ DART/trunk/models/POP/shell_scripts/assimilate.csh 2014-04-04 20:11:25 UTC (rev 6872)
@@ -35,6 +35,8 @@
set COPY = 'cp -fv --preserve=timestamps'
set LINK = 'ln -fvs'
set REMOVE = 'rm -fr'
+ set TASKS_PER_NODE = `echo $LSB_SUB_RES_REQ | sed -ne '/ptile/s#.*\[ptile=\([0-9][0-9]*\)]#\1#p'`
+ setenv MP_DEBUG_NOTIMEOUT yes
set BASEOBSDIR = /glade/p/image/Observations/WOD09
set LAUNCHCMD = mpirun.lsf
@@ -62,9 +64,7 @@
#-------------------------------------------------------------------------
set FILE = `head -n 1 rpointer.ocn_0001.restart`
-set FILE = $FILE:t
set FILE = $FILE:r
-set MYCASE = `echo $FILE | sed -e "s#\..*##"`
set OCN_DATE_EXT = `echo $FILE:e`
set OCN_DATE = `echo $FILE:e | sed -e "s#-# #g"`
set OCN_YEAR = `echo $OCN_DATE[1] | bc`
@@ -124,6 +124,19 @@
echo "`date` -- END COPY BLOCK"
+# If possible, use the round-robin approach to deal out the tasks.
+# Since the ensemble manager is not used by pop_to_dart or dart_to_pop,
+# it is OK to set it here and have it used by all routines.
+
+if ($?TASKS_PER_NODE) then
+ if ($#TASKS_PER_NODE > 0) then
+ ${COPY} input.nml input.nml.$$
+ sed -e "s#layout.*#layout = 2#" \
+ -e "s#tasks_per_node.*#tasks_per_node = $TASKS_PER_NODE#" input.nml.$$ >! input.nml
+ ${REMOVE} input.nml.$$
+ endif
+endif
+
#=========================================================================
# Block 2: Stage the files needed for SAMPLING ERROR CORRECTION
#
@@ -177,19 +190,23 @@
# so they should be named something like prior_inflate_restart.YYYY-MM-DD-SSSSS
#
# NOTICE: inf_initial_from_restart and inf_sd_initial_from_restart are somewhat
-# problematic. During the bulk of an experiment, these should be FALSE, since
+# problematic. During the bulk of an experiment, these should be TRUE, since
# we want to read existing inflation files. However, the first assimilation
-# might need these to be TRUE and then subsequently be set to FALSE.
-# There are two ways to handle this.
-# 1) Create the initial files offline with values of unity by using
-# 'fill_inflation_restart' and stage them with the appropriate names
-# in the RUNDIR.
+# might need these to be FALSE and then subsequently be set to TRUE.
+# There are two ways to handle this:
+#
+# 1) Create the initial files offline (perhaps with 'fill_inflation_restart')
+# and stage them with the appropriate names in the RUNDIR.
+# You must manually remove the pop_inflation_cookie file
+# from the RUNDIR in this case.
+# - OR -
# 2) create a cookie file called RUNDIR/pop_inflation_cookie
# The existence of this file will cause this script to set the
# namelist appropriately. This script will 'eat' the cookie file
# to prevent this from happening for subsequent executions. If the
# inflation file does not exist for them, and it needs to, this script
-# should die.
+# should die. The CESM_DART_config script automatically creates a cookie
+# file to support this option.
#
# The strategy is to use the LATEST inflation file from the CESM 'rundir'.
# After an assimilation, the new inflation values/files will be moved to
@@ -348,7 +365,7 @@
# make sure there are no old output logs hanging around
$REMOVE output.${member}.pop_to_dart
- set OCN_RESTART_FILENAME = `printf ${MYCASE}.pop_%04d.r.${OCN_DATE_EXT}.nc ${member}`
+ set OCN_RESTART_FILENAME = `printf ${CASE}.pop_%04d.r.${OCN_DATE_EXT}.nc ${member}`
set OCN_NML_FILENAME = `printf pop2_in_%04d ${member}`
set DART_IC_FILENAME = `printf filter_ics.%04d ${member}`
set DART_RESTART_FILE = `printf filter_restart.%04d ${member}`
@@ -383,19 +400,20 @@
# Will result in a set of files : 'filter_restart.xxxx'
#
# DART namelist settings required:
-# &filter_nml: async = 0,
-# &filter_nml: adv_ens_command = "./no_model_advance.csh",
-# &filter_nml: restart_in_file_name = 'filter_ics'
-# &filter_nml: restart_out_file_name = 'filter_restart'
-# &filter_nml: obs_sequence_in_name = 'obs_seq.out'
-# &filter_nml: obs_sequence_out_name = 'obs_seq.final'
-# &filter_nml: init_time_days = -1,
-# &filter_nml: init_time_seconds = -1,
-# &filter_nml: first_obs_days = -1,
-# &filter_nml: first_obs_seconds = -1,
-# &filter_nml: last_obs_days = -1,
-# &filter_nml: last_obs_seconds = -1,
-# &ensemble_manager_nml: single_restart_file_in = .false.
+# &filter_nml: async = 0,
+# &filter_nml: adv_ens_command = "no_CESM_advance_script",
+# &filter_nml: restart_in_file_name = 'filter_ics'
+# &filter_nml: restart_out_file_name = 'filter_restart'
+# &filter_nml: obs_sequence_in_name = 'obs_seq.out'
+# &filter_nml: obs_sequence_out_name = 'obs_seq.final'
+# &filter_nml: init_time_days = -1,
+# &filter_nml: init_time_seconds = -1,
+# &filter_nml: first_obs_days = -1,
+# &filter_nml: first_obs_seconds = -1,
+# &filter_nml: last_obs_days = -1,
+# &filter_nml: last_obs_seconds = -1,
+# &ensemble_manager_nml: single_restart_file_in = .false.
+# &ensemble_manager_nml: single_restart_file_out = .false.
#
#=========================================================================
@@ -431,7 +449,7 @@
# Accomodate any possible inflation files
# 1) rename file to reflect current date
-# 2) move to CENTRALDIR so the DART INFLATION BLOCK works next time and
+# 2) move to RUNDIR so the DART INFLATION BLOCK works next time and
# that they can get archived.
foreach FILE ( ${PRIOR_INF_OFNAME} ${POSTE_INF_OFNAME} ${PRIOR_INF_DIAG} ${POSTE_INF_DIAG} )
Modified: DART/trunk/models/cam/shell_scripts/assimilate.csh
===================================================================
--- DART/trunk/models/cam/shell_scripts/assimilate.csh 2014-04-04 18:23:35 UTC (rev 6871)
+++ DART/trunk/models/cam/shell_scripts/assimilate.csh 2014-04-04 20:11:25 UTC (rev 6872)
@@ -35,10 +35,10 @@
set COPY = 'cp -fv --preserve=timestamps'
set LINK = 'ln -fvs'
set REMOVE = 'rm -fr'
+ set TASKS_PER_NODE = `echo $LSB_SUB_RES_REQ | sed -ne '/ptile/s#.*\[ptile=\([0-9][0-9]*\)]#\1#p'`
+ setenv MP_DEBUG_NOTIMEOUT yes
# BASEOBSDIR needs the whole path name except for the date directory and file name.
- # You can set this to point to directories of synthetic observations, too.
- # set BASEOBSDIR = /glade/p/image/Observations/Synthetic/UVT_ne30_12H
set BASEOBSDIR = /glade/p/image/Observations/ACARS
set LAUNCHCMD = mpirun.lsf
breaksw
@@ -65,9 +65,7 @@
#-------------------------------------------------------------------------
set FILE = `head -n 1 rpointer.atm_0001`
-set FILE = $FILE:t
set FILE = $FILE:r
-set MYCASE = `echo $FILE | sed -e "s#\..*##"`
set ATM_DATE_EXT = `echo $FILE:e`
set ATM_DATE = `echo $FILE:e | sed -e "s#-# #g"`
set ATM_YEAR = `echo $ATM_DATE[1] | bc`
@@ -127,6 +125,19 @@
echo "`date` -- END COPY BLOCK"
+# If possible, use the round-robin approach to deal out the tasks.
+# Since the ensemble manager is not used by cam_to_dart or dart_to_cam,
+# it is OK to set it here and have it used by all routines.
+
+if ($?TASKS_PER_NODE) then
+ if ($#TASKS_PER_NODE > 0) then
+ ${COPY} input.nml input.nml.$$
+ sed -e "s#layout.*#layout = 2#" \
+ -e "s#tasks_per_node.*#tasks_per_node = $TASKS_PER_NODE#" input.nml.$$ >! input.nml
+ ${REMOVE} input.nml.$$
+ endif
+endif
+
#=========================================================================
# Block 2: Stage the files needed for SAMPLING ERROR CORRECTION
#
@@ -180,9 +191,9 @@
# so they should be named something like prior_inflate_restart.YYYY-MM-DD-SSSSS
#
# NOTICE: inf_initial_from_restart and inf_sd_initial_from_restart are somewhat
-# problematic. During the bulk of an experiment, these should be FALSE, since
+# problematic. During the bulk of an experiment, these should be TRUE, since
# we want to read existing inflation files. However, the first assimilation
-# might need these to be TRUE and then subsequently be set to FALSE.
+# might need these to be FALSE and then subsequently be set to TRUE.
# There are two ways to handle this:
#
# 1) Create the initial files offline (perhaps with 'fill_inflation_restart')
@@ -366,23 +377,19 @@
mkdir -p $MYTEMPDIR
cd $MYTEMPDIR
- # Turns out the .h0. files are timestamped with the START of the
- # run, which is *not* ATM_DATE_EXT ... I just link to a whatever
- # is convenient (since the info is static).
# make sure there are no old output logs hanging around
-
$REMOVE output.${member}.cam_to_dart
- set ATM_INITIAL_FILENAME = `printf ../../${MYCASE}.cam_%04d.i.${ATM_DATE_EXT}.nc ${member}`
- set ATM_HISTORY_FILENAME = `ls -1t ../../${MYCASE}.cam*.h0.* | head -n 1`
+ set ATM_INITIAL_FILENAME = `printf ${CASE}.cam_%04d.i.${ATM_DATE_EXT}.nc ${member}`
+ set ATM_HISTORY_FILENAME = `printf ${CASE}.cam_%04d.h0.${ATM_DATE_EXT}.nc ${member}`
set DART_IC_FILENAME = `printf filter_ics.%04d ${member}`
set DART_RESTART_FILE = `printf filter_restart.%04d ${member}`
sed -e "s#dart_ics#../${DART_IC_FILENAME}#" \
-e "s#dart_restart#../${DART_RESTART_FILE}#" < ../input.nml >! input.nml
- ${LINK} $ATM_INITIAL_FILENAME caminput.nc
- ${LINK} $ATM_HISTORY_FILENAME cam_phis.nc
+ ${LINK} ../../$ATM_INITIAL_FILENAME caminput.nc
+ ${LINK} ../../$ATM_HISTORY_FILENAME cam_phis.nc
echo "starting cam_to_dart for member ${member} at "`date`
${EXEROOT}/cam_to_dart >! output.${member}.cam_to_dart &
@@ -408,6 +415,8 @@
# Will result in a set of files : 'filter_restart.xxxx'
#
# DART namelist settings required:
+# &filter_nml: async = 0,
+# &filter_nml: adv_ens_command = "no_advance_script",
# &filter_nml: restart_in_file_name = 'filter_ics'
# &filter_nml: restart_out_file_name = 'filter_restart'
# &filter_nml: obs_sequence_in_name = 'obs_seq.out'
@@ -426,11 +435,11 @@
# CAM:static_init_model() always needs a caminput.nc and a cam_phis.nc
# for geometry information, etc.
-set ATM_INITIAL_FILENAME = ../${MYCASE}.cam_0001.i.${ATM_DATE_EXT}.nc
-set ATM_HISTORY_FILENAME = `ls -1t ../${MYCASE}.cam*.h0.* | head -n 1`
+set ATM_INITIAL_FILENAME = ${CASE}.cam_0001.i.${ATM_DATE_EXT}.nc
+set ATM_HISTORY_FILENAME = ${CASE}.cam_0001.h0.${ATM_DATE_EXT}.nc
-${LINK} $ATM_INITIAL_FILENAME caminput.nc
-${LINK} $ATM_HISTORY_FILENAME cam_phis.nc
+${LINK} ../$ATM_INITIAL_FILENAME caminput.nc
+${LINK} ../$ATM_HISTORY_FILENAME cam_phis.nc
echo "`date` -- BEGIN FILTER"
${LAUNCHCMD} ${EXEROOT}/filter || exit -7
@@ -507,7 +516,7 @@
set inst_string = `printf _%04d $member`
- set ATM_INITIAL_FILENAME = ${MYCASE}.cam${inst_string}.i.${ATM_DATE_EXT}.nc
+ set ATM_INITIAL_FILENAME = ${CASE}.cam${inst_string}.i.${ATM_DATE_EXT}.nc
${LINK} ${ATM_INITIAL_FILENAME} cam_initial${inst_string}.nc || exit -9
Modified: DART/trunk/models/clm/shell_scripts/assimilate.csh
===================================================================
--- DART/trunk/models/clm/shell_scripts/assimilate.csh 2014-04-04 18:23:35 UTC (rev 6871)
+++ DART/trunk/models/clm/shell_scripts/assimilate.csh 2014-04-04 20:11:25 UTC (rev 6872)
@@ -35,6 +35,8 @@
set COPY = 'cp -fv --preserve=timestamps'
set LINK = 'ln -fvs'
set REMOVE = 'rm -fr'
+ set TASKS_PER_NODE = `echo $LSB_SUB_RES_REQ | sed -ne '/ptile/s#.*\[ptile=\([0-9][0-9]*\)]#\1#p'`
+ setenv MP_DEBUG_NOTIMEOUT yes
set BASEOBSDIR = /glade/p/image/Observations/land
set LAUNCHCMD = mpirun.lsf
@@ -57,6 +59,7 @@
set COPY = 'cp -fv --preserve=timestamps'
set LINK = 'ln -fvs'
set REMOVE = 'rm -fr'
+ set TASKS_PER_NODE = $MAX_TASKS_PER_NODE
set BASEOBSDIR = /your/observation/directory/here
set LAUNCHCMD = "mpiexec -n $NTASKS"
@@ -157,6 +160,19 @@
echo "`date` -- END COPY BLOCK"
+# If possible, use the round-robin approach to deal out the tasks.
+# Since the ensemble manager is not used by clm_to_dart or dart_to_clm,
+# it is OK to set it here and have it used by all routines.
+
+if ($?TASKS_PER_NODE) then
+ if ($#TASKS_PER_NODE > 0) then
+ ${COPY} input.nml input.nml.$$
+ sed -e "s#layout.*#layout = 2#" \
+ -e "s#tasks_per_node.*#tasks_per_node = $TASKS_PER_NODE#" input.nml.$$ >! input.nml
+ ${REMOVE} input.nml.$$
+ endif
+endif
+
#=========================================================================
# Block 2: Stage the files needed for SAMPLING ERROR CORRECTION
#
@@ -205,15 +221,29 @@
# files to be as listed above. When being archived, the filenames get a
# unique extension (describing the assimilation time) appended to them.
#
-# The inflation file is essentially a duplicate of the model state ...
-# it is slaved to a specific geometry. The initial files are created
-# offline with values of unity. For the purpose of this script, they are
-# thought to be the output of a previous assimilation, so they should be
-# named something like prior_inflate_restart.YYYY-MM-DD-SSSSS
+# The inflation file is essentially a duplicate of the DART model state ...
+# For the purpose of this script, they are the output of a previous assimilation,
+# so they should be named something like prior_inflate_restart.YYYY-MM-DD-SSSSS
#
-# The first inflation file can be created with 'fill_inflation_restart'
-# which can be built in the usual DART manner.
+# NOTICE: inf_initial_from_restart and inf_sd_initial_from_restart are somewhat
+# problematic. During the bulk of an experiment, these should be TRUE, since
+# we want to read existing inflation files. However, the first assimilation
+# might need these to be FALSE and then subsequently be set to TRUE.
+# There are two ways to handle this:
#
+# 1) Create the initial files offline (perhaps with 'fill_inflation_restart')
+# and stage them with the appropriate names in the RUNDIR.
+# You must manually remove the clm_inflation_cookie file
+# from the RUNDIR in this case.
+# - OR -
+# 2) create a cookie file called RUNDIR/clm_inflation_cookie
+# The existence of this file will cause this script to set the
+# namelist appropriately. This script will 'eat' the cookie file
+# to prevent this from happening for subsequent executions. If the
+# inflation file does not exist for them, and it needs to, this script
+# should die. The CESM_DART_config script automatically creates a cookie
+# file to support this option.
+#
@@ Diff output truncated at 40000 characters. @@
More information about the Dart-dev
mailing list