###############################################################################
# cccjob_subs contains subroutines used by cccjob
#
# $Id: cccjob_subs.pm 671 2012-05-04 23:35:22Z acrnrls $
###############################################################################
package cccjob_subs;
###############################################################################
# variables used in these functions that may be set or used externally are:
# %JOBDEF
# %VARS
# @vlist
# @Unused_clargs
# $Invoked_name
# $Extended_usage
# $Verbose
# $start_time
# $stop_time
# $Template_Expand_Global
# $Template_Insert_Global
# $Sub_in_update_section
# $Rep_in_update_section
# $Always_add_updates
###############################################################################
require Exporter;
@ISA = qw(Exporter);
@EXPORT    = qw(cmd_line show_usage show_JOBDEF format_vlist parse_job parse_job_template
  expand_template parse_job_here_docs parse_job_sections substitute_job
  change_var_vals insert_updates define_job on_cccjob_path add_jobtag read_jobtag
  split_job_string new_mon_year diff_in_months filter_updates file_body update_cpp_i
  ref2str_iopts str2ref_iopts set_iopts );
@EXPORT_OK = qw(cmd_line show_usage show_JOBDEF format_vlist parse_job parse_job_template
   expand_template parse_job_here_docs parse_job_sections substitute_job
   change_var_vals insert_updates define_job on_cccjob_path add_jobtag read_jobtag
   split_job_string new_mon_year diff_in_months filter_updates file_body update_cpp_i
   ref2str_iopts str2ref_iopts set_iopts );

use shell_var_subs;
use vars qw( %JOB_COUNT );

sub cmd_line { my ($clarg, $OPTS) = @_;
  use strict;
  # Process non-option command line arguments
  # Any invalid args encountered will be pushed onto Unused_clargs
  my $vname;
  my $vval;
  my $vopt;
  my $o;
  my $vn;

  # check for per variable options
  # per variable options are any of the letters [aArR] following
  # the final : found in the command line argument
  ($vopt) = $clarg =~ /:([aArR]+)$/;
  $vopt = '' unless defined $vopt;
  # strip options from the end of clarg, if any
  if ($vopt) {$clarg =~ s/:[aArR]+$//};
  # append any options passed in via OPTS
  if (defined $OPTS->{VOPT}) {$vopt .= $OPTS->{VOPT}};

  # If the command line arg is a valid file name then read it
  if (-r $clarg) {
    # Read file contents into the string $script
    # This file must contain syntactically correct bourne shell script
    # Note that only static variable definitions are extracted (ie the script
    # is not executed) so constructs like a=2; b=$a will return the
    # string "$a", not 2, as the value of the variable b.
    my $script = '';
    open(SCRIPT, "<$clarg") ||
      die "cmd_line: cannot open file $clarg\n  Stopped";
    # Remove full line comments
    while (<SCRIPT>) {next if /^\s*#/; $script .= $_};
    close(SCRIPT);

    # Extract all shell variables and their definitions from $script
    my %vars = shell_var_subs::find_all_vars($script, {VERBOSE=>0});

    # recursively call cmd_line with each variable def found in the file
    for $vname (sort keys %vars) {
      # Only the first value for vname found in the file will be used
      ($vval) = @{$vars{$vname}};
      # If the value contains "#" then quote it
      if ($vval =~ /#/) {
        if   ($vval !~ /"/) {$vval = '"' . "$vval" . '"'}
        elsif ($vval !~ /'/) {$vval = "'" . "$vval" . "'"};
      };
      # quote blanks in the value
      if ($vval =~ / /) {
        if   ($vval !~ /"/) {$vval = '"' . "$vval" . '"'}
        elsif ($vval !~ /'/) {$vval = "'" . "$vval" . "'"};
      };
      my $pair = "$vname=$vval";
      cmd_line($pair, {VOPT => $vopt});
    }
    return 1;

  }

  # Otherwise look for vname=value pairs
  if ($clarg =~ /=/) {
    # args of the form vname=value
    ($vname,$vval) = split '=', $clarg, 2;

    # strip all white space from vname
    $vname =~ s/\s+//g;

    # strip single or double quotes from vval
    $vval =~ s/^\s*'(.*)'$/$1/;
    $vval =~ s/^\s*"(.*)"$/$1/;

    # remove any existing vname from vlist
    @main::vlist = grep {($vn) = split '=', $_; $vn ne $vname} @main::vlist;

    # Identify this variable as set by the user
    $main::usrset{$vname} = 1;

    # process per variable options, if any
    if ($vopt) {
      foreach (split '',$vopt) {
        /:/ and do {                           next};  # throw away the :
        /a/ and do {$main::VARS{$vname}{add} = 0;    next};  # unset addvar
        /A/ and do {$main::VARS{$vname}{add} = 1;    next};  # set addvar
        /r/ and do {$main::VARS{$vname}{remove} = 0; next};  # unset remvar
        /R/ and do {$main::VARS{$vname}{remove} = 1; next};  # set remvar
        die "Unknown option $_ for variable ${clarg}:$_\n  Stopped";
      }
    }

    # add to vlist
    push @main::vlist, "$vname=$vval";
  } else {
    # add this arg to the unused list
    push @main::unused_clargs, $clarg
  }

  # always return true. Invalid args are dealt with in main
  return 1;
}

sub show_usage { my ($top_msg, $OPTS) = @_;
  use strict;
  sub show_JOBDEF;
  my $prev_output_fh;

  my $full_help = 0;
  if (defined $OPTS->{FULL_HELP}) {$full_help = 1};

  # Pipe output through a pager if one is available
  my $pager = "less";
  chomp(my $fullpath = `which $pager 2>/dev/null`);
  unless ( -x $fullpath) {
    unless ($pager = $ENV{PAGER}) {$pager = "more"};
    chomp($fullpath = `which $pager 2>/dev/null`);
    unless ( -x $fullpath) {$pager = "more"};
    chomp($fullpath = `which $pager 2>/dev/null`);
    unless ( -x $fullpath) {$pager = "pg"};
    chomp($fullpath = `which $pager 2>/dev/null`);
    unless ( -x $fullpath) {$pager = ""};
  };
  if ($pager) {
    # only pipe to a pager if one is available
    open(PG_PIPE,"|$pager");
    $prev_output_fh = select;
    select PG_PIPE;
    $| = 1; # set autoflush for PG_PIPE
  };

  if (defined $OPTS->{SHOW_JOBDEF_TOP}) {
    show_JOBDEF($top_msg, {FULL_HELP=>$full_help});
    $top_msg = '';
  };

  print "="x80,"\n";
  if ($top_msg) {print "$top_msg\n\n"};
  print <<EOR;
  Usage: $main::Invoked_name --joblist=jlist [options] [vname=value] [var_def_file]
Purpose: Create a job string and save it in a local file

  The joblist is the only mandatory command line argument.
  --joblist=jlist[:myaqsDMJSn[:N|Nm|Ny]]
      ...create a job string internally using info from jlist.
         jlist is a white space separated list of words which
         may be file names or keywords that will be recognized internally.
         If the list contains white space then it must be quoted.
         Each word may optionally be followed by an "=key_modifier" string
         and/or a colon (:) plus one or more single character positioning options.
         A second colon (:) may be appended to the keyword followed by an interval
         of the form N, Nm or Ny where N is a positive integer. This indicates that
         the current job should be split into multiple jobs that differ only in the
         first and last year/month parameters, then these jobs will be inserted at
         the point determined by the positioning option.

         The format for any job definition in jlist is

         keyword[=key_modifier][:options[:interval]]

         where the quantities in square brackets are optional. If a key_modifier is
         supplied for a keyword that does not accept modifiers it will be silently ignored.
EOR
  unless ($full_help) {
    print <<EOR;
         Use --help for more info.
EOR
  };
  if ($full_help) {
    print <<EOR;

         Any jobs that are not modified by an option or are modified by the
         passthru (:n) option will be placed at the beginning of the job string
         in the order they appear on the command line. Note that the special (:s)
         option may be used to place jobs at the end of the job string.
         If the yearly (y) or monthly (m) option is preceded by an integer, N say,
         then that job will be inserted at the beginning or end of every N years or
         N months depending on the sign of N.

         Valid positioning options are:

           n = passthru  insert unmodified job at start of job string
      [int]m = monthly   insert at start of every month
               if preceeded by an positive integer, N, insert at end of every N months
               if preceeded by an negative integer, -N, insert at start of every N months
      [int]y = yearly    insert at end of year
               if preceeded by an positive integer, N, insert at end of every N years
               if preceeded by an negative integer, -N, insert at start of every N years
           a = annual    insert at start of year
           q = quarterly insert at start of each quarter
           s = special   insert at end of entire job string
           D = DJF       insert at end of DJF
           M = MAM       insert at end of MAM
           J = JJA       insert at end of JJA
           S = SON       insert at end of SON

         Valid sub interval options are (here N is a positive integer):

           N  ...divide the job into sub intervals of N months each
           Nm ...same as above
           Ny ...divide the job into sub intervals of N years each

           e.g. A keyword of the form "mdump:y:4m" would insert 3 mdump jobs into
           the job string at the end of every year. The first would dump months 1 to 4.
           The second would dump months 5 to 8. The third would dump months 9 to 12.

           If the specified sub interval does not divide evenly into the interval
           determined by the positioning option, then the last sub interval will
           be shortened to fit.
           e.g. A keyword of the form "mdump:y:5m" would put 3 mdump jobs into the
           job string at the end of every year. The first would dump months 1 to 5.
           The second would dump months 6 to 10 and the third would dump months 11 and 12.

           It is an error to specify a sub interval that is greater than the interval
           determined by the positioning option.
           e.g. A keyword of the form "mdump:y:14m" is invalid because the sub interval
           specified (14 months) is greater than 1 year.

         Valid keywords are:
EOR
    show_JOBDEF('', {FULL_HELP=>$full_help});
    print <<EOR;

         keywords that accept key modifiers:
           custom=file    ...insert the contents of file into the custom job
           cloop=file  or  cloop=file1\%file2\%file3
                          ...In the first form insert the contents of file inside
                             the predefined year/month loop in the cloop job string.
                             In the second form, insert the contents of file1 just
                             before the predefined year/month loop, the contents of
                             file2 inside the loop and the contents of file3 just
                             after the loop. Any of file1, file2 or file3 may be null,
                             in which case no substitution is done for that position.
           pool_mul=pool_mul_id ...replace the variable pool_mul_id in this pool_mul job
           load_list=file ...file is a disk file containing file names to be
                             loaded from cfs, one file name per line.
           dump_list=file ...file is a disk file containing file names to be
                             dumped to cfs, one file name per line.
           del_list=file  ...file is a disk file containing file names to be
                             deleted, one file name per line.

         Certain predefined modules may be "cloned" so that multiple versions, each
         with their own set of variable definitions, may be inserted into the same
         job string. The predefined modules which may be cloned are:

           cloop, custom, mload, mdump, mdelete, load_list, dump_list, del_list

         Cloning is accomplished by appending an arbitrary string of alphanumeric
         characters to the end of a predefined module name.
         e.g. a joblist specified as --joblist='mdump:m mdump_diag:y'
         will insert two sepearate copies of mdump into your job string. The first
         copy will be inserted every month and the second copy will be inserted at
         the end of every year. Any variable names used by the original module that
         are of the form mdump_* will be replaced in the cloned copy by variable
         names of the form mdump_diag_*. So, for example, setting mdump_prefix=xx
         and mdump_diag_prefix=yy on the command line will produce a job string that
         will dump files with the "xx" prefix monthly and files with the "yy" prefix
         yearly.

         The list of keywords may be expanded by the user. This is accomplished
         by creating a file containing the users job string and naming it xxx_jobdef
         where xxx will be the keyword associated with this job. The file xxx_jobdef
         should be placed in the user's home directory in a subdirectory named cccjobs.
         Alternatively the environment variable CCCJOB_PATH may be set to a colon (:)
         separated list of directories, each of which will be searched for files with
         names of the form *_jobdef. Any files that match will be read and added to the
         list of keywords. These keywords will then be recognized by the --joblist
         command line option.

         It is also possible to set the keyword and provide a brief description of
         the job within the xxx_jobdef file itself. This is not required but provides
         added flexibility for the user. To set the keyword add a comment line of
         the form
         # keyword :: mykey
         anywhere within the xxx_jobdef file. mykey will then be the keyword associated
         with this job and will override the file name derived keyword. To set a one
         line job description that will appear next to the users keyword whenever the
         job list is printed via the --help command line option, add a line of the form
         # description :: a short job description
         anywhere within the xxx_jobdef file. Everthing after the :: on that line will
         be used as the job description.

EOR
  };

  print <<EOR;

  Any vname=value pair found on the command line will be globally substituted in the
  output job string. Any file name found on the command line will be scanned for
  vname=value pairs and all vname=value pairs found there will be globally substituted.
EOR

  if ($full_help) {
    print <<EOR;
  This file must contain syntactically correct bourne shell script except for the
  possible addition of per variable modifiers as described below. Note that only
  static variable definitions are extracted (ie the script is not executed) so constructs
  such as a=2; b=\$a will return the string "\$a", not 2, as the value of the variable b.

  Each vname=value pair supplied on the command line may be optionally
  followed by a colon (:) and a list of single letter modifiers.
  Valid modifiers are:
     a ...unset the addvar flag for this variable (overrides --addvar option)
     A ...set the addvar flag for this variable   (overrides --addvar option)
     r ...unset the remvar flag for this variable (overrides --remvar option)
     R ...set the remvar flag for this variable   (overrides --remvar option)
  example: user=acrnxyz:A  means add user=acrnxyz to every job even if the
           job does not already contain a definition for user.

  Variables given null values will be removed from the job string.
  example: vname=  on the command line will remove all occurences of vname
           from the job string.
           This behavior may be overridden with the --noremvar option.

  Certain variables will always be set internally, they are:
    uxxx     ...a typical CCCma model file name is of the form
    runid       \${uxxx}_\${runid}_\${year}_m\${mon}_gs
    RUNID    ...upper case of runid
    crawork  ...filename of the job string that is placed in ~/.queue/.crawork
    username ...user's login name
    user     ...user's real name
    initsp   ...initialization switch (on/off)
    samerun  ...compile switch (on/off)
    ksteps   ...the maximum number of time steps a model job will run before resubmission
  These internal settings may be overridden by adding the appropriate vname=value
  pair to the command line or removed entirely by using the --emptyvlist command
  line option.

  The months variable may be used to control the number of months a model job runs
  per submission to the batch queue. The wall clock limit on the batch queue and the
  complexity of your job will determine what the value of months should be. The default
  value for months is 1. If months is set to any positive integer, N, greater than 1 then
  the model job will attempt to run for N months per submission. If months is set to
  a fraction, 1/N, then the model job will run for 1/N months per submission.

  Adding a variable definition for offset_kount on the command line will add an
  offset to the internally calculated kfinal value used in model job strings.
  Normally this is not desired but may be useful in certain situations.

EOR
    if (scalar(@main::vlist)) {
      print "  The current vname=value list contains:\n";
      foreach (@main::vlist) {printf "%20s=%-20s\n", split '=',$_,2}
    } else {print "  The current vname=value list is empty\n"};
  };
  print <<EOR;

options:
EOR
  if ($full_help) {
    print <<EOR;
  Unique abbreviations of option names will also work e.g. --job=fname
  All options are preceeded by either 1 or 2 dashes.
  The equals sign (if present) may be replaced by a space.
  Things enclosed in square brackets [...] indicate optional quantities.
  Be careful to quote things like whitespace or shell wildcard characters when
  used as part of a command line argument to avoid conflict with the shell.
EOR
  };
  print <<EOR;
    --output=fname
        ...Put the job string into a file named fname
    --updates=flist[:fi]
        ...Add updates to the job string from file fname.
EOR
  if ($full_help) {
    print <<EOR;
           flist is a comma or whitespace separated list of file names. Each file
           in this list must contain updates. Updates from each file will be
           merged into a single set of updates (beware of overlaps, no checking is
           done). Multiple --updates options are allowed and all files will be read
           in the order they appear on the command line. These file names should be
           full path names of valid files on the machine on which $main::Invoked_name
           is executed. If any of these files do not exist the job will abort.

           The updates file should contain "### update.*section" lines to indicate
           the position at which the updates are to be inserted into the job string.
           "section" may be one of "script", "model", "sub", "ocean" or "ocean sub".
           Any lines found outside of the update sections will be ignored.
           Existing submission jobs can be used to supply updates for the current
           job string but note that only the first job in the existing string will be
           read in that case. Model, sub and ocean updates will only be inserted
           into jobs for which samerun=off and an appropriate "### update.*section"
           line exists. Script updates will be inserted into every job containing
           an "### update.*script" line.

           The :f switch may be used to force updates to be added regardless of the
           value of samerun. However, updates will never be added unless the correct
           "### update.*section" line exists in the the job.

           The :i switch may be used to ignore errors that would otherwise cause the
           script to abort when updates are supplied but not used.
EOR
  };
  print <<EOR;
    --substitute='/pat/repl/[i]'
        ...Substitute repl for pat everywhere it is found in the job string.
EOR
  if ($full_help) {
    print <<EOR;
           The last delimiter may be followed by the letter i for a case
           insensitive substitution. The pattern is any valid perl regular
           expression and the delimiters can be any single character
           not found in either pat or repl. The entire expression should be
           quoted to avoid conflict with the shell.
           With multiple --sub options substitutions will occur in the order
           they appear on the command line.
EOR
  };
  print <<EOR;
    --start_time=year:month:day
        ...Specify the starting year, month and day.
EOR
  if ($full_help) {
    print <<EOR;
           The colon or space separated year month and day arguments should
           be integers in the ranges year>0, month=1-12 and day=1-31. If any
           of these values are missing (e.g. --start=2::4 has month missing)
           then defaults will be supplied. The defaults are --start=1:1:1.
           There must be at least one "monthly" section in the --joblist=...
           list for this option to have any meaning.
EOR
  };
  print <<EOR;
    --stop_time=year:month:day
        ...Specify the stopping year, month and day.
EOR
  if ($full_help) {
    print <<EOR;
           The colon or space separated year month and day arguments should
           be integers in the ranges year>0, month=1-12 and day=1-31. If any
           of these values are missing (e.g. --start=2::4 has month missing)
           then defaults will be supplied. The defaults are --stop=1:12:31.
           kfinal overrides this option if kfinal is set on the command line.
           There must be at least one "monthly" section in the --joblist=...
           list for this option to have any meaning.
EOR
  };
  if (1 == 2) { # --year_range and --month_range are depreciated
  print <<EOR;
    --year_range='n,m'
        ...Specify the starting and ending year of the run.
EOR
  if ($full_help) {
    print <<EOR;
           *** This option is overridden by --start_time and/or --stop_time.
           The argument is a comma or space separated pair of numbers.
           The first number of the pair is the first year of the run and the
           second number is the last. If the first number is missing it will
           be set to 1. If the second number is missing it will be set to the
           value of the first number. Years less than 0 are not allowed.
           Note that there must be at least 1 word in the --joblist=... arg
           with the "monthly" (:m) option appended for the --year_range option
           to have any meaning.
           If kfinal is set on the command line then the job will run until it
           reaches kfinal time steps and the end year set with --year_range
           is ignored, however, the start year is still valid.
           example: --year='1,5' implies 5 years from year 1 to year 5
                    --year='8' implies 1 year from year 8 to year 8
EOR
  };
  print <<EOR;
    --month_range='n,m'
        ...Specify the starting month in the first year and the ending month
           in the last year of the run.
EOR
  if ($full_help) {
    print <<EOR;
           *** This option is overridden by --start_time and/or --stop_time.
           The argument is a comma or space separated pair of numbers.
           The first number of the pair is the first month in the first year
           of the run and the second number is the last month in the last year.
           If the first number is missing it will be set to 1 and if the second
           number is missing it will be set to 12. The default range is 1,12.
           If kfinal is set on the command line then the job will run until it
           reaches kfinal time steps and the end month set with --month_range
           is ignored, however, the start month is still valid.
EOR
  };
  }; # end of --year_range and --month_range depreciated documentation
  print <<EOR;
    --[no]initial_conditions
        ...[do not]/do start the run from initial conditions.
    --restart=rsfile
        ...start the run from an existing restart file named rsfile.
EOR
  if ($full_help) {
    print <<EOR;
           If rsfile is missing or null then users must set the restart file name
           appropriately (ie define the variable start) in their model job string.
           Setting either initial_conditions or restart will negate the other.
           If multiple initial_conditions and restart options are specified only
           the last one on the command line will be recognized. Setting initsp=off
           or initsp=on on the command line will override either of these options.
           The default is to start from initial conditions.
    --emptyvlist
        ...Empty the internal list containing vname=value pairs.
           All variable assignments made prior to this option on the command
           line as well as any internally defined values will be deleted.
EOR
  };
  my $LOG_DIR = (getpwuid($<))[7]."/cccjobs/log";
  print <<EOR;
    --log
        ...Log sufficient info about the current invocation of $main::Invoked_name
           to be able to recreate the job string at a later date.
EOR
  if ($full_help) {
    print <<EOR;
           This log will appear in a subdirectory of $LOG_DIR.
           The name of the subdir will be formed from the current runid together
           with a date stamp and current pid. The dir $LOG_DIR
           will be created if it does not already exist.
EOR
  };
  print <<EOR;
    --display_hist
        ...Page through your $main::Invoked_name command line history.
    --edit_hist
        ...Edit then execute previous $main::Invoked_name command lines.
EOR
  print <<EOR;
    --show_jobdef
        ...Show a list of keywords with a short description for each predefined job
    --jdef-repo
        ...An absolute path name to a git repository containing a jobdefs subdir
           (default is system dependent)
    --jdef-path
        ...The path name of the jobdefs subdir relative to the root of this git repository
           (default lib/jobdefs)
    --jdef-rev
        ...A commit-ish string to identify which revision of the job definition files
           will be extracted from the git repository (default HEAD)
    --help
        ...Display expanded usage info. Invoke as "$main::Invoked_name --help > help.txt"
           to get a file named help.txt containing this info.
    --verbose
        ...Increase verbosity (additive)
EOR
  if ($full_help) {
    print <<EOR;
    --quiet
        ...Run silently
    --[no]addvar
        ...[do not add]/add variable definitions when they do not
           already exist in the job string (default is not to add them)
    --[no]remvar
        ...[do not remove]/remove null defined variables from the
           job string. If true then including vname= on the command line will
           remove all instances of the variable vname from the job string.
           The default is to remove.
    --[no]runid_substitute
        ...[do not replace]/replace the string containing the \"old\" value
           of runid with the \"new\" value of runid. The \"old\" value is the
           value found in the input job string/template/file and the new value
           is the value of runid set via the command line or set internally.
           The default is not to do this global substitution.
    --[no]template_expand_global
        ...[do not use]/use a sub shell to expand variable definitions
           found in the \"global\" section of a job string template.
           The default is to not do any variable expansion this way.
    --[no]template_insert_global
        ...[do not insert]/insert the \"global\" section into other
           sections found in a job string template. The default is to
           do these insertions. The same result may be achieved using
           vname=value pairs on the command line.
    --[no]tag
        ...[do not add]/add an info tag to the top of the output file.
           The default is to add the tag.
EOR
  };
  print "End of usage --- ",`date`;
  close(PG_PIPE);
  # Reset stdout (in case this routine returns control at some point in the future)
  if ($prev_output_fh) {select $prev_output_fh};
  exit 0;
}

sub show_JOBDEF {
  use strict;
  my ($msg) = shift;
  if (defined $msg) {print "$msg\n"};

  my ($OPTS) = shift;

  my $full_help = 0;
  if (defined $OPTS->{FULL_HELP}) {$full_help = $OPTS->{FULL_HELP}};

  printf "%20s  ::  ","file_name";
  print "any text file containing a job string or job string template found in\n";
  printf "%26s%s"," ","  1) the current working directory\n";
  my $homedir = (getpwuid($<))[7];
  printf "%26s%s"," ","any file with the name file_name_jobdef found in\n";
  printf "%26s%s"," ","  2) the directory $homedir/cccjobs\n";
  printf "%26s%s"," ","  3) any directory named in the environment variable CCCJOB_PATH\n";
  my $found = 0;
  print " " x 5,"---- predefined aliases ----\n";
  foreach (sort keys %main::Predef_job_seq) {
    $found = 1;
    printf "%20s  ::  ",$_;
    my $nl = 0;
    if (defined $main::Predef_job_seq{$_}{description}) {
      print $main::Predef_job_seq{$_}{description};
      $nl = 1;
    };
    if (defined $main::Predef_job_seq{$_}{cmd_line}) {
      print $nl==1 ? "\n" . " " x 26 : "";
      print "alias for ",$main::Predef_job_seq{$_}{cmd_line};
    };
    print "\n";
    if ($full_help) {
      if (exists $main::Predef_job_seq{$_}{comment}) {
        print " " x 26,"$main::Predef_job_seq{$_}{comment}\n";
      };
    };
  };
  print " " x 5,"---- predefined job strings ----\n";
  foreach (sort keys %{ $main::JOBDEF{job} }) {
    if (defined $main::JOBDEF{hide}{$_} ) {
      # Ignore jobs if hide directive exists and begins with a y or Y
      next if ($main::JOBDEF{hide}{$_} =~ /^ *[yY]/);
    };
    if (exists $main::JOBDEF{user_supplied}{$_}) {
      next if $main::JOBDEF{user_supplied}{$_};
    };
    $found = 1;
    printf "%20s  ::  ",$_;
    if (defined $main::JOBDEF{description}{$_} ) {
      print "$main::JOBDEF{description}{$_}\n";
    } else {
      if (defined $main::JOBDEF{job}{$_} ) {print "\n"}
      else {print "Invalid\n"};
    };
  };
  print " " x 5,"---- user supplied job strings ----\n";
  foreach (sort keys %{ $main::JOBDEF{job} }) {
    if (defined $main::JOBDEF{hide}{$_} ) {
      # Ignore jobs if hide directive exists and begins with a y or Y
      next if ($main::JOBDEF{hide}{$_} =~ /^ *[yY]/);
    };
    if (exists $main::JOBDEF{user_supplied}{$_}) {
      next unless $main::JOBDEF{user_supplied}{$_};
    };
    $found = 1;
    printf "%20s  ::  ",$_;
    if (defined $main::JOBDEF{description}{$_} ) {
      print "$main::JOBDEF{description}{$_}\n";
    } else {
      if (defined $main::JOBDEF{job}{$_} ) {print "\n"}
      else {print "Invalid\n"};
    };
    if ($full_help) {
      if (exists $main::JOBDEF{user_supplied}{$_}) {
        print " " x 26,"found in $main::JOBDEF{path}{$_}\n";
      };
    };
  };
  unless ($found) {print "         No valid keywords. JOBDEF hash is empty.\n"};
};


sub format_vlist {
  use strict;
  my @vin = @_;
  my @vout = ();
  my $vname;
  my $vval;
  my $vopt;
  my $str;

  # list all known condef parameters
  my @condef_param = ("acctcom","amiplot","auto","backup","begplot","bg",
                      "blckcld","both","cfs","cmc","crd","debug",
                      "ecmwf","fiz","float1","fullfan","gcm1","gcm2plus",
                      "gcmlist","gcmtsav","ggcld","legates","mom","msplot",
                      "multiexp","naplot","newtape","nextjob","nobeg","nodiag",
                      "nogcmsub","nojoin","nolist","noprint","nosic","noxlfimp",
                      "ocean","obsdat","obslevs","pakjob","pcpplot","plotlev",
                      "plotvsp","pooladd","pooled","prof","protect","pscale",
                      "pseason","psplot","qg","qplot","repfile","resfix",
                      "shade","splot","subarea","sv","svcrd","svvic",
                      "tplot","tssave","uplot","vic","vplot","xmu",
                      "xtracld","xtradif","zdplot","zxcld","gridpr","specpr",
                      "specsig","gpfile","gptfile","gpufile","gpvfile","gpwfile",
                      "gpxfile","gpzfile","spqfil","uxstats","vxstats","wxstats",
                      "ggsave","spsave","zxsave","initsp","samerun","script",
                      "parallel_io","openmp");

  foreach (@vin) {
    # args of the form vname=value
    ($vname,$vval) = split '=', $_, 2;
    ($vopt) = $vval =~ /(:\w*)$/;
    if ($vopt) {$vval =~ s/:\w*$//}  # strip options from the end of vval
    else {$vopt=''};                 # set vopt null to be used below

    # strip white space and quotes on condef parameters
    # then check for "on" or "off" values
    foreach (@condef_param) {
      if ($vname eq $_) {
        $vval =~ s/\s+//g;     # strip all white space
        $vval =~ /^".*"$/ and $vval =~ s/^"(.*)"$/$1/; # strip double quotes
        $vval =~ /^'.*'$/ and $vval =~ s/^'(.*)'$/$1/; # strip single quotes
        unless ($vval eq "off" or $vval eq "on") {
          $str = "A condef parameter $vname=$vval ";
          $str .= "has a value that is neither \"off\" nor \"on\".\n  Stopped";
          die "$str";
        };
      };
    };

    # remove white space from the value of variables used to construct file names
    if ($vname eq "runid" or $vname eq "flabel" or $vname eq "prefix"
        or $vname eq "uxxx") {
      $vval =~ s/\s*//g;
    };

    # certain variables require a particular format
    $vval = sprintf "%10d",   $vval if $vname eq "kfinal";
    $vval = sprintf "%5d",    $vval if $vname eq "ksteps";
    $vval = sprintf "%3.3d",  $vval if $vname eq "year";
    $vval = sprintf "%3.3d",  $vval if $vname eq "year_restart";
    $vval = sprintf "%2.2d",  $vval if $vname eq "mon";
    $vval = sprintf "%2.2d",  $vval if $vname eq "mon_restart";
    $vval = sprintf "%3.3d",  $vval if $vname eq "year_rdiag_start";
    $vval = sprintf "%3.3d",  $vval if $vname eq "run_start_year";
    $vval = sprintf "%3.3d",  $vval if $vname eq "run_stop_year";
    $vval = sprintf "%2.2d",  $vval if $vname eq "run_start_month";
    $vval = sprintf "%2.2d",  $vval if $vname eq "run_stop_month";
    $vval = sprintf "%5d",    $vval if $vname eq "issp";
    $vval = sprintf "%5d",    $vval if $vname eq "isgg";
    $vval = sprintf "%5d",    $vval if $vname eq "israd";
    $vval = sprintf "%5d",    $vval if $vname eq "isbeg";
    $vval = sprintf "%5d",    $vval if $vname eq "jlatpr";
    $vval = sprintf "%5d",    $vval if $vname eq "icsw";
    $vval = sprintf "%5d",    $vval if $vname eq "iclw";
    $vval = sprintf "%5d",    $vval if $vname eq "iepr";
    $vval = sprintf "%5d",    $vval if $vname eq "istr";
    $vval = sprintf "%5d",    $vval if $vname eq "isoc";
    $vval = sprintf "%5d",    $vval if $vname eq "ires";
    $vval = sprintf "%5d",    $vval if $vname eq "iyear";
    $vval = sprintf "%5d",    $vval if $vname eq "iday";
    $vval = sprintf "%5d",    $vval if $vname eq "incd";
    $vval = sprintf "%5.2f",  $vval if $vname eq "gmt";
    $vval = sprintf "%5d",    $vval if $vname eq "ifdiff";
    $vval = sprintf "%5d",    $vval if $vname eq "isen";
    $vval = sprintf "%5d",    $vval if $vname eq "lon";
    $vval = sprintf "%5d",    $vval if $vname eq "lond";
    $vval = sprintf "%5d",    $vval if $vname eq "ioztyp";
    $vval = sprintf "%5d",    $vval if $vname eq "ilev";
    $vval = sprintf "%5d",    $vval if $vname eq "levs";
    $vval = sprintf "%5d",    $vval if $vname eq "lmt";
    $vval = sprintf "%5d",    $vval if $vname eq "nlatd";
    $vval = sprintf "%5d",    $vval if $vname eq "lonsld";
    $vval = sprintf "%5d",    $vval if $vname eq "nlat";
    $vval = sprintf "%5d",    $vval if $vname eq "lonsl";
    $vval = sprintf "%5d",    $vval if $vname eq "ntrac";
    $vval = sprintf "%5d",    $vval if $vname eq "itraca";
    $vval = sprintf "%5s",    $vval if $vname eq "coord";
    $vval = sprintf "%5s",    $vval if $vname eq "moist";
    $vval = sprintf "%4s",    $vval if $vname eq "itrvar";
    $vval = sprintf "%10.4G", $vval if $vname eq "plid";
    $vval = sprintf "%9.1f",  $vval if $vname eq "delt";
    $vval = sprintf "%5d",    $vval if $vname eq "lay";
    $vval = sprintf "%5d",    $vval if $vname =~ /^g\d\d$/;
    $vval = sprintf "%5d",    $vval if $vname =~ /^h\d\d$/;
    # sref and spow are namelist input, no need no reformat
    # $vval = sprintf "%10.3G", $vval if $vname eq "sref";
    # $vval = sprintf "%10.3G", $vval if $vname eq "spow";

    # ocean variables
    $vval = sprintf "%5.5d",  $vval if $vname eq "iocean";
    $vval = sprintf "%5.5d",  $vval if $vname eq "icemod";
    $vval = sprintf "%5.5d",  $vval if $vname eq "ocnmod";
    $vval = sprintf "%5.5d",  $vval if $vname eq "rivmod";
    $vval = sprintf "%5.5d",  $vval if $vname eq "iflux";
    $vval = sprintf "%5.5d",  $vval if $vname eq "couplermod";
    $vval = sprintf "%5.5d",  $vval if $vname eq "ncount";
    $vval = sprintf "%7d",    $vval if $vname eq "atmsteps";
    $vval = sprintf "%7d",    $vval if $vname eq "ocnsteps";
    $vval = sprintf "%5.5d",  $vval if $vname eq "couplermod";
    $vval = sprintf "%5.5d",  $vval if $vname eq "biogeochem";
    $vval = sprintf "%5.5d",  $vval if $vname eq "ctemmod";

    # RCM variables
    $vval = sprintf "%10d",   $vval if $vname eq "MRCKSTART";
    $vval = sprintf "%10d",   $vval if $vname eq "MRCKTOTAL";
    $vval = sprintf "%10d",   $vval if $vname eq "MRCRADINIT";
    $vval = sprintf "%10d",   $vval if $vname eq "MRCRADFINI";
    $vval = sprintf "%5d",    $vval if $vname eq "MRCRADIDAY";
    $vval = sprintf "%5.2f",  $vval if $vname eq "MRCRADGMT";
    $vval = sprintf "%10d",   $vval if $vname eq "GCMKINIT";
    $vval = sprintf "%10d",   $vval if $vname eq "GCMKFINI";
    $vval = sprintf "%5d",    $vval if $vname eq "GCMARCH";
    $vval = sprintf "%5d",    $vval if $vname eq "MRCARC";
    $vval = sprintf "%10d",   $vval if $vname eq "MRCDATE";
    $vval = sprintf "%10.1f", $vval if $vname eq "DELTGCM";
    $vval = sprintf "%10.1f", $vval if $vname eq "DELT";
    $vval = sprintf "%5d",    $vval if $vname eq "LON";
    $vval = sprintf "%5d",    $vval if $vname eq "LAT";
    $vval = sprintf "%5d",    $vval if $vname eq "LRT";
    $vval = sprintf "%5d",    $vval if $vname eq "LMT";
    $vval = sprintf "%5d",    $vval if $vname eq "KTR";
    $vval = sprintf "%10.1f", $vval if $vname eq "HR_PWAT";

    if ($main::Verbose > 3) {
      printf "format_vlist :: vname=%-20svvalue=%-20s\n",$vname,$vval;
    };
    push @vout, "${vname}=$vval$vopt";
  };
  return @vout;
}

sub parse_job { my @job_in = @_;
  use strict;
  my $job;
  my @L1;
  my @L2;
  my $href = {};
  my %TEMPLATE = ();
  my %H1 = ();
  my %H2 = ();
  my %RET = ();
  my $str;
  my $i;
  my $sec;
  my $subsec;
  my $id;
  my $subid;
  my $doc_end;

  # First ensure that the job string array contains 1 job per element.
  # A job is terminated by a line of the form /^ *# *end_of_job *$/
  my @job_list=();
  foreach $str (@job_in) {
    if ($str =~ /(^|\n)#MARKER:/s) {
      # assume that this is a job string template and put the
      # expanded job string into $str
      %TEMPLATE = parse_job_template($str);
      $str = expand_template(%TEMPLATE);
    };
    # strip any whitespace following the /^ *# *end_of_job *$/ line
    $str =~ s/( *# *end_of_job *\n)\s*/$1/s;
    # read the file line by line breaking it into individual jobs delimited
    # at the end by a line of the form /^ *# *end_of_job *$/
    $job = "";
    foreach (split /\n/,$str) {
      $job = $job . $_ . "\n";
      if ($_ =~ /^ *# *end_of_job *$/ ) {
        push @job_list, $job;
        $job = "";
      }
    }
    # push whatever is left over onto the end of job_list
    if ($job) {
      push @job_list, $job;
      if ($main::Verbose > 5) {print "last job \n${job}"};
    }
  }

  # Add job delimiters if they do not already exist
  foreach (@job_list) {
    # Strip leading white space and prepend a shebang line to each job
    # that does not have one
    # unless (s@^\s*(#! */bin/[^\n]*\n)@$1@s) {$_ = "#!/bin/sh\n" . $_};
    # append an #end_of_job line if not present
    unless (s/(# *end_of_job)\s*$/$1\n/s) {$_ .= "#end_of_job\n"};
  };

  if ($main::Verbose > 10) {
    $i=1;
    foreach (@job_list) {print '-'x40,$i++,'-'x40,"\n",$_};
  };

  # subdivide each input job string and put into @L1
  @L1 = ();
  my $seqno=0;
  foreach $job (@job_list) {
    # Set the job sequence number
    $seqno++;
    # Split the job into an array of "standard" here documents
    %H1 = parse_job_here_docs($job);
    @L2 = ();
    while ($sec = shift @{$H1{section}}) {
      $id = shift @{$H1{id}};
      $doc_end = shift @{$H1{doc_end}};
      # Split each section into an array of subsections
      %H2 = parse_job_sections($sec, $doc_end);
      while ($subsec = shift @{$H2{section}}) {
        $subid = shift @{$H2{id}};
        # push a record onto the L2 array
        $href = {};
        $href->{job}      = $H1{job};  # the current job in a single string
        # $href->{jobdoc}  = $H2{job};  # the current here doc in a single string
        $href->{section}  = $subsec;
        $href->{id}       = $id;
        $href->{subid}    = $subid;
        $href->{sequence} = $seqno;
        push @L2, $href;
        if ($main::Verbose > 3) {
          print "+++++++++++++++ $id :: $subid ++++++++++++++++\n";
          print "$subsec\n";
        };
      };
    };
    push @L1, @L2;
  };
  return @L1;
}

sub parse_job_template { my ($template) = @_;
  use strict;
  # parse a job template string. The string should contain special lines
  # of the form /^#MARKER:.*/ which will be used to break the string into
  # subsections.
  my $section;
  my %TEMPLATE;
  my $Vtrip=6;
  my $stamp;
  my $tmpfile;
  my $cwext;
  my $sname;
  my $Ldelim;
  my $Rdelim;
  my $sec1;
  my $sec2;
  my $i;
  my $global;
  my $ret;
  my $n;

  if ($main::Verbose > $Vtrip) {
    print "\nParsing TEMPLATE found in job list:\n";
  }

  # "global definition" section
  $section="";
  foreach (split /\n/,$template) {
    last if (/^ *# *end_of_global_def/ or
             m=^#!/bin/sh=             or
             /^#deck /                 or
             /^ *# *MARKER:/);
    next if (/^#/ or /^\s*$/);
    $section .= $_ . "\n";
  }
  if ($section) {
    $section = "#  * " . '.' x 21 . " Definition of Global Parameters " .
                '.' x 21 . "\n" .$section;
  }

  if ($main::Template_Expand_Global and $section) {
    # Run this section through a subshell to expand variables.
    # This is a bad idea since it can easily lead to undefined variables
    # in the global section but is included as an option to be backward
    # compatible with the old create_run and create_diag scripts.
    # A better way is to add relevant variable defs to the VARS hash.

    # write to $tmpfile after sourcing all section lines in a subshell
    chomp($stamp = `date "+%H%M%S"$$`) unless ($stamp);
    $tmpfile="_${stamp}_";
    open(SH,"|sh") || die "cannot fork sh";
    # cwext added for backward compatability. Should use VARS hash instead
    if ($cwext) {print SH "cwext=$cwext}"};
    foreach (split /\n/,$section) {print SH $_,"\n"};
    print SH "set +x\ncat >$tmpfile <<EOF\n$section\nEOF\n";
    close SH || die "problem running sh";

    # read the contents of $tmpfile back into the $section variable
    $section = '';
    open (GLOBAL, "<$tmpfile") || die "cannot open $tmpfile";
    while (<GLOBAL>) {$section .= $_};
    close GLOBAL;
    unlink $tmpfile;
  }

  $TEMPLATE{global} = [($section)];

  foreach $sname ("monthly","yearly","special",
                  "annual","DJF","MAM","JJA","SON","quarterly") {
    # assign left and right delimiter regexes for each section
    if ($sname eq "monthly") {
      $Ldelim = '# *MARKER:[^\n]*Monthly Section[^\n]*\n';
      $Rdelim = '# *MARKER:[^\n]*End of Monthly[^\n]*\n';
    }
    elsif ($sname eq "yearly") {
      $Ldelim = '# *MARKER:[^\n]*Yearly Section[^\n]*\n';
      $Rdelim = '# *MARKER:[^\n]*End of Yearly[^\n]*\n';
    }
    elsif ($sname eq "special") {
      $Ldelim = '# *MARKER:[^\n]*Special Section[^\n]*\n';
      $Rdelim = '# *MARKER:[^\n]*End of Special[^\n]*\n';
    }
    elsif ($sname eq "annual") {
      $Ldelim = '# *MARKER:[^\n]*Annually Section[^\n]*\n';
      $Rdelim = '# *MARKER:[^\n]*End of Annually[^\n]*\n';
    }
    elsif ($sname eq "DJF") {
      $Ldelim = '# *MARKER:[^\n]*Seasonal DJF Section[^\n]*\n';
      $Rdelim = '# *MARKER:[^\n]*End of DJF Seasonal[^\n]*\n';
    }
    elsif ($sname eq "MAM") {
      $Ldelim = '# *MARKER:[^\n]*Seasonal MAM Section[^\n]*\n';
      $Rdelim = '# *MARKER:[^\n]*End of MAM Seasonal[^\n]*\n';
    }
    elsif ($sname eq "JJA") {
      $Ldelim = '# *MARKER:[^\n]*Seasonal JJA Section[^\n]*\n';
      $Rdelim = '# *MARKER:[^\n]*End of JJA Seasonal[^\n]*\n';
    }
    elsif ($sname eq "SON") {
      $Ldelim = '# *MARKER:[^\n]*Seasonal SON Section[^\n]*\n';
      $Rdelim = '# *MARKER:[^\n]*End of SON Seasonal[^\n]*\n';
    }
    elsif ($sname eq "quarterly") {
      $Ldelim = '# *MARKER:[^\n]*Quarterly Section[^\n]*\n';
      $Rdelim = '# *MARKER:[^\n]*End of Quarterly[^\n]*\n';
    }
    else {
      print "ERROR looking for $sname section in template\n";
      next;
    };
    # put all sections found in the template into S_list
    my @S_list = ();
    ($section) = $template =~ /(?:$Ldelim)(.*)(?:$Rdelim)/s;
    if ($section) {
      while ($section =~ /($Ldelim|$Rdelim)/) {
        # Find multiple ocurrences of this section
        ($sec1) = $section =~ /^(.*?)(?:$Rdelim)/s;
        ($sec2) = $section =~ /(?:$Ldelim)(.*)/s;
        push @S_list, $sec1;
        $section = $sec2;
      };
      push @S_list,$section;
      #strip leading white space on each section
      foreach (@S_list) {s/^\s+//};
    };
    if ($main::Verbose > $Vtrip) {
      foreach $i (0 .. $#S_list) {
        print '='x40," $sname section ${i}:\n$S_list[$i]";
      };
    };
    $TEMPLATE{$sname} = [ @S_list ];
  };

  # Look for "Every N Months" or "Every N Years" sections
  my @Llist = ();
  my @V2 = ();

# print "template:\n$template\n";
# my @Mlist = $template =~ /(^\s*#\s*MARKER:.*)/gm;
# print "Mlist:\n @Mlist\n";

  # Get a list of all left delimiters for "Every N Months" sections found in the template
  @Llist = $template =~ /(# *MARKER:[^\n]*Every +-?\d+ +Months[^\n]*\n)/gs;
  # remove duplicate values from Llist
  @V2 = @Llist;
  foreach my $delim (@V2) {
    my ($N,$rem) = $delim =~ /# *MARKER:[^\n]*Every +(-?\d+) +Months(.*)/;
    $delim = '# *MARKER:[^\n]*Every +'.$N.' +Months$rem';
    my $found = 0;
    @Llist = grep {$found+=/$delim/; if (!/$delim/ or ($found == 1)){1}else{0}} @Llist;
  };
  if ($main::Verbose > 10) {print "Every N Months Llist = \n @Llist\n"};

  # Loop over the left delimiter list extracting sections and adding them to TEMPLATE
  foreach my $curr_Ldelim (@Llist) {
    my ($N,$rem) = $curr_Ldelim =~ /# *MARKER:[^\n]*Every +(-?\d+) +Months(.*)/;
    my ($opts) = $rem =~ /(<.*>)/;
    # revert curr_Ldelim to a regex
    $curr_Ldelim = '# *MARKER:[^\n]*Every +'.$N.' +Months' . $rem . '\n';
    # Strip any leading "Section" from rem
    $rem =~ s/^\s*Section\s*//i;
    my $curr_Rdelim = '# *MARKER:[^\n]*End of +'.$N.' +Months' . $rem. '\n';

    my $curr_sname = 'every'.$N.'months';
    $curr_sname .= $opts if $opts;

    # put all sections found in the template into S_list
    my @S_list = ();
    my $Ld = '';
    my $Rd = '';
    ($Ld,$section,$Rd) = $template =~ /($curr_Ldelim)(.*)($curr_Rdelim)/s;
    if ($section) {
      while ($section =~ /($curr_Ldelim|$curr_Rdelim)/) {
        # Find multiple ocurrences of this section
        ($sec1) = $section =~ /^(.*?)(?:$curr_Rdelim)/s;
        ($sec2) = $section =~ /(?:$curr_Ldelim)(.*)/s;
        push @S_list, $sec1;
        $section = $sec2;
      }
      push @S_list,$section;
      #strip leading white space on each section
      foreach (@S_list) {s/^\s+//};
    }
    if ($main::Verbose > 10) {
      foreach $i (0 .. $#S_list) {
        print '='x40," $curr_sname section ${i}:\n$S_list[$i]";
      }
    }
    $TEMPLATE{$curr_sname} = [ @S_list ];
    if ($main::Verbose > 10) {
      print "$curr_Ldelim\n";
      my $count = 0;
      foreach (@S_list) {
        $count++;
        foreach ( split_job_string($_) ) {
          my $jobname = read_jobtag($_,{NAME_TAG=>1});
          print "*** $count  jobname: $jobname\n";
        }
      }
    }
  }

  # Get a list of all left delimiters for "Every N Years" sections found in the template
  @Llist = $template =~ /(# *MARKER:[^\n]*Every +-?\d+ +Years[^\n]*\n)/gs;
  # remove duplicate values from Llist
  @V2 = @Llist;
  foreach my $delim (@V2) {
    my ($N,$rem) = $delim =~ /# *MARKER:[^\n]*Every +(-?\d+) +Years(.*)/;
    $delim = '# *MARKER:[^\n]*Every +'.$N.' +Years$rem';
    my $found = 0;
    @Llist = grep {$found+=/$delim/; if (!/$delim/ or ($found == 1)){1}else{0}} @Llist;
  };
  if ($main::Verbose > 10) {print "Every N Years Llist = @Llist\n"};

  # Loop over the left delimiter list extracting sections and adding them to TEMPLATE
  foreach my $curr_Ldelim (@Llist) {
    my ($N,$rem) = $curr_Ldelim =~ /# *MARKER:[^\n]*Every +(-?\d+) +Years(.*)/;
    my ($opts) = $rem =~ /(<.*>)/;
    # revert curr_Ldelim to a regex
    $curr_Ldelim = '# *MARKER:[^\n]*Every +'.$N.' +Years[^\n]*\n';
    my $curr_Rdelim = '# *MARKER:[^\n]*End of +'.$N.' +Years[^\n]*\n';
    my $curr_sname = 'every'.$N.'years';
    $curr_sname .= $opts if $opts;

    # put all sections found in the template into S_list
    my @S_list = ();
    ($section) = $template =~ /(?:$curr_Ldelim)(.*)(?:$curr_Rdelim)/s;
    if ($section) {
      while ($section =~ /($curr_Ldelim|$curr_Rdelim)/) {
        # Find multiple ocurrences of this section
        ($sec1) = $section =~ /^(.*?)(?:$curr_Rdelim)/s;
        ($sec2) = $section =~ /(?:$curr_Ldelim)(.*)/s;
        push @S_list, $sec1;
        $section = $sec2;
      };
      push @S_list,$section;
      #strip leading white space on each section
      foreach (@S_list) {s/^\s+//};
    }
    if ($main::Verbose > 10) {
      foreach $i (0 .. $#S_list) {
        print '='x40," $curr_sname section ${i}:\n$S_list[$i]";
      }
    }
    $TEMPLATE{$curr_sname} = [ @S_list ];
  }

  # Insert the global section into other sections before a line of the form
  # /#.*Parmsub Parameters/ but only if $Template_Insert_Global is set.
  # only the first global section found in the template will be used
  ($global) = @{ $TEMPLATE{global} };
  if ($main::Template_Insert_Global and $global) {
    foreach $sname (keys %TEMPLATE) {
      if ($sname eq "global") {next};
      foreach $i (0 .. $#{ $TEMPLATE{$sname} }) {
        $ret = $TEMPLATE{$sname}[$i] =~
                s/(#.*Parmsub Parameters.*$)/\n$global$1/gm;
        if ($ret and $main::Verbose > $Vtrip) {
          print "Substitutions of global defs into $sname section: $ret\n";
        };
      };
    };
  };

  if ($main::Verbose > $Vtrip) {
    # print each member of TEMPLATE
    foreach $sname (keys %TEMPLATE) {
      $n = scalar(@{ $TEMPLATE{$sname} });
      printf "Number of %10s sections found in TEMPLATE: %d\n",$sname,$n;
    };
  };
  if ($main::Verbose > $Vtrip) {
    # print each member of TEMPLATE
    foreach $sname (keys %TEMPLATE) {
      foreach $i (0 .. $#{ $TEMPLATE{$sname} }) {
        printf "%s TEMPLATE %s %20s section %d\n",'='x15,'='x15,$sname,$i;
        print $TEMPLATE{$sname}[$i];
      };
    };
    print "\n";
  };

  return %TEMPLATE;
}

sub expand_template { my (%TEMPLATE) = @_;
  use strict;
  # Create a job string from the info supplied in the hash of lists TEMPLATE.
  # Only the first element (if any) of the arrays assigned to each keyword
  # in the TEMPLATE hash will be used. The keywords are currently
  # "monthly","yearly","special","annual","DJF","MAM","JJA","SON","quarterly",
  # /every-?\d+years/,/every-?\d+months/
  # Each keyword represents a (possibly null) section that may be inserted
  # into the job string at particular intervals.

  # use private copies of parmsub and condef parameters
  my $delt; my $months; my $months_run; my $samerun; my $initsp; my $kfinal; my $year;
  my $mon; my $year_restart; my $mon_restart;
  my $cpl_new_rtd; my $nemo_new_rtd;

  my $Vtrip=6;
  my $str;
  my $offset_year;
  my $offset_kount=0;
  my $start_mon;
  my $end_mon;
  my $previous_year;
  my $previous_month;
  my %annual;
  my %quarterly;
  my %monthly;
  my %DJF;
  my %MAM;
  my %JJA;
  my %SON;
  my %yearly;
  my %special;
  my %prevyear;
  my %prevmonth;
  my $rv1;
  my $rv2;
  my $rv3;
  my $rv4;
  my $rv5;
  my $rv6;
  my $rv7;
  my $rv8;
  my $rv9;
  my $rv10;
  my $rv11;
  my $W1;
  my $W2;
  my $W3;
  my $W4;
  my $W5;
  my $W6;
  my $W7;
  my $W8;
  my $W9;
  my $current_month;
  my $next_year;
  my $next_month;
  my $next_kount;

  # month names
  my @MONname=("JAN","FEB","MAR","APR","MAY","JUN",
             "JUL","AUG","SEP","OCT","NOV","DEC");

  # number of days in each month of the year
  my @MONdays=(0,31,28,31,30,31,30,31,31,30,31,30,31);

  # number of days since Jan 1 to the end of each month
  my @eom_day=(0,31,59,90,120,151,181,212,243,273,304,334,365);

  # Define a function that will return true (1) or false (0) given a string value
  my $ToF = sub {
    my ($str) = shift;
    my $ret = -1;
    $ret = 0 if $str =~ /^no$/i;
    $ret = 0 if $str =~ /^off$/i;
    $ret = 0 if $str =~ /^0$/i;
    $ret = 1 if $str =~ /^yes$/i;
    $ret = 1 if $str =~ /^on$/i;
    $ret = 1 if $str =~ /^1$/i;
    if ($ret < 0) {
      die "ToF: Invalid input value --> $str <--\n Stopped";
    }
    return wantarray ? ($ret) : $ret;
  };

  # Get a list of keys in TEMPLATE of the form everyNyears and everyNmonths
  # as well as a list of all job names found in TEMPLATE
  my @Nyearskey = ();
  my @Nmonthskey = ();
  my @AllJobNames;
  foreach (keys %TEMPLATE) {
    /every-?\d+years/  and do {push @Nyearskey, $_};
    /every-?\d+months/ and do {push @Nmonthskey, $_};
    ($str) = @{$TEMPLATE{$_}};
    next unless $str;
    push @AllJobNames, read_jobtag($str,{NAME_TAG=>1});
    # push $job_by_freq{$_}, read_jobtag($str,{NAME_TAG=>1});
  }

  if (scalar(@Nyearskey)) {
    # Sort these keys according to frequency
    my @negkeys = ();
    my @poskeys = ();
    foreach (@Nyearskey) {
      my ($N) = /every(-?\d+)years/;
      if ($N<0) {push @negkeys,$_}
      else      {push @poskeys,$_}
    }
    sub yearbyfreq {
      # Negative frequencies will be montonically decreasing
      # Positive frequencies will be montonically increasing
      my ($Na) = $a =~ /every-?(\d+)years/;
      my ($Nb) = $b =~ /every-?(\d+)years/;
      $Na <=> $Nb;
    }
    undef @Nyearskey;
    push @Nyearskey, sort yearbyfreq @negkeys;
    push @Nyearskey, sort yearbyfreq @poskeys;
  }

  if (scalar(@Nmonthskey)) {
    # Sort these keys according to frequency
    my @negkeys = ();
    my @poskeys = ();
    foreach (@Nmonthskey) {
      my ($N) = /every(-?\d+)months/;
      if ($N<0) {push @negkeys,$_}
      else      {push @poskeys,$_}
    }
    sub monbyfreq {
      # Negative frequencies will be montonically decreasing
      # Positive frequencies will be montonically increasing
      my ($Na) = $a =~ /every-?(\d+)months/;
      my ($Nb) = $b =~ /every-?(\d+)months/;
      $Na <=> $Nb;
    }
    undef @Nmonthskey;
    push @Nmonthskey, sort monbyfreq @negkeys;
    push @Nmonthskey, sort monbyfreq @poskeys;
  }

  if ($main::Verbose > 10) {print "AllJobNames = @AllJobNames\n"};
  if ($main::Verbose > 10) {print "Nyearskey = @Nyearskey\n"};
  if ($main::Verbose > 10) {print "Nmonthskey:\n",join("\n",@Nmonthskey),"\n"};

  # Look for delt in the VARS hash (value input from the command line).
  # If it is not found there look in the "months" section of the
  # TEMPLATE hash (read from the input file). If not in either place
  # then delt will default to 1200 (3 time steps per hour).
  if (exists $main::VARS{delt}) {
    $delt = $main::VARS{delt}{value};
  } else {
    ($str) = @{$TEMPLATE{monthly}};
    ($delt) = shell_var_subs::var_find($str,"delt");
  };
  unless ($delt) {
    $delt = "   1200.0";
    if ($main::Verbose > 1) {
      warn "\n*** WARNING *** No input value for delt was provided ...delt is set to $delt\n";
      push @main::warnings, "*** WARNING *** No input value for delt was provided ...delt is set to $delt";
    };
  };
  if ($delt < 1) {die "delt=$delt is out of range, stopped"};


  # DELTGCM is the time step for the GCM according to the RCM parameter file
  my $rcm_DELTGCM = '';
  if (exists $main::VARS{DELTGCM}) {
    $rcm_DELTGCM = $main::VARS{DELTGCM}{value};
  }
  elsif ($main::RCM_PARAMS) {
    ($rcm_DELTGCM) = var_find($main::RCM_PARAMS,"DELTGCM");
  };
  unless ($rcm_DELTGCM) {
    $rcm_DELTGCM = "   1200.0";
  };

  if ($main::MK_RCM_DATA) {
    # if rcm_init was invoked on the command line then ensure
    # that DELTGCM and delt are consistent
    unless ($delt == $rcm_DELTGCM) {
      print "       delt=$delt\nrcm_DELTGCM=$rcm_DELTGCM\n";
      warn "GCM time step is reset to delt=$rcm_DELTGCM\n";
      push @main::warnings, "GCM time step is reset to delt=$rcm_DELTGCM";
      $delt = $rcm_DELTGCM;
    };
  };

  # Determine a time step for the RCM
  my $rcm_DELT;
  if (exists $main::VARS{DELT}) {
    $rcm_DELT = $main::VARS{DELT}{value};
  } else {
    ($rcm_DELT) = shell_var_subs::var_find($main::RCM_PARAMS,"DELT");
  };
  unless ($rcm_DELT) {
    $rcm_DELT = "    900.0";
    if ($main::MK_RCM_DATA and $main::Verbose > 1) {
      warn "\n*** WARNING *** No input value for RCM DELT was provided ...set to $rcm_DELT\n";
      push @main::warnings, "*** WARNING *** No input value for RCM DELT was provided ...set to $rcm_DELT";
    };
  };

  my $steps_per_hour = 3600.0/$delt;
  my $steps_per_day = 86400.0/$delt;
  my $steps_per_year = 365*$steps_per_day;

  # ksteps must be set so that it is at least as large as the
  # number of steps in the longest month given the current delt
  my $ksteps = 31*$steps_per_day;
  $ksteps = sprintf "%5d",$ksteps;
  unless (exists $main::VARS{ksteps}) {
    # add this to the VARS hash only if it is not already there
    # this way a user may override this computed value for ksteps
    my $rank = 0;
    foreach (keys %main::VARS) {
      # determine the highest rank currently in the VARS hash
      $main::VARS{$_}{rank} > $rank and $rank = $main::VARS{$_}{rank}
    };
    $main::VARS{ksteps}{value}  = $ksteps;
    $main::VARS{ksteps}{add}    = $main::Addvar;
    $main::VARS{ksteps}{remove} = $main::Remvar;
    $main::VARS{ksteps}{rank}   = ++$rank;
  };

  # define an array containing the number of time steps from hour 00:00
  # to the end of hour nn:00 of the day (nn=00..24)
  my @eoh_kount;
  foreach (0..24) {push @eoh_kount,$_*$steps_per_hour};

  # define an array containing the number of time steps from Jan 1 (hour 00:00)
  # to the end of each day of the year
  my @eod_kount;
  foreach (0..365) {push @eod_kount,$_*$steps_per_day};

  # define an array containing the number of time steps to the end of each month
  my @eom_kount;
  foreach (@eom_day) {push @eom_kount,$_*$steps_per_day};

  # define timestep counts at the end of each month for the RCM
  my $rcm_steps_per_hour = 3600.0 / $rcm_DELT;
  my $rcm_steps_per_day = 86400.0 / $rcm_DELT;
  my $rcm_steps_per_year = 365*$rcm_steps_per_day;
  my @rcm_eom_kount;
  foreach (@eom_day) {push @rcm_eom_kount,$_*$rcm_steps_per_day};

  # MRCARC is the master saving interval (I5 format) in an RCM run
  my $rcm_MRCARC = '';
  if (exists $main::VARS{MRCARC}) {
    $rcm_MRCARC = $main::VARS{MRCARC}{value};
  }
  else {
    ($rcm_MRCARC) = var_find($main::RCM_PARAMS,"MRCARC");
  };

  # MRCDATE is date and time (yyyymmddhh format) at the start of an RCM run
  my $rcm_MRCDATE = '';
  if (exists $main::VARS{MRCDATE}) {
    $rcm_MRCDATE = $main::VARS{MRCDATE}{value};
  }
  else {
    ($rcm_MRCDATE) = var_find($main::RCM_PARAMS,"MRCDATE");
  };

  if ($main::Verbose > 1) {
    print "delt=$delt  steps_per_day=$steps_per_day  steps_per_year=$steps_per_year\n";
    print "eom_kount:  ",join(" ",@eom_kount),"\n";
  };

  # Set a flag if DJF for run_start_year is to be included in the string
  my $force_first_djf = 0;
  if (exists $main::VARS{force_first_djf} and defined $main::VARS{force_first_djf}) {
    $force_first_djf = &$ToF( $main::VARS{force_first_djf}{value} );
  }

  # Set a flag to indicate whether or not a job should be included to
  # pool the first season in the job string if less than 3 months preceed
  # the end of that season
  # e.g. If the run starts in month 4 and force_first_sea is true then
  # the a job to pool MAM will be inserted even though the first month (March)
  # is not in teh job string. It is assumed that the user will supply the
  # missing month of data (the missing diagnostic files for March)
  my $force_first_sea = 0;
  if (exists $main::VARS{force_first_sea} and defined $main::VARS{force_first_sea}) {
    $force_first_sea = &$ToF( $main::VARS{force_first_sea}{value} );
  }

  # define *_mon_offset variables used to add offsets to the default
  # current_(month|year), previous_(month|year) and next_(month|year)
  # variables that are used by some jobs (e.g. mdump, mload, mdelete,
  # load_list, dump_list, del_list)
  my $mload_mon_offset = 0;
  if (exists $main::VARS{mload_mon_offset}) {
    $mload_mon_offset = $main::VARS{mload_mon_offset}{value};
  };
  my $mdump_mon_offset = 0;
  if (exists $main::VARS{mdump_mon_offset}) {
    $mdump_mon_offset = $main::VARS{mdump_mon_offset}{value};
  };
  my $mdelete_mon_offset = 0;
  if (exists $main::VARS{mdelete_mon_offset}) {
    $mdelete_mon_offset = $main::VARS{mdelete_mon_offset}{value};
  };
  my $load_list_mon_offset = 0;
  if (exists $main::VARS{load_list_mon_offset}) {
    $load_list_mon_offset = $main::VARS{load_list_mon_offset}{value};
  };
  my $dump_list_mon_offset = 0;
  if (exists $main::VARS{dump_list_mon_offset}) {
    $dump_list_mon_offset = $main::VARS{dump_list_mon_offset}{value};
  };
  my $del_list_mon_offset = 0;
  if (exists $main::VARS{del_list_mon_offset}) {
    $del_list_mon_offset = $main::VARS{del_list_mon_offset}{value};
  };
  my $custom_mon_offset = 0;
  if (exists $main::VARS{custom_mon_offset}) {
    $custom_mon_offset = $main::VARS{custom_mon_offset}{value};
  };
  my $cloop_mon_offset = 0;
  if (exists $main::VARS{cloop_mon_offset}) {
    $cloop_mon_offset = $main::VARS{cloop_mon_offset}{value};
  };

  # $months is the number of months to run per submission. It can have any
  # integer value greater than 0. If it is not found in the VARS hash (not
  # input via the command line) then it will default to 1.
  if (exists $main::VARS{months}) {
    $months = $main::VARS{months}{value};
  } else {
    $months = 1;
    # Add the months variable to the VARS hash so that the value of
    # months in the final job string will be consistent throughout
    my $rank = 0;
    foreach (keys %main::VARS) {
      # determine the highest rank currently in the VARS hash
      $main::VARS{$_}{rank} > $rank and $rank = $main::VARS{$_}{rank}
    };
    $main::VARS{months}{value}  = $months;
    $main::VARS{months}{add}    = $main::Addvar;
    $main::VARS{months}{remove} = $main::Remvar;
    $main::VARS{months}{rank}   = 1+$rank;
  };
  # $months_run is the number of months to run per execution. It can have any
  # integer value greater than 0. If it is not found in the VARS hash (not
  # input via the command line) then it will default to 1.
  if (exists $main::VARS{months_run}) {
    $months_run = $main::VARS{months_run}{value};
  } else {
    $months_run = 1;
    # Add the months_run variable to the VARS hash so that the value of
    # months_run in the final job string will be consistent throughout
    my $rank = 0;
    foreach (keys %main::VARS) {
      # determine the highest rank currently in the VARS hash
      $main::VARS{$_}{rank} > $rank and $rank = $main::VARS{$_}{rank}
    };
    $main::VARS{months_run}{value}  = $months_run;
    $main::VARS{months_run}{add}    = $main::Addvar;
    $main::VARS{months_run}{remove} = $main::Remvar;
    $main::VARS{months_run}{rank}   = 1+$rank;
  };
  if ($months =~ /\//) {
    # months is of the form p/q ...request q submissions for every p months
    # e.g. months=1/3 means resubmit 3 times every month
    my ($p,$q) = split '/',$months;
    # strip all white space from p and q
    $p =~ s/\s*//g;
    $q =~ s/\s*//g;
    $p =~ /^[0-9]+$/
      or die "Numerator of months value ($p) is not a positive integer.\n Stopped";
    $q =~ /^[0-9]+$/
      or die "Denominator of months value ($q) is not a positive integer.\n Stopped";
    if ($q <= 1) {die "Denominator of months value ($q) is out of range.\n Stopped"};
    $months=$p;
    # ksteps must be set so that $q*$ksteps is ge 31 days
    $ksteps = ($months*31*$steps_per_day)/$q + 1;
    $ksteps = sprintf "%5d",$ksteps;

    # add or overwrite months and ksteps in the VARS hash
    my $rank = 0;
    foreach (keys %main::VARS) {
      # determine the highest rank currently in the VARS hash
      $main::VARS{$_}{rank} > $rank and $rank = $main::VARS{$_}{rank}
    };
    if (exists $main::VARS{months}) {
      $main::VARS{months}{value}  = $months;
    } else {
      $main::VARS{months}{value}  = $months;
      $main::VARS{months}{add}    = $main::Addvar;
      $main::VARS{months}{remove} = $main::Remvar;
      $main::VARS{months}{rank}   = ++$rank;
    };
    if (exists $main::VARS{ksteps}) {
      $main::VARS{ksteps}{value}  = $ksteps;
    } else {
      $main::VARS{ksteps}{value}  = $ksteps;
      $main::VARS{ksteps}{add}    = $main::Addvar;
      $main::VARS{ksteps}{remove} = $main::Remvar;
      $main::VARS{ksteps}{rank}   = ++$rank;
    };
    if ($main::Verbose > 0) {
      print "\nUser has requested $q submission",($q>1)?"s":""," every ",
             ($months>1)?"$months months":"month"," ...";
      print "setting months=$months and ksteps=$ksteps\n";
    };

    # verify that ksteps is in the job string
    ($str) = @{$TEMPLATE{monthly}};
    my ($ksteps_in) = shell_var_subs::var_find($str,"ksteps");
    unless ($ksteps_in) {
      print "\nA request for $q submission",($q>1)?"s":""," every ",
             ($months>1)?"$months months":"month"," was made\n";
      print "This requires the parmsub parameter ksteps, which was not found in the job string.\n";
      die "Stopped";
    };
  };
  if ($months < 1) {die "months=$months is out of range, stopped"};

  # Remove initsp, samerun, year, mon, year_restart, mon_restart, kfinal,
  # days, obsday, mon1, mon2 and mon3 from the VARS hash so that a global
  # replacement does not overwrite values set here. All of these values are
  # set here and may change from job to job in the job string.
  if (exists $main::VARS{year}) {delete $main::VARS{year}};
  if (exists $main::VARS{mon}) {delete $main::VARS{mon}};
  if (exists $main::VARS{year_restart}) {delete $main::VARS{year_restart}};
  if (exists $main::VARS{mon_restart}) {delete $main::VARS{mon_restart}};
  # if (exists $main::VARS{days}) {delete $main::VARS{days}};
  # if (exists $main::VARS{obsday}) {delete $main::VARS{obsday}};
  if (exists $main::VARS{mon1}) {delete $main::VARS{mon1}};
  if (exists $main::VARS{mon2}) {delete $main::VARS{mon2}};
  if (exists $main::VARS{mon3}) {delete $main::VARS{mon3}};
  my $kfinal_in_vlist = '';
  if (exists $main::VARS{kfinal}) {
    $kfinal_in_vlist = $main::VARS{kfinal}{value};
    delete $main::VARS{kfinal}
  };
  if (exists $main::VARS{samerun}) {
    $samerun = $main::VARS{samerun}{value};
    if ($main::Verbose > 2) {print "found in VARS: samerun = $samerun\n"};
    delete $main::VARS{samerun};
  } else {
    $samerun = "off";
  }
  if (exists $main::VARS{initsp}) {
    $initsp = $main::VARS{initsp}{value};
    if ($main::Verbose > 2) {print "found in VARS: initsp = $initsp\n"};
    delete $main::VARS{initsp};
  } else {
    $initsp = "on";
  }
  if (exists $main::VARS{cpl_new_rtd}) {
    $cpl_new_rtd = $main::VARS{cpl_new_rtd}{value};
    if ($main::Verbose > 2) {print "found in VARS: cpl_new_rtd = $cpl_new_rtd\n"};
    delete $main::VARS{cpl_new_rtd};
  } else {
    $cpl_new_rtd = "on";
  }
  if (exists $main::VARS{nemo_new_rtd}) {
    $nemo_new_rtd = $main::VARS{nemo_new_rtd}{value};
    if ($main::Verbose > 2) {print "found in VARS: nemo_new_rtd = $nemo_new_rtd\n"};
    delete $main::VARS{nemo_new_rtd};
  } else {
    $nemo_new_rtd = "on";
  }

  # Also remove RCM variables M1,M2,RUNP,MRCKSTART,MRCKTOTAL,MRCRADINIT,
  # MRCRADFINI,MRCRADIDAY,GCMKINIT,GCMKFINI from the VARS hash
  if (exists $main::VARS{M1})         {delete $main::VARS{M1}};
  if (exists $main::VARS{M2})         {delete $main::VARS{M2}};
  if (exists $main::VARS{RUNP})       {delete $main::VARS{RUNP}};
  if (exists $main::VARS{MRCKSTART})  {delete $main::VARS{MRCKSTART}};
  if (exists $main::VARS{MRCKTOTAL})  {delete $main::VARS{MRCKTOTAL}};
  if (exists $main::VARS{MRCRADINIT}) {delete $main::VARS{MRCRADINIT}};
  if (exists $main::VARS{MRCRADFINI}) {delete $main::VARS{MRCRADFINI}};
  if (exists $main::VARS{MRCRADIDAY}) {delete $main::VARS{MRCRADIDAY}};
  if (exists $main::VARS{GCMKINIT})   {delete $main::VARS{GCMKINIT}};
  if (exists $main::VARS{GCMKFINI})   {delete $main::VARS{GCMKFINI}};

  if (exists $main::VARS{offset_kount}) {
    $offset_kount = $main::VARS{offset_kount}{value}
  } else {
    $offset_kount = 0;
  };

  # Define variables needed to specify the length of the job string
  # These variables are available in the start_time and stop_time hashes
  # that are defined in main.
  my $run_start_year = 1;
  if (exists $main::start_time{year}{value}) {
    $run_start_year = $main::start_time{year}{value};
  };
  my $run_end_year = $run_start_year;
  if (exists $main::stop_time{year}{value}) {
    $run_end_year = $main::stop_time{year}{value};
  };
  my $run_start_month = 1;
  if (exists $main::start_time{month}{value}) {
    $run_start_month = $main::start_time{month}{value};
  };
  my $run_end_month = 12;
  if (exists $main::stop_time{month}{value}) {
    $run_end_month = $main::stop_time{month}{value};
  };
  my $run_start_dom = 1;
  if (exists $main::start_time{day}{value}) {
    $run_start_dom = $main::start_time{day}{value};
  };
  my $run_end_dom = -1;
  if (exists $main::stop_time{day}{value}) {
    $run_end_dom = $main::stop_time{day}{value};
  };

  if ($main::start_time{day}{set_by_user} or
      $main::stop_time{day}{set_by_user}) {
    # when either run_start_dom or run_end_dom is set
    # on the command line make sure that months=1
    if ($months != 1) {
      print "*** WARNING *** months has been reset to 1 because\n";
      print "                start day or end day was specified by the user\n";
      push @main::warnings, "*** WARNING *** months has been reset to 1 because";
      push @main::warnings, "                start day or end day was specified by the user";
    };
    $months = 1;
    $main::VARS{months}{value}  = $months;
  };

  if ($kfinal_in_vlist) {
    # When kfinal is set on the command line it will override the run_end...
    # variables that were set above.

    # when kfinal is set on the command line set months=1
    # This should eventually be changed to allow arbitrary values for months
    # but is a simple fix for the short term
    if ($months != 1) {
      print "*** WARNING *** months has been reset to 1 because\n";
      print "                kfinal was specified by the user\n";
      push @main::warnings, "*** WARNING *** months has been reset to 1 because";
      push @main::warnings, "                kfinal was specified by the user";
    };
    $months = 1;
    $main::VARS{months}{value}  = $months;

    my $kfinal = 1*$kfinal_in_vlist;
    if ($kfinal < 0) {
      die "*** ERROR *** kfinal=$kfinal is out of range.\n  Stopped";
    };
    if ($kfinal%$steps_per_year == 0) {
      $run_end_year = int($kfinal/$steps_per_year);
    } else {
      $run_end_year = int($kfinal/$steps_per_year+1);
    };
    if ($run_end_year < 1) {$run_end_year=1};
    my $steps_in_last_year = $kfinal - ($run_end_year-1)*$steps_per_year;
    my $i=0;
    foreach (@eom_kount) {
      $i++;
      if ($steps_in_last_year <= $_) {last};
    };
    $run_end_month = ($i>1) ? $i-1 : 1;
    my $steps_in_last_month = $steps_in_last_year - $eom_kount[$run_end_month-1];
    if ($steps_in_last_month%$steps_per_day == 0) {
      $run_end_dom = int($steps_in_last_month/$steps_per_day);
    } else {
      $run_end_dom = int($steps_in_last_month/$steps_per_day+1);
    };
    if ($run_end_dom < 1) {$run_end_dom=1};

    # ensure that the run_start_... variables are consistent with these new
    # values for the run_end_... variables
    if ($run_start_year > $run_end_year) {
      # Start from Jan 1 of $run_end_year
      $run_start_year=$run_end_year;
      $run_start_month=1;
      $run_start_dom=1;
    };
    if (($run_start_year == $run_end_year) and
        ($run_start_month > $run_end_month)) {
      # Start from Jan 1 of $run_end_year
      $run_start_month=1;
      $run_start_dom=1;
    };
    if (($run_start_year  == $run_end_year) and
        ($run_start_month == $run_end_month) and
        ($run_start_dom   > $run_end_dom)) {
      # Start from the first day of $run_end_dom
      $run_start_dom=1;
    };
    if ($main::Verbose > 10) {
      print "steps_in_last_year=$steps_in_last_year   ";
      print "steps_in_last_month=$steps_in_last_month\n";
      print "run_start_year=$run_start_year  run_start_month=$run_start_month   ";
      print "run_start_dom=$run_start_dom\n";
      print "run_end_year=$run_end_year    run_end_month=$run_end_month     ";
      print "run_end_dom=$run_end_dom   kfinal=$kfinal\n";
    };
  };

  # when starting from initial conditions the first month must be January
  if ($initsp eq "on" and ($run_start_month != 1 or $run_start_dom != 1)) {
    warn "\n*** WARNING *** The start month and day was changed to JAN 1 because
                the job is starting from initial conditions.\n";
    push @main::warnings,
      "*** WARNING *** The start month and day was changed to JAN 1 because
                the job is starting from initial conditions.";
    $run_start_month = 1;
    $run_start_dom=1;
    $main::start_time{month}{value} = $run_start_month;
  };

  # Check that the total number of months requested is divisible by $months
  my $run_total_months;
  if ($run_start_year == $run_end_year) {
    $run_total_months = $run_end_month - $run_start_month + 1;
  } else {
    $run_total_months = 13-$run_start_month + $run_end_month
                        + 12*($run_end_year - $run_start_year - 1);
  };
  if ($main::Verbose > 2) {
    print "months=$months   start_year=$run_start_year  start_mon=$run_start_month   ";
    print "end_year=$run_end_year  end_mon=$run_end_month\n";
    print "run_total_months = $run_total_months\n";
  }
  if (not $main::Override_total_months_check) {
    if ($run_total_months%$months != 0) {
      # reset run_end_year and/or run_end_month by adding months so that
      # $run_total_months becomes the next largest multiple of months
      my $madd = $months - $run_total_months%$months;
      if ($main::Verbose > 10) {print "months = $months   madd = $madd\n"};
      if (($run_end_month + $madd) > 12) {
        $run_end_year = $run_end_year + int(($run_end_month + $madd)/12);
        $run_end_month = ($run_end_month + $madd)%12;
        if ($run_end_month == 0) {
          $run_end_month = 12;
          $run_end_year = $run_end_year - 1;
        }
      } else {
        $run_end_month = $run_end_month + $madd;
      }
      # reset run_total_months
      if ($run_start_year == $run_end_year) {
        $run_total_months = $run_end_month - $run_start_month + 1;
      } else {
        $run_total_months = 13-$run_start_month + $run_end_month
                            + 12*($run_end_year - $run_start_year - 1);
      }
      warn "\n*** WARNING *** The last month was changed to year=$run_end_year, month=$run_end_month
                  so that it would fall on a multiple of months=$months\n";
      push @main::warnings,
        "*** WARNING *** The last month was changed to year=$run_end_year, month=$run_end_month
                  so that it would fall on a multiple of months=$months";
    }
  }

  if ($run_end_year < $run_start_year) {
    die "end_year=$run_end_year is less that start_year=$run_start_year, stopped";
  };
  if ($run_start_month < 1 or $run_start_month > 12) {
    die "start_mon=$run_start_month is out of range, stopped";
  };
  if ($run_end_month < 1 or $run_end_month > 12) {
    die "end_mon=$run_end_month is out of range, stopped";
  };
  if ($run_end_year == $run_start_year and
      $run_end_month < $run_start_month) {
    die "end_mon=$run_end_month is less that start_mon=$run_start_month.\n  Stopped";
  };
  if ($main::Verbose > $Vtrip) {
    print "months=$months  start_year=$run_start_year  ";
    print "end_year=$run_end_year  start_mon=$run_start_month  ";
    print "end_mon=$run_end_month\n";
    print "samerun=$samerun  initsp=$initsp\n";
  };

  # offset_year is included to fufill the same function as create_run's ayear
  # It is included for backward compatability
  if (exists $main::VARS{offset_year}) {
    $offset_year = $main::VARS{offset_year}{value}
  } else {
    $offset_year = 1;
  };

  # checks on run_start|end_dom
  if ($run_end_dom < 0) {$run_end_dom = $MONdays[$run_end_month]};
  if ($run_start_dom < 1) {
    warn "run_start_dom=$run_start_dom is out of range. Resetting to 1\n";
    push @main::warnings,
      "run_start_dom=$run_start_dom is out of range. Resetting to 1";
    $run_start_dom = 1;
  };
  if ($run_start_dom > $MONdays[$run_start_month]) {
    print "*** WARNING *** run_start_dom = $run_start_dom is greater than ";
    print "$MONdays[$run_start_month], the number of days in month $run_start_month\n";
    print "*** WARNING *** run_start_dom is reset to $MONdays[$run_start_month]\n";
    $run_start_dom = $MONdays[$run_start_month];
    push @main::warnings,
      "*** WARNING *** run_start_dom = $run_start_dom is greater than
      $MONdays[$run_start_month], the number of days in month $run_start_month
      *** WARNING *** run_start_dom is reset to $MONdays[$run_start_month]";
  };
  if ($run_end_dom < 1) {
    warn "run_end_dom=$run_end_dom is out of range. Resetting to 1\n";
    push @main::warnings,
      "run_end_dom=$run_end_dom is out of range. Resetting to 1";
    $run_end_dom = 1;
  };
  if ($run_end_dom > $MONdays[$run_end_month]) {
    print "*** WARNING *** run_end_dom = $run_end_dom is greater than ";
    print "$MONdays[$run_end_month], the number of days in month $run_end_month\n";
    print "*** WARNING *** run_end_dom is reset to $MONdays[$run_end_month]\n";
    $run_end_dom = $MONdays[$run_end_month];
    push @main::warnings,
      "*** WARNING *** run_end_dom = $run_end_dom is greater than
    $MONdays[$run_end_month], the number of days in month $run_end_month
    *** WARNING *** run_end_dom is reset to $MONdays[$run_end_month]";
  };

  # Determine the day of year for the start and end of the run
  my $run_start_doy = $eom_day[$run_start_month-1] + $run_start_dom;
  my $run_end_doy = $eom_day[$run_end_month-1] + $run_end_dom;
  if ($run_end_doy > $eom_day[$run_end_month]) {
    print "run_end_dom = $run_end_dom is greater than the number of days in ";
    print "month $run_end_month = $MONdays[$run_end_month]\n";
    die "   Stopped";
  };
  if ($main::Verbose > 10) {
    print "run_start_dom = $run_start_dom   run_end_dom = $run_end_dom\n";
    print "run_start_doy = $run_start_doy   run_end_doy = $run_end_doy\n";
  };
  if (($run_start_year == $run_end_year) and ($run_end_doy < $run_start_doy)) {
    print "run_end_doy = $run_end_doy is less than run_start_doy = $run_start_doy";
    die "  Stopped";
  };

  # Determine the kount value at the beginning and end of the run
  my $kount_at_start = ($run_start_year-$offset_year) * $steps_per_year
                    +   $eod_kount[$run_start_doy-1] + $offset_kount;
  my $kount_at_end = ($run_end_year-$offset_year) * $steps_per_year
                  +   $eod_kount[$run_end_doy] + $offset_kount;
  # Force kount_at_end to be kfinal if kfinal is set on the command line
  if ($kfinal_in_vlist) {$kount_at_end = 1*$kfinal_in_vlist};

  # Set run_total_... variables
  my $run_total_steps = $kount_at_end - $kount_at_start;
  # Note: run_total_years and run_total_days may or may not be integer
  my $run_total_years = $run_total_steps/$steps_per_year;
  my $run_total_days = $run_total_steps/$steps_per_day;
  if ($main::Verbose > 10) {
    print "run_total_years=$run_total_years  run_total_months=$run_total_months   ";
    print "run_total_days=$run_total_days   run_total_steps=$run_total_steps\n";
  };

  if ($main::Verbose > 5) {
    print "kount_at_start = $kount_at_start   kount_at_end = $kount_at_end  ";
    print "run_total_steps = $run_total_steps\n";
  };

  if ($kount_at_end < $kount_at_start) {
    print "*** ERROR *** kount_at_end=$kount_at_end is less than ";
    print "kount_at_start=$kount_at_start\n";
    die "   Stopped";
  };

  if ($main::Verbose > 0) {
    print "\nBuilding a job string from year $run_start_year";
    print ", month ",sprintf("%2d",$run_start_month),",";
    print " day ",sprintf("%2d",$run_start_dom)," (kount = $kount_at_start)\n";
    print "                        to year $run_end_year";
    print ", month ",sprintf("%2d",$run_end_month),",";
    print " day ",sprintf("%2d",$run_end_dom)," (kount = $kount_at_end)";
    print " in $months month chunks\n";
    if ($initsp eq "on") {print "Starting from initial conditions"}
    elsif ($initsp eq "off") {print "Continuing from a restart file"}
    else {if ($main::Verbose > 1) {print "initsp is neither on nor off"}};
    print "...the job will run for a total of ",sprintf("%g",$run_total_days)," days";
    print " or $run_total_steps time steps\n";
  };

  my @kount_list=();
  push @kount_list,$kount_at_start;
  my $job = '';
  $mon = 12;
  my $year_last_iter = $run_start_year;
  my $mon_last_iter = $run_start_month;
  my $kount_last_iter = $kount_at_start;
  my $force_last_iter=0;
  my $first_loop_iter=1;
  YEARLOOP: for ($year=$run_start_year; $year<=$run_end_year; $year++) {

    $year = sprintf("%3.3d",$year);

    if ($year == $run_start_year) {
      $start_mon = $run_start_month;
      $year_last_iter = $run_start_year;
      $mon_last_iter = $run_start_month;
    } else {$start_mon = $mon+$months-12};
    if ($year == $run_end_year) {
      $end_mon = $run_end_month;
    } else {$end_mon = 12};

    # Initialize the previous_(year|month) hashes
    if ($year == $run_start_year) {
      # The first time one of these sections is put into the job string the
      # previous year and month will be the start of the job string
      $previous_year = $year;
      $previous_month = $start_mon;
      $annual{previous_year}     = sprintf("%3.3d",$previous_year);
      $annual{previous_month}    = sprintf("%2.2d",$previous_month);
      $monthly{previous_year}    = sprintf("%3.3d",$previous_year);
      $monthly{previous_month}   = sprintf("%2.2d",$previous_month);
      $yearly{previous_year}     = sprintf("%3.3d",$previous_year);
      $yearly{previous_month}    = sprintf("%2.2d",$previous_month);
      $special{previous_year}    = sprintf("%3.3d",$previous_year);
      $special{previous_month}   = sprintf("%2.2d",$previous_month);
      foreach (@Nyearskey) {
        $prevyear{$_}  = sprintf("%3.3d",$previous_year);
        $prevmonth{$_} = sprintf("%2.2d",$previous_month);
      };
      foreach (@Nmonthskey) {
        $prevyear{$_}  = sprintf("%3.3d",$previous_year);
        $prevmonth{$_} = sprintf("%2.2d",$previous_month);
      };
    };

    # loop over months appending sections to $job
    MONTHLOOP: for ($mon=$start_mon; $mon<=$end_mon; $mon+=$months) {
      $mon = sprintf("%2.2d",$mon);

      if ($mon+$months_run-1<=12) {
	  $kfinal = $offset_kount + ($year-$offset_year) * $steps_per_year + $eom_kount[$mon+$months_run-1];
      } else {
	  $kfinal = $offset_kount + ($year+1-$offset_year) * $steps_per_year + $eom_kount[$mon+$months_run-13];
      }
      if ($kfinal > $kount_at_end) {
        $kfinal = 1*$kount_at_end;
        $force_last_iter = 1;
      };
      push @kount_list,$kfinal;
      $kfinal = sprintf("%10d",$kfinal);

      my $kount_this_month = $eom_kount[$mon]-$eom_kount[$mon-1];
      if ($main::Verbose > 5) {
        my $kount_diff = $kfinal-$kount_last_iter;
        print "kfinal=$kfinal   kfinal-kount_last_iter=",$kount_diff,
              "  kount_this_month=",$kount_this_month,
              ($kount_diff==$kount_this_month)? "  ...ok" : "  ...not ok","\n";
      };

      my $kount_start_iter = $kfinal-$kount_this_month;
      if ($first_loop_iter) {$kount_start_iter = $kount_at_start};
      $kount_start_iter = sprintf("%10d",$kount_start_iter);

      # year_last_iter and mon_last_iter are used after YEARLOOP finishes
      # kount_last_iter is used to define next_kount as well as after YEARLOOP
      $year_last_iter  = $year;
      $mon_last_iter   = $mon;
      $kount_last_iter = $kfinal;
      unless ($kfinal == $kount_at_end) {
        my $m = $mon + $months - 1;
        if ($m > 12) {
          $kount_last_iter += (int($m/12)-1) * $steps_per_year
                              + ($eom_kount[12] - $eom_kount[$mon])
                              + $eom_kount[$m%12];
        } else {
          $kount_last_iter += $eom_kount[$m] - $eom_kount[$mon];
        }
      }
      if ($main::Verbose > 5) {
        print "mon=$mon  year=$year  kount_last_iter=$kount_last_iter\n";
      }

      if ($mon == 1) {
        $mon_restart = sprintf("%2.2d",12);
        $year_restart = sprintf("%3.3d",$year-1);
      }
      else {
        $mon_restart = sprintf("%2.2d",$mon-1);
        $year_restart = sprintf("%3.3d",$year);
      }

      # first_job_in_run is set true when the current year/month are equal to
      # run_start_(year|month). If run_start_(year|month) are not set then
      # first_job_in_run is true when first_loop_iter is true.
      # first_job_in_run is used below to set samerun and initsp appropriately
      # on the first job in the run. Previously first_loop_iter was used
      # for this purpose but when a long run is continued by creating a new
      # job string for the next chunk of the run, we do not want to set samerun
      # and initsp on the first time through this loop since it will not be
      # the start of the run even though it is the first job in the string.
      my $first_job_in_run = 0;
      if ( exists $main::parin{run_start_year} ) {
        my $tmp_year = $main::parin{run_start_year};
        my $tmp_mon = 1;
        if ( exists $main::parin{run_start_month} ) {
          $tmp_mon = $main::parin{run_start_month};
        } else {
          warn "*** WARNING *** You have defined run_start_year but not run_start_month.
                Assuming run_start_month = 1";
        }
        if ( $year == $tmp_year and $mon == $tmp_mon ) {
          $first_job_in_run = 1;
        }
      } else {
        $first_job_in_run = $first_loop_iter;
      }

      # Set samerun and initsp appropriately for the first job in the run
      if ($initsp eq "on" and $first_job_in_run) {
        # initsp=on implies changes to 4 other variables:
        # samerun, year_restart, mon and mon_restart
        $samerun="off";
        $year_restart=sprintf("%3.3d",$year);
        $mon = "01";
        $mon_restart=sprintf("%2.2d",$mon);
      } else {
        unless ($first_job_in_run) {
          # On the first job in the run use the values defined above
          # Otherwise set initsp=off and samerun=on
          $initsp="off";
          $samerun="on";
        }
      }

      if ( exists $main::parin{samerun_override} ) {
        # Allow the user to override the above setting of samerun
        $samerun = $main::parin{samerun_override}
      }

      # Set cpl_new_rtd and nemo_new_rtd appropriately for the first job in the run
      unless ($first_job_in_run) {
        $cpl_new_rtd  = "off";
        $nemo_new_rtd = "off";
      }
      if ( exists $main::parin{cpl_new_rtd_override} ) {
        # Allow the user to override the above setting of cpl_new_rtd
        $cpl_new_rtd = $main::parin{cpl_new_rtd_override}
      }
      if ( exists $main::parin{nemo_new_rtd_override} ) {
        # Allow the user to override the above setting of nemo_new_rtd
        $nemo_new_rtd = $main::parin{nemo_new_rtd_override}
      }

      # define next_year and next_month as the year and month that is $months-1 months
      # ahead of the current year and mon. This will be the last month of a model run
      # that runs for $months months from the current year and month.
      $next_month = $mon + $months -  1;
      $next_year  = $year + int(($next_month-1)/12);
      if ($next_month > 12) {$next_month = 1 + ($next_month-1)%12};
      $next_year  = sprintf("%3.3d",$next_year);
      $next_month = sprintf("%2.2d",$next_month);
      $next_kount = sprintf("%10d",$kount_last_iter);

      # skip the remainder of the loop when $kount_start_iter is negative
      # if ($kount_start_iter < 0) {
      #   $first_loop_iter = 0;
      #   next;
      # };

      my %line_out = ();
      if ( 1 ) {
        $line_out{pfx} = "kount=$next_kount  year=$next_year  month=$next_month  ";
        $line_out{pfx} .= "day=$MONdays[$next_month]  ...";
      } else {
        $line_out{pfx} = "FROM  kount=" . sprintf("%10d",$kount_start_iter);
        $line_out{pfx} .= "  year=$year  mon=$mon  TO  ";
        $line_out{pfx} .= "kount=$next_kount  year=$next_year  month=$next_month  ...";
      }
      $line_out{wrote} = 0;

      if ($main::Verbose > 0) {
        if ($main::Verbose > 2) {
          print "+++ kfinal=$kfinal  initsp=$initsp   samerun=$samerun  ";
          print "year_restart=$year_restart  mon_restart=$mon_restart\n";
        }
        # print "$line_out{pfx}";
        # print "FROM  kount=",sprintf("%10d",$kount_start_iter);
        # print "  year=$year  mon=$mon  TO  ";
        # print "kount=$next_kount  year=$next_year  month=$next_month  ...";
      }

      # Assign RCM variables

      # RCM parameters that change from month to month are:
      # M1=001_m03_              ...current year/month
      # M2=001_m04_              ...next year/month
      # RUNP=op2r_001_m02_       ...name of previous run
      # RUN=op2r_$M1             ...name of current run
      # MRCKSTART='      5664'   ...first time step of the RCM run
      # MRCKTOTAL='      8640'   ...last time step of the RCM run
      # MRCRADINIT='      5664'  ...first radiation time step of the RCM run
      # MRCRADFINI='      8640'  ...last radiation time step of the RCM run
      # MRCRADIDAY='   60'       ...Julian day corresponding to MRCRADINIT
      # MRCRADGMT=' 0.00'        ...GMT hour of the day corresponding to MRCRADINIT
      # GCMKINIT='      4248'    ...first time step from the CRCM data
      # GCMKFINI='      6480'    ...last time step from the CRCM data

      my $rcm_next_month = $mon + $months;
      my $rcm_next_year  = $year + int(($rcm_next_month-1)/12);
      if ($rcm_next_month > 12) {$rcm_next_month = 1 + ($rcm_next_month-1)%12};
      $rcm_next_year  = sprintf("%3.3d",$rcm_next_year);
      $rcm_next_month = sprintf("%2.2d",$rcm_next_month);
      my $rcm_M1         = "${year}_m${mon}_";
      my $rcm_M2         = "${rcm_next_year}_m${rcm_next_month}_";
      my $rcm_RUNP       =
      "$main::VARS{runid}{value}_$monthly{previous_year}_m$monthly{previous_month}_";
      my $rcm_RUN        = "$main::VARS{runid}{value}_$rcm_M1";
      my $rcm_MRCKTOTAL  = ($year-1) * $rcm_steps_per_year + $rcm_eom_kount[$mon];
      my $rcm_kount_this_month = $rcm_eom_kount[$mon] - $rcm_eom_kount[$mon-1];
      my $rcm_MRCKSTART  = $rcm_MRCKTOTAL - $rcm_kount_this_month;
      if ($rcm_MRCKSTART == 0) {$rcm_RUNP = 0};
      $rcm_MRCKSTART = sprintf("%10d",$rcm_MRCKSTART);
      $rcm_MRCKTOTAL = sprintf("%10d",$rcm_MRCKTOTAL);
      my $rcm_MRCRADINIT = "$rcm_MRCKSTART";
      if ($rcm_MRCKSTART == 0) {$rcm_MRCRADINIT = $rcm_MRCARC};
      my $rcm_MRCRADFINI = "$rcm_MRCKTOTAL";
      $rcm_MRCRADINIT = sprintf("%10d",$rcm_MRCRADINIT);
      $rcm_MRCRADFINI = sprintf("%10d",$rcm_MRCRADFINI);
      my $rcm_MRCRADIDAY = 1 + int($rcm_MRCRADINIT/$rcm_steps_per_day)%365;
      $rcm_MRCRADIDAY = sprintf("%5d",$rcm_MRCRADIDAY);
      my $rcm_MRCRADGMT  = int($rcm_MRCRADINIT/$rcm_steps_per_hour)%24;
      $rcm_MRCRADGMT = sprintf("%5.2f",$rcm_MRCRADGMT);
      my $rcm_GCMKINIT   = "$kount_start_iter";
      my $rcm_GCMKFINI   = "$kfinal";

      if ($main::MK_RCM_DATA and $main::Verbose > 1) {
        print "\nrcm_M1=$rcm_M1\n";
        print "rcm_M2=$rcm_M2\n";
        print "rcm_RUNP=$rcm_RUNP\n";
        print "rcm_RUN =$rcm_RUN\n";
        print "rcm_MRCKSTART=$rcm_MRCKSTART\n";
        print "rcm_MRCKTOTAL=$rcm_MRCKTOTAL\n";
        print "rcm_MRCRADINIT=$rcm_MRCRADINIT\n";
        print "rcm_MRCRADFINI=$rcm_MRCRADFINI\n";
        print "rcm_MRCRADIDAY=$rcm_MRCRADIDAY\n";
        print "rcm_MRCRADGMT=$rcm_MRCRADGMT\n";
        print "rcm_GCMKINIT=$rcm_GCMKINIT\n";
        print "rcm_GCMKFINI=$rcm_GCMKFINI\n";
      };

      #========== multi year section --- insert at start of every Nth year ==========
      if ( $mon == 1 ) {
        foreach my $Nkey (@Nyearskey) {
          next if ($kfinal >= $kount_at_end);  # do not add on last loop iteration
          my ($N,$curr_iopt) = $Nkey =~ /every(-?\d+)years(.*)/;
          next unless $N < 0;
          # print "\nNkey=$Nkey   N=$N\n";
          # insert at the start of every N years for negative values of N
          my $absN = abs($N);

          my $curr_iopt_href = str2ref_iopts( $curr_iopt, {ID=>"${N}y"} );
          $curr_iopt_href = {} unless $curr_iopt_href;

          my $my_start_year = $run_start_year;
          my $my_start_mon  = $run_start_month;
          my $my_stop_year  = $run_end_year;
          my $my_stop_mon   = $run_end_month;
          $my_start_year = $curr_iopt_href->{start_year} if $curr_iopt_href->{start_year};
          $my_start_mon  = $curr_iopt_href->{start_mon}  if $curr_iopt_href->{start_mon};
          $my_stop_year  = $curr_iopt_href->{stop_year}  if $curr_iopt_href->{stop_year};
          $my_stop_mon   = $curr_iopt_href->{stop_mon}   if $curr_iopt_href->{stop_mon};

          my $my_mon_offset   = 0;
          $my_mon_offset = $curr_iopt_href->{mon_offset} if $curr_iopt_href->{mon_offset};

          my $addnow = $next_year - $my_start_year;
          next unless ($addnow%$absN == 0);

          # copy the multi year section template to a temporary string
          ($str) = @{$TEMPLATE{$Nkey}};
          if ($str) {
            # split the current string into individual jobs
            # delimited by #end_of_job at the bottom
            my @splitjob = split_job_string($str);

            # perform section specific substitutions
            $str = '';
            foreach my $thisjob (@splitjob) {
              # Ignore empty job strings
              next if ($thisjob =~ /^\s*$/s);

              # get the name of the current job in jobname and
              # remove the job tag from the current job string
              my ($jobname) = read_jobtag($thisjob,{NAME_TAG=>1});

              # Extract info about splitting this job into multiple jobs
              my ($split_thisjob) = read_jobtag($thisjob,{SPLIT_TAG=>1});

              # Get a hash ref to internal job specific options
              my $iopt_href = read_jobtag($thisjob,{IOPT_TAG=>1});

              # determine a year/month range, outside of which this job
              # should not be included in the final job string
              my $first_year_included = $run_start_year;
              my $first_mon_included  = $run_start_month;
              my $last_year_included  = $run_end_year;
              my $last_mon_included   = $run_end_month;
              $first_year_included = $iopt_href->{start_year} if $iopt_href->{start_year};
              $first_mon_included  = $iopt_href->{start_mon}  if $iopt_href->{start_mon};
              $last_year_included  = $iopt_href->{stop_year}  if $iopt_href->{stop_year};
              $last_mon_included   = $iopt_href->{stop_mon}   if $iopt_href->{stop_mon};

              # current_(year|month) and next_(year|month) are used by certain
              # modules (e.g. mdump, mdelete, mload). These modules generate
              # file names based on these values.
              # Note that previous_(year|month) is not set here so that mload etc. can
              # determine the correct range of months to generate file names for

              # the value of mon_offset will depend on the type of job
              my $mon_offset = 0;
              if ($jobname =~ /^\s*mload\s*$/) {
                $mon_offset = $mload_mon_offset;
              } elsif ($jobname =~ /^\s*mdump\s*$/) {
                $mon_offset = $mdump_mon_offset;
              } elsif ($jobname =~ /^\s*mdelete\s*$/) {
                $mon_offset = $mdelete_mon_offset;
              } elsif ($jobname =~ /^\s*load_list\s*$/) {
                $mon_offset = $load_list_mon_offset;
              } elsif ($jobname =~ /^\s*dump_list\s*$/) {
                $mon_offset = $dump_list_mon_offset;
              } elsif ($jobname =~ /^\s*del_list\s*$/) {
                $mon_offset = $del_list_mon_offset;
              } elsif ($jobname =~ /^\s*custom\d*\s*$/) {
                $mon_offset = $custom_mon_offset;
              } elsif ($jobname =~ /^\s*cloop\s*$/) {
                $mon_offset = $cloop_mon_offset;
              };

              my ($curr_mon, $curr_year) = new_mon_year($mon, $year, $mon_offset);
              my ($monpN, $yearpN) = new_mon_year(12, $year+$absN-1, $mon_offset);

              # Do not include this job if it would appear before
              # the start of the include range determined above
              next if ($curr_year < $first_year_included);
              if ($curr_year == $first_year_included) {
                next if ($curr_mon < $first_mon_included);
              }

              # Do not include this job if it would extend past
              # the end of the include range determined above
              next if ($yearpN > $last_year_included);
              if ($yearpN == $last_year_included) {
                next if ($monpN > $last_mon_included);
              }

              # Count the number of times this job gets used at this frequency
              my $job_id = "${jobname}_${N}y";
              if ( exists $JOB_COUNT{$job_id} ) {
                # Increment the count for this job at this frequency
                $JOB_COUNT{$job_id} += 1;
              } else {
                # Initialize the count
                $JOB_COUNT{$job_id} = 1;
              }
              my $curr_job_count = $JOB_COUNT{$job_id};

              # Add parallel job delimiters to specified jobs
              my $OPTref = {CURR_MON        => $monpN,
                            CURR_YEAR       => $yearpN,
                            PREV_MON        => $curr_mon,
                            PREV_YEAR       => $curr_year,
                            RUN_START_YEAR  => $run_start_year,
                            RUN_START_MONTH => $run_start_month,
                            RUN_END_YEAR    => $run_end_year,
                            RUN_END_MONTH   => $run_end_month};
              my $pre_print = '';
              my $post_print = '';
              ($thisjob, $pre_print, $post_print) =
                 insert_ext_delim($thisjob,$jobname,$OPTref);

              if ($main::Verbose == 1) {
                my $jinfo = " ${jobname}:${N}y";
                if ($split_thisjob) {$jinfo = " ${jobname}:${N}y:${split_thisjob}m"}
                my $strng = $pre_print . $jinfo . $post_print;
                if ( $line_out{pfx} ) {
                  $strng = $line_out{pfx} . $pre_print . $jinfo . $post_print;
                  $line_out{pfx} = "";
                }
                $line_out{wrote} = 1;
                print "$strng";
              }

              # replace variable values in $thisjob
              # next_year and next_month are used since this should be inserted at
              # the end of $months months after current $year,$mon counter values
              ($thisjob) = shell_var_subs::var_replace($thisjob,"year",$year);
              ($thisjob) = shell_var_subs::var_replace($thisjob,"mon",$mon);

#              if ($curr_job_count == 1 and $jobname eq "xnemo") {
              if ( $curr_job_count == 1 ) {
                if (exists $main::VARS{nemo_restart}) {
                  # If a nemo restart file name is available then
                  # set it only on the first job of the string
                  my $nemo_restart = $main::VARS{nemo_restart}{value};
                  # Remove nemo_restart from the VARS hash so that it does not
                  # get set in any other jobs in this string
                  delete $main::VARS{nemo_restart};
                  if ( $nemo_restart ) {
                    # Redefine nemo_restart only when it has a non null value
                    # strip any three digit numerical extension
                    $nemo_restart  =~ s/(.*)(\.[0-9]{3})?$/$1/;
                    ($thisjob) = shell_var_subs::var_replace($thisjob,"nemo_restart",$nemo_restart);
                  }
                }
                if (exists $main::VARS{nemo_exec}) {
                  # If a nemo_exec is set by the user then
                  # set it only on the first job of the string
                  my $nemo_exec = $main::VARS{nemo_exec}{value};
                  # Remove nemo_exec from the VARS hash so that it does not
                  # get set in any other jobs in this string
                  delete $main::VARS{nemo_exec};
                  if ( $nemo_exec ) {
                    # Redefine nemo_exec only when it has a non null value
                    ($thisjob) = shell_var_subs::var_replace($thisjob,"nemo_exec",$nemo_exec);
                  }
                }
                if (exists $main::VARS{nemo_nn_it000}) {
                  # If nemo_nn_it000 is set by the user then
                  # set it only on the first job of the string
                  my $nemo_nn_it000 = $main::VARS{nemo_nn_it000}{value};
                  # Remove nemo_nn_it000 from the VARS hash so that it does not
                  # get set in any other jobs in this string
                  delete $main::VARS{nemo_nn_it000};
                  if ( $nemo_nn_it000 ) {
                    # Redefine nemo_nn_it000 only when it has a non null value
                    ($thisjob) = shell_var_subs::var_replace($thisjob,"nemo_nn_it000",$nemo_nn_it000);
                  }
                }
                if (exists $main::VARS{nemo_from_rest}) {
                  # If nemo_from_rest is set by the user then
                  # set it only on the first job of the string
                  my $nemo_from_rest = $main::VARS{nemo_from_rest}{value};
                  # Remove nemo_from_rest from the VARS hash so that it does not
                  # get set in any other jobs in this string
                  delete $main::VARS{nemo_from_rest};
                  # Redefine nemo_from_rest in the current job only
                  ($thisjob) = shell_var_subs::var_replace($thisjob,"nemo_from_rest",$nemo_from_rest);
                }
                if (exists $main::VARS{nemo_force_namelist}) {
                  # If nemo_force_namelist is set by the user then
                  # set it only on the first job of the string
                  my $nemo_force_namelist = $main::VARS{nemo_force_namelist}{value};
                  # Remove nemo_force_namelist from the VARS hash so that it does not
                  # get set in any other jobs in this string
                  delete $main::VARS{nemo_force_namelist};
                  # Redefine nemo_force_namelist in the current job only
                  ($thisjob) = shell_var_subs::var_replace($thisjob,"nemo_force_namelist",$nemo_force_namelist);
                }
	      }

              # define current values for parameters to be substituted

              ($thisjob) = shell_var_subs::var_replace($thisjob,"current_year",
                               sprintf("%3.3d",$curr_year));
              ($thisjob) = shell_var_subs::var_replace($thisjob,"current_month",
                               sprintf("%2.2d",$curr_mon));
              ($thisjob) = shell_var_subs::var_replace($thisjob,"next_year",
                               sprintf("%3.3d",$yearpN));
              ($thisjob) = shell_var_subs::var_replace($thisjob,"next_month",
                               sprintf("%2.2d",$monpN));

              if ($split_thisjob) {
                # Insert multiple jobs with differences in only certain parameters
                # These are currently:
                # current_year, current_month, next_year, next_month
                my $mdelt = int($split_thisjob);
                my $year_a = $curr_year;
                my $mon_a  = $curr_mon;
                my ($mon_b, $year_b) = new_mon_year($mon_a, $year_a, $mdelt-1);
                my $delt_mon = diff_in_months($year_b,$mon_b,$yearpN,$monpN);
                if ($delt_mon < 0) {
                  print "\nAttempting to split $jobname into $mdelt month intervals.\n";
                  my $rmf = 1 + diff_in_months($curr_year,$curr_mon,$yearpN,$monpN);
                  print "This is greater than the current interval of $rmf months.\n";
                  die "Stopped";
                }
                while ( $delt_mon >= 0 ) {
                  if ($main::Verbose > 3) {
                    print "\n$jobname   ";
                    print "year_a=$year_a mon_a=$mon_a ... year_b=$year_b mon_b=$mon_b\n";
                  }
                  ($thisjob) = shell_var_subs::var_replace($thisjob,"current_year",
                               sprintf("%3.3d",$year_a));
                  ($thisjob) = shell_var_subs::var_replace($thisjob,"current_month",
                               sprintf("%2.2d",$mon_a));
                  ($thisjob) = shell_var_subs::var_replace($thisjob,"next_year",
                               sprintf("%3.3d",$year_b));
                  ($thisjob) = shell_var_subs::var_replace($thisjob,"next_month",
                               sprintf("%2.2d",$mon_b));
                  $str .= $thisjob;
                  # delt_mon=0 means mon_b,year_b is equal to monpN,yearpN
                  last if $delt_mon == 0;
                  ($mon_a, $year_a) = new_mon_year($mon_b, $year_b, 1);
                  ($mon_b, $year_b) = new_mon_year($mon_a, $year_a, $mdelt-1);
                  $delt_mon = diff_in_months($year_b,$mon_b,$yearpN,$monpN);
                  if ($delt_mon < 0) {
                    # mon_b, year_b is later than monpN, yearpN
                    ($mon_b, $year_b) = ($monpN, $yearpN);
                    $delt_mon = 0;
                  }
                }
              } else {
                $str .= $thisjob;
              }
	    };

            # set previous_(year/month) to the month after the the last time
            # the section was inserted
            $prevmonth{$Nkey} = sprintf("%2.2d",$mon + 1);
            if ($prevmonth{$Nkey} > 12) {
              $prevmonth{$Nkey} = sprintf("%2.2d",1);
              $prevyear{$Nkey} = sprintf("%3.3d",$year + 1);
            } else {
              $prevyear{$Nkey} = $year;
            }

            # Append to the job string
            $job .= $str;
          };
        };
      };

      #========== multi monthly section --- insert at the start of every N months ==========
      foreach my $Nkey (@Nmonthskey) {
        my ($N,$curr_iopt) = $Nkey =~ /every(-?\d+)months(.*)/;
        #insert at the start of every N months for negative values of N
        next unless $N < 0;
        next if ($kfinal > $kount_at_end);  # do not add on last loop iteration
        my $absN = abs($N);

        my $curr_iopt_href = str2ref_iopts( $curr_iopt, {ID=>"${N}m"} );
        $curr_iopt_href = {} unless $curr_iopt_href;

        my $mm_start_year = $run_start_year;
        my $mm_start_mon  = $run_start_month;
        my $mm_stop_year  = $run_end_year;
        my $mm_stop_mon   = $run_end_month;
        $mm_start_year = $curr_iopt_href->{start_year} if $curr_iopt_href->{start_year};
        $mm_start_mon  = $curr_iopt_href->{start_mon}  if $curr_iopt_href->{start_mon};
        $mm_stop_year  = $curr_iopt_href->{stop_year}  if $curr_iopt_href->{stop_year};
        $mm_stop_mon   = $curr_iopt_href->{stop_mon}   if $curr_iopt_href->{stop_mon};

        my $mm_mon_offset   = 0;
        $mm_mon_offset = $curr_iopt_href->{mon_offset} if $curr_iopt_href->{mon_offset};
        $mm_mon_offset = 0;

        # determine the total number of months since the start of the run
        my $addnow = ($year-$mm_start_year)*12 + $mon - $mm_start_mon;
        next if $addnow < 0;
        next unless ($addnow%$absN == 0);

        # copy the monthly section template to a temporary string
        ($str) = @{$TEMPLATE{$Nkey}};
        if ($str) {
          # split the current string into individual jobs
          # delimited by #end_of_job at the bottom
          my @splitjob = split_job_string($str);

          # define current values for parameters to be substituted
          my $obsday = $MONname[$mon-1];
          my $monn = $obsday;
          $monn =~ tr/A-Z/a-z/;   # lower case month name

          # perform section specific substitutions
          $str = '';
          foreach my $thisjob (@splitjob) {
            # Ignore empty job strings
            next if ($thisjob =~ /^\s*$/s);

            # get the name of the current job in jobname and
            # remove the job tag from the current job string
            my ($jobname) = read_jobtag($thisjob,{NAME_TAG=>1});

            # Extract info about splitting this job into multiple jobs
            my ($split_thisjob) = read_jobtag($thisjob,{SPLIT_TAG=>1});

	    # Get a hash ref to internal job specific options
            my $iopt_href = read_jobtag($thisjob,{IOPT_TAG=>1});

            # determine a year/month range, outside of which this job
            # should not be included in the final job string
            my $first_year_included = $run_start_year;
            my $first_mon_included  = $run_start_month;
            my $last_year_included  = $run_end_year;
            my $last_mon_included   = $run_end_month;
            $first_year_included = $iopt_href->{start_year} if $iopt_href->{start_year};
            $first_mon_included  = $iopt_href->{start_mon}  if $iopt_href->{start_mon};
            $last_year_included  = $iopt_href->{stop_year}  if $iopt_href->{stop_year};
            $last_mon_included   = $iopt_href->{stop_mon}   if $iopt_href->{stop_mon};

            if ($absN%$months != 0 ) {
              warn "\n*** WARNING *** ";
              warn "  The \"every $N months\" increment found in ${jobname}:${N}m is not a multiple of months=$months.\n\n";
              # warn "Resetting to every -$months months\n";
              # $N = -$months;
              # $absN = $months;
            }

            # current_(year|month) and next_(year|month) are used by certain
            # modules (e.g. mdump, mdelete, mload). These modules generate
            # file names based on these values.
            # Note that previous_(year|month) is not set here so that mload etc. can
            # determine the correct range of months to generate file names for

            # the value of mon_offset will depend on the type of job
            my $mon_offset = 0;
            if ($jobname =~ /^\s*mload\s*$/) {
              $mon_offset = $mload_mon_offset;
            } elsif ($jobname =~ /^\s*mdump\s*$/) {
              $mon_offset = $mdump_mon_offset;
            } elsif ($jobname =~ /^\s*mdelete\s*$/) {
              $mon_offset = $mdelete_mon_offset;
            } elsif ($jobname =~ /^\s*load_list\s*$/) {
              $mon_offset = $load_list_mon_offset;
            } elsif ($jobname =~ /^\s*dump_list\s*$/) {
              $mon_offset = $dump_list_mon_offset;
            } elsif ($jobname =~ /^\s*del_list\s*$/) {
              $mon_offset = $del_list_mon_offset;
            } elsif ($jobname =~ /^\s*custom\d*\s*$/) {
              $mon_offset = $custom_mon_offset;
            } elsif ($jobname =~ /^\s*cloop\s*$/) {
              $mon_offset = $cloop_mon_offset;
            }

            my ($curr_mon, $curr_year) = new_mon_year($mon, $year, $mon_offset);
            my ($monpN, $yearpN) = new_mon_year($mon, $year, $absN - 1 + $mon_offset);

            # Do not include this job if it would appear before
            # the start of the include range determined above
            next if ($curr_year < $first_year_included);
            if ($curr_year == $first_year_included) {
              next if ($curr_mon < $first_mon_included);
            }

            # Do not include this job if it would extend past
            # the end of the include range determined above
            next if ($yearpN > $last_year_included);
            if ($yearpN == $last_year_included) {
              next if ($monpN > $last_mon_included);
            }

            # Count the number of times this job gets used at this frequency
            my $job_id = "${jobname}_${N}m";
            if ( exists $JOB_COUNT{$job_id} ) {
              # Increment the count for this job at this frequency
              $JOB_COUNT{$job_id} += 1;
            } else {
              # Initialize the count
              $JOB_COUNT{$job_id} = 1;
            }
            my $curr_job_count = $JOB_COUNT{$job_id};

            # Add parallel job delimiters to specified jobs
            my $OPTref = {CURR_MON        => $monpN,
                          CURR_YEAR       => $yearpN,
                          PREV_MON        => $curr_mon,
                          PREV_YEAR       => $curr_year,
                          RUN_START_YEAR  => $run_start_year,
                          RUN_START_MONTH => $run_start_month,
                          RUN_END_YEAR    => $run_end_year,
                          RUN_END_MONTH   => $run_end_month};
            my $pre_print = '';
            my $post_print = '';
            ($thisjob, $pre_print, $post_print) =
               insert_ext_delim($thisjob,$jobname,$OPTref);

            if ($main::Verbose == 1) {
              my $jinfo = " ${jobname}:${N}m";
              if ($split_thisjob) {$jinfo = " ${jobname}:${N}m:${split_thisjob}m"}
              my $strng = $pre_print . $jinfo . $post_print;
              if ( $line_out{pfx} ) {
                $strng = $line_out{pfx} . $pre_print . $jinfo . $post_print;
                $line_out{pfx} = "";
              }
              $line_out{wrote} = 1;
              print "$strng";
            }

            ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,"monn",   $monn);
            ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,"year",   $year);
            ($thisjob,$rv2) = shell_var_subs::var_replace($thisjob,"mon",    $mon);
            ($thisjob,$rv3) = shell_var_subs::var_replace($thisjob,"days",   $obsday);
            ($thisjob,$rv4) = shell_var_subs::var_replace($thisjob,"obsday", $obsday);
            ($thisjob,$rv5) = shell_var_subs::var_replace($thisjob,"kfinal", $kfinal);
            ($thisjob,$rv6) = shell_var_subs::var_replace($thisjob,"year_restart",
                                  $year_restart);
            ($thisjob,$rv7) = shell_var_subs::var_replace($thisjob,"mon_restart",
                                  $mon_restart);
            ($thisjob,$rv8) = shell_var_subs::var_replace($thisjob,"samerun",$samerun);
            ($thisjob,$rv9) = shell_var_subs::var_replace($thisjob,"initsp", $initsp);
            ($thisjob,$rv10)= shell_var_subs::var_replace($thisjob,"cpl_new_rtd",  $cpl_new_rtd);
            ($thisjob,$rv11)= shell_var_subs::var_replace($thisjob,"nemo_new_rtd", $nemo_new_rtd);
            if ($first_loop_iter and $main::Restart_fname) {
              # If a restart file name is available then set start in the string
              my $start = $main::Restart_fname;
              # strip any three digit numerical extension as well as a trailing "rs"
              # A gcm model job will use ${start}rs as the restart file name
              $start  =~ s/(.*)[cor]s(\.[0-9]{3})?$/$1/;
              ($thisjob) = shell_var_subs::var_replace($thisjob,"start",$start);
            }

            if ($first_loop_iter and exists $main::VARS{nemo_restart}) {
              # If a nemo restart file name is available then
              # set it only on the first job of the string
              my $nemo_restart = $main::VARS{nemo_restart}{value};
              # Remove nemo_restart from the VARS hash so that it does not
              # get set in any other jobs in this string
              delete $main::VARS{nemo_restart};
              if ( $nemo_restart ) {
                # Redefine nemo_restart only when it has a non null value
                # strip any three digit numerical extension
                $nemo_restart  =~ s/(.*)(\.[0-9]{3})?$/$1/;
                ($thisjob) = shell_var_subs::var_replace($thisjob,"nemo_restart",$nemo_restart);
              }
            }
            if ($first_loop_iter and exists $main::VARS{nemo_exec}) {
              # If a nemo_exec is set by the user then
              # set it only on the first job of the string
              my $nemo_exec = $main::VARS{nemo_exec}{value};
              # Remove nemo_exec from the VARS hash so that it does not
              # get set in any other jobs in this string
              delete $main::VARS{nemo_exec};
              if ( $nemo_exec ) {
                # Redefine nemo_exec only when it has a non null value
                ($thisjob) = shell_var_subs::var_replace($thisjob,"nemo_exec",$nemo_exec);
              }
            }
            if (exists $main::VARS{nemo_nn_it000}) {
              # If nemo_nn_it000 is set by the user then
              # set it only on the first job of the string
              my $nemo_nn_it000 = $main::VARS{nemo_nn_it000}{value};
              # Remove nemo_nn_it000 from the VARS hash so that it does not
              # get set in any other jobs in this string
              delete $main::VARS{nemo_nn_it000};
              if ( $nemo_nn_it000 ) {
                # Redefine nemo_nn_it000 only when it has a non null value
                ($thisjob) = shell_var_subs::var_replace($thisjob,"nemo_nn_it000",$nemo_nn_it000);
              }
            }
            if ($first_loop_iter and exists $main::VARS{nemo_from_rest}) {
              # If nemo_from_rest is set by the user then
              # set it only on the first job of the string
              my $nemo_from_rest = $main::VARS{nemo_from_rest}{value};
              # Remove nemo_from_rest from the VARS hash so that it does not
              # get set in any other jobs in this string
              delete $main::VARS{nemo_from_rest};
              # Redefine nemo_from_rest in the current job only
              ($thisjob) = shell_var_subs::var_replace($thisjob,"nemo_from_rest",$nemo_from_rest);
            }
            if ($first_loop_iter and exists $main::VARS{nemo_force_namelist}) {
              # If nemo_force_namelist is set by the user then
              # set it only on the first job of the string
              my $nemo_force_namelist = $main::VARS{nemo_force_namelist}{value};
              # Remove nemo_force_namelist from the VARS hash so that it does not
              # get set in any other jobs in this string
              delete $main::VARS{nemo_force_namelist};
              # Redefine nemo_force_namelist in the current job only
              ($thisjob) = shell_var_subs::var_replace($thisjob,"nemo_force_namelist",$nemo_force_namelist);
            }

            ($thisjob) = shell_var_subs::var_replace($thisjob,"current_year",
                              sprintf("%3.3d",$curr_year));
            ($thisjob) = shell_var_subs::var_replace($thisjob,"current_month",
                              sprintf("%2.2d",$curr_mon));
            ($thisjob) = shell_var_subs::var_replace($thisjob,"next_year",
                              sprintf("%3.3d",$yearpN));
            ($thisjob) = shell_var_subs::var_replace($thisjob,"next_month",
                              sprintf("%2.2d",$monpN));

            if ($split_thisjob) {
              # Insert multiple jobs with differences in only certain parameters
              # These are currently:
              # current_year, current_month, next_year, next_month
              my $mdelt = int($split_thisjob);
              my $year_a = $curr_year;
              my $mon_a  = $curr_mon;
              my ($mon_b, $year_b) = new_mon_year($mon_a, $year_a, $mdelt-1);
              my $delt_mon = diff_in_months($year_b,$mon_b,$yearpN,$monpN);
              if ($delt_mon < 0) {
                print "\nAttempting to split $jobname into $mdelt month intervals.\n";
                my $rmf = 1 + diff_in_months($curr_year,$curr_mon,$yearpN,$monpN);
                print "This is greater than the current interval of $rmf months.\n";
                die "Stopped";
              }
              while ( $delt_mon >= 0 ) {
                if ($main::Verbose > 3) {
                  print "\n$jobname   ";
                  print "year_a=$year_a mon_a=$mon_a ... year_b=$year_b mon_b=$mon_b\n";
                }
                ($thisjob) = shell_var_subs::var_replace($thisjob,"current_year",
                             sprintf("%3.3d",$year_a));
                ($thisjob) = shell_var_subs::var_replace($thisjob,"current_month",
                             sprintf("%2.2d",$mon_a));
                ($thisjob) = shell_var_subs::var_replace($thisjob,"next_year",
                             sprintf("%3.3d",$year_b));
                ($thisjob) = shell_var_subs::var_replace($thisjob,"next_month",
                             sprintf("%2.2d",$mon_b));
                $str .= $thisjob;
                # delt_mon=0 means mon_b,year_b is equal to monpN,yearpN
                last if $delt_mon == 0;
                ($mon_a, $year_a) = new_mon_year($mon_b, $year_b, 1);
                ($mon_b, $year_b) = new_mon_year($mon_a, $year_a, $mdelt-1);
                $delt_mon = diff_in_months($year_b,$mon_b,$yearpN,$monpN);
                if ($delt_mon < 0) {
                  # mon_b, year_b is later than monpN, yearpN
                  ($mon_b, $year_b) = ($monpN, $yearpN);
                  $delt_mon = 0;
                }
              }
            } else {
              $str .= $thisjob;
            }
          };
          # set previous_(year/month) to the last time the section was inserted
          $prevmonth{$Nkey} = sprintf("%2.2d",$mon + 1);
          if ($prevmonth{$Nkey} > 12) {
            $prevmonth{$Nkey} = sprintf("%2.2d",1);
            $prevyear{$Nkey} = sprintf("%3.3d",$year + 1);
          } else {
            $prevyear{$Nkey} = $year;
          };

          # Append to the job string
          $job .= $str;
        };
      };

      #========== annual section --- insert at start of year ==========
      if ( $mon == 1 ) {
        # copy the annual section template to a temporary string
        ($str) = @{$TEMPLATE{annual}};
        if ($str) {
          # split the current string into individual jobs
          # delimited by #end_of_job at the bottom
          my @splitjob = split_job_string($str);

          # perform section specific substitutions
          $str = '';
          foreach my $thisjob (@splitjob) {
            # Ignore empty job strings
            next if ($thisjob =~ /^\s*$/s);

            # get the name of the current job in jobname and
            # remove the job tag from the current job string
            my ($jobname) = read_jobtag($thisjob,{NAME_TAG=>1});

            # Extract info about splitting this job into multiple jobs
            my ($split_thisjob) = read_jobtag($thisjob,{SPLIT_TAG=>1});

	    # Get a hash ref to internal job specific options
            my $iopt_href = read_jobtag($thisjob,{IOPT_TAG=>1});

            # determine a year/month range, outside of which this job
            # should not be included in the final job string
            my $first_year_included = $run_start_year;
            my $first_mon_included  = $run_start_month;
            my $last_year_included  = $run_end_year;
            my $last_mon_included   = $run_end_month;
            $first_year_included = $iopt_href->{start_year} if $iopt_href->{start_year};
            $first_mon_included  = $iopt_href->{start_mon}  if $iopt_href->{start_mon};
            $last_year_included  = $iopt_href->{stop_year}  if $iopt_href->{stop_year};
            $last_mon_included   = $iopt_href->{stop_mon}   if $iopt_href->{stop_mon};

            # the value of mon_offset will depend on the type of job
            my $mon_offset = 0;
            if ($jobname =~ /^\s*mload\s*$/) {
              $mon_offset = $mload_mon_offset;
            } elsif ($jobname =~ /^\s*mdump\s*$/) {
              $mon_offset = $mdump_mon_offset;
            } elsif ($jobname =~ /^\s*mdelete\s*$/) {
              $mon_offset = $mdelete_mon_offset;
            } elsif ($jobname =~ /^\s*load_list\s*$/) {
              $mon_offset = $load_list_mon_offset;
            } elsif ($jobname =~ /^\s*dump_list\s*$/) {
              $mon_offset = $dump_list_mon_offset;
            } elsif ($jobname =~ /^\s*del_list\s*$/) {
              $mon_offset = $del_list_mon_offset;
            } elsif ($jobname =~ /^\s*custom\d*\s*$/) {
              $mon_offset = $custom_mon_offset;
            } elsif ($jobname =~ /^\s*cloop\s*$/) {
              $mon_offset = $cloop_mon_offset;
            };

            my ($curr_mon, $curr_year) = new_mon_year($mon, $year, $mon_offset);
            my ($nxt_mon, $nxt_year) = new_mon_year($mon, $year, 11 + $mon_offset);

            # Do not include this job if it would appear before
            # the start of the include range determined above
            next if ($curr_year < $first_year_included);
            if ($curr_year == $first_year_included) {
              next if ($curr_mon < $first_mon_included);
            }

            # Do not include this job if it would appear after
            # the end of the include range determined above
            next if ($nxt_year > $last_year_included);
            if ($nxt_year == $last_year_included) {
              next if ($nxt_mon > $last_mon_included);
            }

            # Count the number of times this job gets used at this frequency
            my $job_id = "${jobname}_a";
            if ( exists $JOB_COUNT{$job_id} ) {
              # Increment the count for this job at this frequency
              $JOB_COUNT{$job_id} += 1;
            } else {
              # Initialize the count
              $JOB_COUNT{$job_id} = 1;
            }
            my $curr_job_count = $JOB_COUNT{$job_id};

            # Add parallel job delimiters to specified jobs
            my $OPTref = {CURR_MON        => $nxt_mon,
                          CURR_YEAR       => $nxt_year,
                          PREV_MON        => $curr_mon,
                          PREV_YEAR       => $curr_year,
                          RUN_START_YEAR  => $run_start_year,
                          RUN_START_MONTH => $run_start_month,
                          RUN_END_YEAR    => $run_end_year,
                          RUN_END_MONTH   => $run_end_month};
            my $pre_print = '';
            my $post_print = '';
            ($thisjob, $pre_print, $post_print) =
               insert_ext_delim($thisjob,$jobname,$OPTref);

            if ($main::Verbose == 1) {
              my $jinfo = " ${jobname}:a";
              if ($split_thisjob) {$jinfo = " ${jobname}:a:${split_thisjob}m"}
              my $strng = $pre_print . $jinfo . $post_print;
              if ( $line_out{pfx} ) {
                $strng = $line_out{pfx} . $pre_print . $jinfo . $post_print;
                $line_out{pfx} = "";
              }
              $line_out{wrote} = 1;
              print "$strng";
            }

            # replace variable values in $thisjob
            ($thisjob) = shell_var_subs::var_replace($thisjob,"year",$year);
            ($thisjob) = shell_var_subs::var_replace($thisjob,"mon",$mon);

            ($thisjob) = shell_var_subs::var_replace($thisjob,"current_year",
                            sprintf("%3.3d",$curr_year));
            ($thisjob) = shell_var_subs::var_replace($thisjob,"current_month",
                            sprintf("%2.2d",$curr_mon));
            ($thisjob) = shell_var_subs::var_replace($thisjob,"next_year",
                            sprintf("%3.3d",$nxt_year));
            ($thisjob) = shell_var_subs::var_replace($thisjob,"next_month",
                            sprintf("%2.2d",$nxt_mon));

            if ($split_thisjob) {
              # Insert multiple jobs with differences in only certain parameters
              # These are currently:
              # current_year, current_month, next_year, next_month
              my $mdelt = int($split_thisjob);
              my $year_a = $curr_year;
              my $mon_a  = $curr_mon;
              my ($mon_b, $year_b) = new_mon_year($mon_a, $year_a, $mdelt-1);
              my $delt_mon = diff_in_months($year_b,$mon_b,$nxt_year,$nxt_mon);
              if ($delt_mon < 0) {
                print "\nAttempting to split $jobname into $mdelt month intervals.\n";
                my $rmf = 1 + diff_in_months($curr_year,$curr_mon,$nxt_year,$nxt_mon);
                print "This is greater than the current interval of $rmf months.\n";
                die "Stopped";
              }
              while ( $delt_mon >= 0 ) {
                if ($main::Verbose > 3) {
                  print "\n$jobname   ";
                  print "year_a=$year_a mon_a=$mon_a ... year_b=$year_b mon_b=$mon_b\n";
                }
                ($thisjob) = shell_var_subs::var_replace($thisjob,"current_year",
                             sprintf("%3.3d",$year_a));
                ($thisjob) = shell_var_subs::var_replace($thisjob,"current_month",
                             sprintf("%2.2d",$mon_a));
                ($thisjob) = shell_var_subs::var_replace($thisjob,"next_year",
                             sprintf("%3.3d",$year_b));
                ($thisjob) = shell_var_subs::var_replace($thisjob,"next_month",
                             sprintf("%2.2d",$mon_b));
                $str .= $thisjob;
                # delt_mon=0 means mon_b,year_b is equal to nxt_mon,nxt_year
                last if $delt_mon == 0;
                ($mon_a, $year_a) = new_mon_year($mon_b, $year_b, 1);
                ($mon_b, $year_b) = new_mon_year($mon_a, $year_a, $mdelt-1);
                $delt_mon = diff_in_months($year_b,$mon_b,$nxt_year,$nxt_mon);
                if ($delt_mon < 0) {
                  # mon_b, year_b is later than nxt_mon, nxt_year
                  ($mon_b, $year_b) = ($nxt_mon, $nxt_year);
                  $delt_mon = 0;
                }
              }
            } else {
              $str .= $thisjob;
            }
          }

          # set previous_(year/month) to the month after the the last time
          # the section was inserted
          $annual{previous_month} = sprintf("%2.2d",$mon + 1);
          if ($annual{previous_month} > 12) {
            $annual{previous_month} = sprintf("%2.2d",1);
            $annual{previous_year} = sprintf("%3.3d",$year + 1);
          } else {
            $annual{previous_year} = $year;
          };

          # Append to the job string
          $job .= $str;
          if ($main::Verbose > $Vtrip) {
            print "=== annual === year=$year  mon=$mon ================\n";
            print "$str";
          };
          if ($main::Verbose > $Vtrip) {
            print "***WARNING*** annual: Failed to replace year\n" unless $rv1;
          };
        };
      };

      #========== quarterly section --- insert at start of quarter ==========
      if ( $mon == 1 || $mon == 4 || $mon == 7  || $mon == 10 ) {
        # copy the quarterly section template to a temporary string
        ($str) = @{$TEMPLATE{quarterly}};
        if ($str) {
          # split the current string into individual jobs
          # delimited by #end_of_job at the bottom
          my @splitjob = split_job_string($str);

          # define current values for parameters to be substituted
          my $mon1 = sprintf("%2.2d",$mon);
          my $mon2 = sprintf("%2.2d",$mon+1);
          my $mon3 = sprintf("%2.2d",$mon+2);
          my $qend_month = $mon + 2;
          my $qend_year = $year;
          if ($qend_month > 12) {
            $qend_year += 1;
            $qend_month = $qend_month - 12;
          };

          # perform section specific substitutions
          $str = '';
          foreach my $thisjob (@splitjob) {
            # Ignore empty job strings
            next if ($thisjob =~ /^\s*$/s);

            # get the name of the current job in jobname and
            # remove the job tag from the current job string
            my ($jobname) = read_jobtag($thisjob,{NAME_TAG=>1});

            # Extract info about splitting this job into multiple jobs
            my ($split_thisjob) = read_jobtag($thisjob,{SPLIT_TAG=>1});

	    # Get a hash ref to internal job specific options
            my $iopt_href = read_jobtag($thisjob,{IOPT_TAG=>1});

            # determine a year/month range, outside of which this job
            # should not be included in the final job string
            my $first_year_included = $run_start_year;
            my $first_mon_included  = $run_start_month;
            my $last_year_included  = $run_end_year;
            my $last_mon_included   = $run_end_month;
            $first_year_included = $iopt_href->{start_year} if $iopt_href->{start_year};
            $first_mon_included  = $iopt_href->{start_mon}  if $iopt_href->{start_mon};
            $last_year_included  = $iopt_href->{stop_year}  if $iopt_href->{stop_year};
            $last_mon_included   = $iopt_href->{stop_mon}   if $iopt_href->{stop_mon};

            # the value of mon_offset will depend on the type of job
            my $mon_offset = 0;
            if ($jobname =~ /^\s*mload\s*$/) {
              $mon_offset = $mload_mon_offset;
            } elsif ($jobname =~ /^\s*mdump\s*$/) {
              $mon_offset = $mdump_mon_offset;
            } elsif ($jobname =~ /^\s*mdelete\s*$/) {
              $mon_offset = $mdelete_mon_offset;
            } elsif ($jobname =~ /^\s*load_list\s*$/) {
              $mon_offset = $load_list_mon_offset;
            } elsif ($jobname =~ /^\s*dump_list\s*$/) {
              $mon_offset = $dump_list_mon_offset;
            } elsif ($jobname =~ /^\s*del_list\s*$/) {
              $mon_offset = $del_list_mon_offset;
            } elsif ($jobname =~ /^\s*custom\d*\s*$/) {
              $mon_offset = $custom_mon_offset;
            } elsif ($jobname =~ /^\s*cloop\s*$/) {
              $mon_offset = $cloop_mon_offset;
            };

            my ($curr_mon, $curr_year) = new_mon_year($mon, $year, $mon_offset);
            my ($nxt_mon, $nxt_year) = new_mon_year($qend_month, $qend_year, $mon_offset);

            # Do not include this job if it would appear before
            # the start of the include range determined above
            next if ($curr_year < $first_year_included);
            if ($curr_year == $first_year_included) {
              next if ($curr_mon < $first_mon_included);
            }

            # Do not include this job if it would appear after
            # the end of the include range determined above
            next if ($nxt_year > $last_year_included);
            if ($nxt_year == $last_year_included) {
              next if ($nxt_mon > $last_mon_included);
            }

            # Count the number of times this job gets used at this frequency
            my $job_id = "${jobname}_q";
            if ( exists $JOB_COUNT{$job_id} ) {
              # Increment the count for this job at this frequency
              $JOB_COUNT{$job_id} += 1;
            } else {
              # Initialize the count
              $JOB_COUNT{$job_id} = 1;
            }
            my $curr_job_count = $JOB_COUNT{$job_id};

            # Add parallel job delimiters to specified jobs
            my $OPTref = {CURR_MON        => $nxt_mon,
                          CURR_YEAR       => $nxt_year,
                          PREV_MON        => $curr_mon,
                          PREV_YEAR       => $curr_year,
                          RUN_START_YEAR  => $run_start_year,
                          RUN_START_MONTH => $run_start_month,
                          RUN_END_YEAR    => $run_end_year,
                          RUN_END_MONTH   => $run_end_month};
            my $pre_print = '';
            my $post_print = '';
            ($thisjob, $pre_print, $post_print) =
               insert_ext_delim($thisjob,$jobname,$OPTref);

            if ($main::Verbose == 1) {
              my $jinfo = " ${jobname}:q";
              if ($split_thisjob) {$jinfo = " ${jobname}:q:${split_thisjob}m"}
              my $strng = $pre_print . $jinfo . $post_print;
              if ( $line_out{pfx} ) {
                $strng = $line_out{pfx} . $pre_print . $jinfo . $post_print;
                $line_out{pfx} = "";
              }
              $line_out{wrote} = 1;
              print "$strng";
            }

            ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,"year",$year);
            ($thisjob,$rv2) = shell_var_subs::var_replace($thisjob,"mon",$mon);
            ($thisjob,$rv3) = shell_var_subs::var_replace($thisjob,"mon1",$mon1);
            ($thisjob,$rv4) = shell_var_subs::var_replace($thisjob,"mon2",$mon2);
            ($thisjob,$rv5) = shell_var_subs::var_replace($thisjob,"mon3",$mon3);

            ($thisjob,$rv6) = shell_var_subs::var_replace($thisjob,"current_year",
                                  sprintf("%3.3d",$curr_year));
            ($thisjob,$rv6) = shell_var_subs::var_replace($thisjob,"current_month",
                                  sprintf("%2.2d",$curr_mon));
            ($thisjob,$rv6) = shell_var_subs::var_replace($thisjob,"next_year",
                                  sprintf("%3.3d",$nxt_year));
            ($thisjob,$rv6) = shell_var_subs::var_replace($thisjob,"next_month",
                                  sprintf("%2.2d",$nxt_mon));

            if ($split_thisjob) {
              # Insert multiple jobs with differences in only certain parameters
              # These are currently:
              # current_year, current_month, next_year, next_month
              my $mdelt = int($split_thisjob);
              my $year_a = $curr_year;
              my $mon_a  = $curr_mon;
              my ($mon_b, $year_b) = new_mon_year($mon_a, $year_a, $mdelt-1);
              my $delt_mon = diff_in_months($year_b,$mon_b,$nxt_year,$nxt_mon);
              if ($delt_mon < 0) {
                print "\nAttempting to split $jobname into $mdelt month intervals.\n";
                my $rmf = 1 + diff_in_months($curr_year,$curr_mon,$nxt_year,$nxt_mon);
                print "This is greater than the current interval of $rmf months.\n";
                die "Stopped";
              }
              while ( $delt_mon >= 0 ) {
                if ($main::Verbose > 3) {
                  print "\n$jobname   ";
                  print "year_a=$year_a mon_a=$mon_a ... year_b=$year_b mon_b=$mon_b\n";
                }
                ($thisjob) = shell_var_subs::var_replace($thisjob,"current_year",
                             sprintf("%3.3d",$year_a));
                ($thisjob) = shell_var_subs::var_replace($thisjob,"current_month",
                             sprintf("%2.2d",$mon_a));
                ($thisjob) = shell_var_subs::var_replace($thisjob,"next_year",
                             sprintf("%3.3d",$year_b));
                ($thisjob) = shell_var_subs::var_replace($thisjob,"next_month",
                             sprintf("%2.2d",$mon_b));
                $str .= $thisjob;
                # delt_mon=0 means mon_b,year_b is equal to nxt_mon,nxt_year
                last if $delt_mon == 0;
                ($mon_a, $year_a) = new_mon_year($mon_b, $year_b, 1);
                ($mon_b, $year_b) = new_mon_year($mon_a, $year_a, $mdelt-1);
                $delt_mon = diff_in_months($year_b,$mon_b,$nxt_year,$nxt_mon);
                if ($delt_mon < 0) {
                  # mon_b, year_b is later than nxt_mon, nxt_year
                  ($mon_b, $year_b) = ($nxt_mon, $nxt_year);
                  $delt_mon = 0;
                }
              }
            } else {
              $str .= $thisjob;
            }
          }

          # Append to the job string
          $job .= $str;
          if ($main::Verbose > $Vtrip) {
            print "=== quarterly === year=$year  mon=$mon ================\n";
            print "$str";
          };
          if ($main::Verbose > $Vtrip) {
            print "***WARNING*** quarterly: Failed to replace year\n" unless $rv1;
            print "***WARNING*** quarterly: Failed to replace mon\n"  unless $rv2;
            print "***WARNING*** quarterly: Failed to replace mon1\n" unless $rv3;
            print "***WARNING*** quarterly: Failed to replace mon2\n" unless $rv4;
            print "***WARNING*** quarterly: Failed to replace mon3\n" unless $rv5;
          };
        };
      };

      #========== monthly section --- insert every $months months ==========
      # For a model job this will be the start of the month whereas for most
      # other jobs (e.g. mdump, mdelete etc.) it will be after the model has
      # run for $months months.
      if ($kount_start_iter > -1) {
        # copy the monthly section template to a temporary string
        ($str) = @{$TEMPLATE{monthly}};
        if ($str) {
          # split the current string into individual jobs
          # delimited by #end_of_job at the bottom
          my @splitjob = split_job_string($str);

          # define current values for parameters to be substituted
          my $obsday = $MONname[$mon-1];
          my $monn = $MONname[$mon-1];
          $monn =~ tr/A-Z/a-z/;   # lower case month name

          # perform section specific substitutions
          $str = '';
          foreach my $thisjob (@splitjob) {
            # Ignore empty job strings
            next if ($thisjob =~ /^\s*$/s);

            # get the name of the current job in jobname
            my ($jobname) = read_jobtag($thisjob,{NAME_TAG=>1});

            # Extract info about splitting this job into multiple jobs
            my ($split_thisjob) = read_jobtag($thisjob,{SPLIT_TAG=>1});

	    # Get a hash ref to internal job specific options
            my $iopt_href = read_jobtag($thisjob,{IOPT_TAG=>1});

            # determine a year/month range, outside of which this job
            # should not be included in the final job string
            my $first_year_included = $run_start_year;
            my $first_mon_included  = $run_start_month;
            my $last_year_included  = $run_end_year;
            my $last_mon_included   = $run_end_month;
            $first_year_included = $iopt_href->{start_year} if $iopt_href->{start_year};
            $first_mon_included  = $iopt_href->{start_mon}  if $iopt_href->{start_mon};
            $last_year_included  = $iopt_href->{stop_year}  if $iopt_href->{stop_year};
            $last_mon_included   = $iopt_href->{stop_mon}   if $iopt_href->{stop_mon};

            # current_(year|month) and previous_(year|month) are used by the modules
            # mdump, mdelete and mload. These modules generate file names based on
            # these values. Since mdump and mdelete are typically run after a model
            # run of $months months we set current_(year|month) to correspond to the
            # end of the model run and previous_(year|month) to correspond to the
            # start of the model run.
            my $mon_offset = 0;
            if ($jobname =~ /^\s*mload\s*$/) {
              $mon_offset = $mload_mon_offset;
            } elsif ($jobname =~ /^\s*mdump\s*$/) {
              $mon_offset = $mdump_mon_offset;
            } elsif ($jobname =~ /^\s*mdelete\s*$/) {
              $mon_offset = $mdelete_mon_offset;
            } elsif ($jobname =~ /^\s*load_list\s*$/) {
              $mon_offset = $load_list_mon_offset;
            } elsif ($jobname =~ /^\s*dump_list\s*$/) {
              $mon_offset = $dump_list_mon_offset;
            } elsif ($jobname =~ /^\s*del_list\s*$/) {
              $mon_offset = $del_list_mon_offset;
            } elsif ($jobname =~ /^\s*custom\d*\s*$/) {
              $mon_offset = $custom_mon_offset;
            } elsif ($jobname =~ /^\s*cloop\s*$/) {
              $mon_offset = $cloop_mon_offset;
            }

            my ($curr_mon, $curr_year) =
                new_mon_year($next_month, $next_year, $mon_offset);
            my ($prev_mon, $prev_year) =
                new_mon_year($mon, $year, $mon_offset);

            if ($main::Verbose > 10) {
              print "\nmon=${mon}: first_year_included=$first_year_included";
              print "   first_mon_included=$first_mon_included\n";
              print "mon=${mon}: last_year_included=$last_year_included";
              print "    last_mon_included=$last_mon_included\n";
              print "mon=${mon}: prev_year=$prev_year   prev_mon=$prev_mon";
              print "mon=${mon}: curr_year=$curr_year   curr_mon=$curr_mon\n";
              while (my ($var,$val) = each %{$iopt_href}) {
                print "mon=${mon} iopt: $var=$val\n";
              }
            }

            # Do not include this job if it would appear before
            # the start of the include range determined above
            next if ($prev_year < $first_year_included);
            if ($prev_year == $first_year_included) {
              next if ($prev_mon < $first_mon_included);
            }

            # Do not include this job if it would appear after
            # the end of the include range determined above
            next if ($curr_year > $last_year_included);
            if ($curr_year == $last_year_included) {
              next if ($curr_mon > $last_mon_included);
            }

            # Count the number of times this job gets used at this frequency
            my $job_id = "${jobname}_m";
            if ( exists $JOB_COUNT{$job_id} ) {
              # Increment the count for this job at this frequency
              $JOB_COUNT{$job_id} += 1;
            } else {
              # Initialize the count
              $JOB_COUNT{$job_id} = 1;
            }
            my $curr_job_count = $JOB_COUNT{$job_id};

            # Add parallel job delimiters to specified jobs
            my $OPTref = {CURR_MON        => $curr_mon,
                          CURR_YEAR       => $curr_year,
                          PREV_MON        => $prev_mon,
                          PREV_YEAR       => $prev_year,
                          RUN_START_YEAR  => $run_start_year,
                          RUN_START_MONTH => $run_start_month,
                          RUN_END_YEAR    => $run_end_year,
                          RUN_END_MONTH   => $run_end_month};
            my $pre_print = '';
            my $post_print = '';
            ($thisjob, $pre_print, $post_print) =
               insert_ext_delim($thisjob,$jobname,$OPTref);

            if ($main::Verbose == 1) {
              my $jinfo = " ${jobname}:m";
              if ($split_thisjob) {$jinfo = " ${jobname}:m:${split_thisjob}m"}
              my $strng = $pre_print . $jinfo . $post_print;
              if ( $line_out{pfx} ) {
                $strng = $line_out{pfx} . $pre_print . $jinfo . $post_print;
                $line_out{pfx} = "";
              }
              $line_out{wrote} = 1;
              print "$strng";
            }

            ($thisjob) = shell_var_subs::var_replace($thisjob,"monn",        $monn);
            ($thisjob) = shell_var_subs::var_replace($thisjob,"year",        $year);
            ($thisjob) = shell_var_subs::var_replace($thisjob,"mon",         $mon);
            ($thisjob) = shell_var_subs::var_replace($thisjob,"days",        $obsday);
            ($thisjob) = shell_var_subs::var_replace($thisjob,"obsday",      $obsday);
            ($thisjob) = shell_var_subs::var_replace($thisjob,"kfinal",      $kfinal);
            ($thisjob) = shell_var_subs::var_replace($thisjob,"year_restart",$year_restart);
            ($thisjob) = shell_var_subs::var_replace($thisjob,"mon_restart", $mon_restart);
            ($thisjob) = shell_var_subs::var_replace($thisjob,"samerun",     $samerun);
            ($thisjob) = shell_var_subs::var_replace($thisjob,"initsp",      $initsp);
            ($thisjob) = shell_var_subs::var_replace($thisjob,"cpl_new_rtd", $cpl_new_rtd);
            ($thisjob) = shell_var_subs::var_replace($thisjob,"nemo_new_rtd",$nemo_new_rtd);
            if ($first_loop_iter and $main::Restart_fname) {
              # If a restart file name is available then set start in the string
              my $start = $main::Restart_fname;
              # strip any three digit numerical extension as well as a trailing "rs"
              # A gcm model job will use ${start}rs as the restart file name
              $start  =~ s/(.*)[cor]s(\.[0-9]{3})?$/$1/;
              # print "\nstart = $start  Restart_fname = $main::Restart_fname\n";
              ($thisjob) = shell_var_subs::var_replace($thisjob,"start",$start);
            };
            if ($first_loop_iter and exists $main::VARS{nemo_restart}) {
              # If a nemo restart file name is available then
              # set it only on the first job of the string
              my $nemo_restart = $main::VARS{nemo_restart}{value};
              # Remove nemo_restart from the VARS hash so that it does not
              # get set in any other jobs in this string
              delete $main::VARS{nemo_restart};
              if ( $nemo_restart ) {
                # Redefine nemo_restart only when it has a non null value
                # strip any three digit numerical extension
                $nemo_restart  =~ s/(.*)(\.[0-9]{3})?$/$1/;
                ($thisjob) = shell_var_subs::var_replace($thisjob,"nemo_restart",$nemo_restart);
              }
            }
            if ($first_loop_iter and exists $main::VARS{nemo_exec}) {
              # If a nemo_exec is set by the user then
              # set it only on the first job of the string
              my $nemo_exec = $main::VARS{nemo_exec}{value};
              # Remove nemo_exec from the VARS hash so that it does not
              # get set in any other jobs in this string
              delete $main::VARS{nemo_exec};
              if ( $nemo_exec ) {
                # Redefine nemo_exec only when it has a non null value
                ($thisjob) = shell_var_subs::var_replace($thisjob,"nemo_exec",$nemo_exec);
              }
            }
            if (exists $main::VARS{nemo_nn_it000}) {
              # If nemo_nn_it000 is set by the user then
              # set it only on the first job of the string
              my $nemo_nn_it000 = $main::VARS{nemo_nn_it000}{value};
              # Remove nemo_nn_it000 from the VARS hash so that it does not
              # get set in any other jobs in this string
              delete $main::VARS{nemo_nn_it000};
              if ( $nemo_nn_it000 ) {
                # Redefine nemo_nn_it000 only when it has a non null value
                ($thisjob) = shell_var_subs::var_replace($thisjob,"nemo_nn_it000",$nemo_nn_it000);
              }
            }
            if ($first_loop_iter and exists $main::VARS{nemo_from_rest}) {
              # If nemo_from_rest is set by the user then
              # set it only on the first job of the string
              my $nemo_from_rest = $main::VARS{nemo_from_rest}{value};
              # Remove nemo_from_rest from the VARS hash so that it does not
              # get set in any other jobs in this string
              delete $main::VARS{nemo_from_rest};
              # Redefine nemo_from_rest in the current job only
              ($thisjob) = shell_var_subs::var_replace($thisjob,"nemo_from_rest",$nemo_from_rest);
            }
            if ($first_loop_iter and exists $main::VARS{nemo_force_namelist}) {
              # If nemo_force_namelist is set by the user then
              # set it only on the first job of the string
              my $nemo_force_namelist = $main::VARS{nemo_force_namelist}{value};
              # Remove nemo_force_namelist from the VARS hash so that it does not
              # get set in any other jobs in this string
              delete $main::VARS{nemo_force_namelist};
              # Redefine nemo_force_namelist in the current job only
              ($thisjob) = shell_var_subs::var_replace($thisjob,"nemo_force_namelist",$nemo_force_namelist);
            }

            ($thisjob) = shell_var_subs::var_replace($thisjob,"current_year",
                             sprintf("%3.3d",$curr_year));
            ($thisjob) = shell_var_subs::var_replace($thisjob,"current_month",
                             sprintf("%2.2d",$curr_mon));
            ($thisjob) = shell_var_subs::var_replace($thisjob,"previous_year",
                             sprintf("%3.3d",$prev_year));
            ($thisjob) = shell_var_subs::var_replace($thisjob,"previous_month",
                             sprintf("%2.2d",$prev_mon));

            # RCM parameters
            if ($main::MK_RCM_DATA) {
              # Add RCM parameters only when RCM forcing data is to be generated
              $thisjob =~ s/^\s*#[ #]*<<INSERT_RCM_PARAMETERS>>[ #]*$/$main::RCM_PARAMS/m;
              ($thisjob) = shell_var_subs::var_replace($thisjob,"M1",$rcm_M1);
              ($thisjob) = shell_var_subs::var_replace($thisjob,"M2",$rcm_M2);
              ($thisjob) = shell_var_subs::var_replace($thisjob,"RUNP",$rcm_RUNP);
              # The parameter RUN is used in both the GCM and the RCM
              ($thisjob) = shell_var_subs::var_replace($thisjob,"RUN",$rcm_RUN);
              ($thisjob) = shell_var_subs::var_replace($thisjob,"MRCKSTART",$rcm_MRCKSTART);
              ($thisjob) = shell_var_subs::var_replace($thisjob,"MRCKTOTAL",$rcm_MRCKTOTAL);
              ($thisjob) = shell_var_subs::var_replace($thisjob,"MRCRADINIT",$rcm_MRCRADINIT);
              ($thisjob) = shell_var_subs::var_replace($thisjob,"MRCRADFINI",$rcm_MRCRADFINI);
              ($thisjob) = shell_var_subs::var_replace($thisjob,"MRCRADIDAY",$rcm_MRCRADIDAY);
              ($thisjob) = shell_var_subs::var_replace($thisjob,"MRCRADGMT",$rcm_MRCRADGMT);
              ($thisjob) = shell_var_subs::var_replace($thisjob,"GCMKINIT",$rcm_GCMKINIT);
              ($thisjob) = shell_var_subs::var_replace($thisjob,"GCMKFINI",$rcm_GCMKFINI);
            }

            if ($split_thisjob) {
              # Insert multiple jobs with differences in only certain parameters
              # These are currently:
              # current_year, current_month, previous_year, previous_month
              my $mdelt = int($split_thisjob);
              my $year_a = $prev_year;
              my $mon_a  = $prev_mon;
              my ($mon_b, $year_b) = new_mon_year($mon_a, $year_a, $mdelt-1);
              my $delt_mon = diff_in_months($year_b,$mon_b,$curr_year,$curr_mon);
              if ($delt_mon < 0) {
                print "\nAttempting to split $jobname into $mdelt month intervals.\n";
                my $rmf = 1 + diff_in_months($prev_year,$prev_mon,$curr_year,$curr_mon);
                print "This is greater than the current interval of $rmf months.\n";
                die "Stopped";
              }
              while ( $delt_mon >= 0 ) {
                if ($main::Verbose > 3) {
                  print "\n$jobname   ";
                  print "year_a=$year_a mon_a=$mon_a ... year_b=$year_b mon_b=$mon_b\n";
                }
                ($thisjob) = shell_var_subs::var_replace($thisjob,"previous_year",
                             sprintf("%3.3d",$year_a));
                ($thisjob) = shell_var_subs::var_replace($thisjob,"previous_month",
                             sprintf("%2.2d",$mon_a));
                ($thisjob) = shell_var_subs::var_replace($thisjob,"current_year",
                             sprintf("%3.3d",$year_b));
                ($thisjob) = shell_var_subs::var_replace($thisjob,"current_month",
                             sprintf("%2.2d",$mon_b));
                $str .= $thisjob;
                # delt_mon=0 means mon_b,year_b is equal to curr_mon,curr_year
                last if $delt_mon == 0;
                ($mon_a, $year_a) = new_mon_year($mon_b, $year_b, 1);
                ($mon_b, $year_b) = new_mon_year($mon_a, $year_a, $mdelt-1);
                $delt_mon = diff_in_months($year_b,$mon_b,$curr_year,$curr_mon);
                if ($delt_mon < 0) {
                  # mon_b, year_b is later than curr_mon, curr_year
                  ($mon_b, $year_b) = ($curr_mon, $curr_year);
                  $delt_mon = 0;
                }
              }
            } else {
              $str .= $thisjob;
            }
          }

          # set $monthly{previous_(year/month)} to the month the last time
          # the section was inserted
          $monthly{previous_month} = sprintf("%2.2d",$mon);
          $monthly{previous_year}  = sprintf("%3.3d",$year);

          # Append to the job string
          $job .= $str;
          if ($main::Verbose > $Vtrip) {
            print "=== monthly === year=$year  mon=$mon ================\n";
            print "$str";
          }
        }
      }

      #==========  DJF section --- insert at end of DJF ==========
      if ( $mon == 2 and
           ($year > $run_start_year or $force_first_djf or $force_first_sea) ) {
        # Do not include DJF for run_start_year unless force_first_djf or
        # force_first_sea is set
        # copy the DJF section template to a temporary string
        ($str) = @{$TEMPLATE{DJF}};
        if ($str) {
          # split the current string into individual jobs
          # delimited by #end_of_job at the bottom
          my @splitjob = split_job_string($str);

          # perform section specific substitutions
          $str = '';
          foreach my $thisjob (@splitjob) {
            # Ignore empty job strings
            next if ($thisjob =~ /^\s*$/s);

            # get the name of the current job in jobname
            my ($jobname) = read_jobtag($thisjob,{NAME_TAG=>1});

            # Extract info about splitting this job into multiple jobs
            my ($split_thisjob) = read_jobtag($thisjob,{SPLIT_TAG=>1});

	    # Get a hash ref to internal job specific options
            my $iopt_href = read_jobtag($thisjob,{IOPT_TAG=>1});

            # determine a year/month range, outside of which this job
            # should not be included in the final job string
            my $first_year_included = $run_start_year;
            my $first_mon_included  = $run_start_month;
            my $last_year_included  = $run_end_year;
            my $last_mon_included   = $run_end_month;
            $first_year_included = $iopt_href->{start_year} if $iopt_href->{start_year};
            $first_mon_included  = $iopt_href->{start_mon}  if $iopt_href->{start_mon};
            $last_year_included  = $iopt_href->{stop_year}  if $iopt_href->{stop_year};
            $last_mon_included   = $iopt_href->{stop_mon}   if $iopt_href->{stop_mon};
            if ($main::Verbose > 10) {
              print "\nDJF: first_year_included=$first_year_included";
              print "   first_mon_included=$first_mon_included\n";
              print "DJF:  last_year_included=$last_year_included";
              print "    last_mon_included=$last_mon_included\n";
              while (my ($var,$val) = each %{$iopt_href}) {
                print "DJF iopt: $var=$val\n";
              }
            }

            my $mon_offset = 0;
            if ($jobname =~ /^\s*mload\s*$/) {
              $mon_offset = $mload_mon_offset;
            } elsif ($jobname =~ /^\s*mdump\s*$/) {
              $mon_offset = $mdump_mon_offset;
            } elsif ($jobname =~ /^\s*mdelete\s*$/) {
              $mon_offset = $mdelete_mon_offset;
            } elsif ($jobname =~ /^\s*load_list\s*$/) {
              $mon_offset = $load_list_mon_offset;
            } elsif ($jobname =~ /^\s*dump_list\s*$/) {
              $mon_offset = $dump_list_mon_offset;
            } elsif ($jobname =~ /^\s*del_list\s*$/) {
              $mon_offset = $del_list_mon_offset;
            } elsif ($jobname =~ /^\s*custom\d*\s*$/) {
              $mon_offset = $custom_mon_offset;
            } elsif ($jobname =~ /^\s*cloop\s*$/) {
              $mon_offset = $cloop_mon_offset;
            }

            my ($curr_mon, $curr_year) =
                new_mon_year($mon, $year, $mon_offset);
            my ($prev_mon, $prev_year) =
                new_mon_year($mon, $year, $mon_offset - 2);

            if ( $force_first_djf or $force_first_sea ) {
              # Reset the local variable first_year_included to its
              # current value minus 1 so that the first DJF of the
              # string will be included. If this is not done then
              # the first DJF will never be included, regardless of
              # the value of force_first_djf
              $first_year_included = $first_year_included - 1;
            }

            # Do not include this job if it would appear before
            # the start of the include range determined above
            next if ($prev_year < $first_year_included);
            if ($prev_year == $first_year_included) {
              next if ($prev_mon < $first_mon_included);
            }

            # Do not include this job if it would appear after
            # the end of the include range determined above
            next if ($curr_year > $last_year_included);
            if ($curr_year == $last_year_included) {
              next if ($curr_mon > $last_mon_included);
            }

            # Count the number of times this job gets used at this frequency
            my $job_id = "${jobname}_DJF";
            if ( exists $JOB_COUNT{$job_id} ) {
              # Increment the count for this job at this frequency
              $JOB_COUNT{$job_id} += 1;
            } else {
              # Initialize the count
              $JOB_COUNT{$job_id} = 1;
            }
            my $curr_job_count = $JOB_COUNT{$job_id};

            # Add parallel job delimiters to specified jobs
            my $OPTref = {CURR_MON        => $curr_mon,
                          CURR_YEAR       => $curr_year,
                          PREV_MON        => $prev_mon,
                          PREV_YEAR       => $prev_year,
                          RUN_START_YEAR  => $run_start_year,
                          RUN_START_MONTH => $run_start_month,
                          RUN_END_YEAR    => $run_end_year,
                          RUN_END_MONTH   => $run_end_month};
            my $pre_print = '';
            my $post_print = '';
            ($thisjob, $pre_print, $post_print) =
               insert_ext_delim($thisjob,$jobname,$OPTref);

            if ($main::Verbose == 1) {
              my $jinfo = " ${jobname}:D";
              if ($split_thisjob) {$jinfo = " ${jobname}:D:${split_thisjob}m"}
              my $strng = $pre_print . $jinfo . $post_print;
              if ( $line_out{pfx} ) {
                $strng = $line_out{pfx} . $pre_print . $jinfo . $post_print;
                $line_out{pfx} = "";
              }
              $line_out{wrote} = 1;
              print "$strng";
            }

            # replace variable values in $thisjob
            ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,"mm01","m12");
            ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,"mm02","m01");
            ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,"mm03","m02");
            ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,"mn01","dec");
            ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,"mn02","jan");
            ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,"mn03","feb");
            ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,"ml01","   31");
            ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,"ml02","   31");
            ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,"ml03","   28");
            ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,"season","djf");
            ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,"days",  "DJF");
            ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,"obsday","DJF");
            ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,"year",$year);
            ($thisjob,$rv2) = shell_var_subs::var_replace($thisjob,"mon", $mon);
            ($thisjob,$rv3) = shell_var_subs::var_replace($thisjob,
                                  "yearm1",sprintf("%3.3d",$year-1));

            ($thisjob) = shell_var_subs::var_replace($thisjob,"current_year",
                             sprintf("%3.3d",$curr_year));
            ($thisjob) = shell_var_subs::var_replace($thisjob,"current_month",
                             sprintf("%2.2d",$curr_mon));
            ($thisjob) = shell_var_subs::var_replace($thisjob,"previous_year",
                             sprintf("%3.3d",$prev_year));
            ($thisjob) = shell_var_subs::var_replace($thisjob,"previous_month",
                             sprintf("%2.2d",$prev_mon));

            if ($split_thisjob) {
              # Insert multiple jobs with differences in only certain parameters
              # These are currently:
              # current_year, current_month, previous_year, previous_month
              my $mdelt = int($split_thisjob);
              my $year_a = $prev_year;
              my $mon_a  = $prev_mon;
              my ($mon_b, $year_b) = new_mon_year($mon_a, $year_a, $mdelt-1);
              my $delt_mon = diff_in_months($year_b,$mon_b,$curr_year,$curr_mon);
              if ($delt_mon < 0) {
                print "\nAttempting to split $jobname into $mdelt month intervals.\n";
                my $rmf = 1 + diff_in_months($prev_year,$prev_mon,$curr_year,$curr_mon);
                print "This is greater than the current interval of $rmf months.\n";
                die "Stopped";
              }
              while ( $delt_mon >= 0 ) {
                if ($main::Verbose > 3) {
                  print "\n$jobname   ";
                  print "year_a=$year_a mon_a=$mon_a ... year_b=$year_b mon_b=$mon_b\n";
                }
                ($thisjob) = shell_var_subs::var_replace($thisjob,"previous_year",
                             sprintf("%3.3d",$year_a));
                ($thisjob) = shell_var_subs::var_replace($thisjob,"previous_month",
                             sprintf("%2.2d",$mon_a));
                ($thisjob) = shell_var_subs::var_replace($thisjob,"current_year",
                             sprintf("%3.3d",$year_b));
                ($thisjob) = shell_var_subs::var_replace($thisjob,"current_month",
                             sprintf("%2.2d",$mon_b));
                $str .= $thisjob;
                # delt_mon=0 means mon_b,year_b is equal to curr_mon,curr_year
                last if $delt_mon == 0;
                ($mon_a, $year_a) = new_mon_year($mon_b, $year_b, 1);
                ($mon_b, $year_b) = new_mon_year($mon_a, $year_a, $mdelt-1);
                $delt_mon = diff_in_months($year_b,$mon_b,$curr_year,$curr_mon);
                if ($delt_mon < 0) {
                  # mon_b, year_b is later than curr_mon, curr_year
                  ($mon_b, $year_b) = ($curr_mon, $curr_year);
                  $delt_mon = 0;
                }
              }
            } else {
              $str .= $thisjob;
            }
          }

          # Append to the job string
          $job .= $str;
          if ($main::Verbose > $Vtrip) {
            print "=== DJF === year=$year  mon=$mon ================\n";
            print "$str";
          };
          if ($main::Verbose > $Vtrip) {
            print "***WARNING*** DJF: Failed to replace year\n"   unless $rv1;
            print "***WARNING*** DJF: Failed to replace yearm1\n" unless $rv3;
          };
        };
      };

      #==========  MAM section --- insert at end of MAM ==========
      if ( $mon == 5 ) {
        # copy the MAM section template to a temporary string
        ($str) = @{$TEMPLATE{MAM}};
        if ($str) {
          # split the current string into individual jobs
          # delimited by #end_of_job at the bottom
          my @splitjob = split_job_string($str);

          # perform section specific substitutions
          $str = '';
          foreach my $thisjob (@splitjob) {
            # Ignore empty job strings
            next if ($thisjob =~ /^\s*$/s);

            # get the name of the current job in jobname and
            # remove the job tag from the current job string
            my ($jobname) = read_jobtag($thisjob,{NAME_TAG=>1});

            # Extract info about splitting this job into multiple jobs
            my ($split_thisjob) = read_jobtag($thisjob,{SPLIT_TAG=>1});

	    # Get a hash ref to internal job specific options
            my $iopt_href = read_jobtag($thisjob,{IOPT_TAG=>1});

            # determine a year/month range, outside of which this job
            # should not be included in the final job string
            my $first_year_included = $run_start_year;
            my $first_mon_included  = $run_start_month;
            my $last_year_included  = $run_end_year;
            my $last_mon_included   = $run_end_month;
            $first_year_included = $iopt_href->{start_year} if $iopt_href->{start_year};
            $first_mon_included  = $iopt_href->{start_mon}  if $iopt_href->{start_mon};
            $last_year_included  = $iopt_href->{stop_year}  if $iopt_href->{stop_year};
            $last_mon_included   = $iopt_href->{stop_mon}   if $iopt_href->{stop_mon};

            my $mon_offset = 0;
            if ($jobname =~ /^\s*mload\s*$/) {
              $mon_offset = $mload_mon_offset;
            } elsif ($jobname =~ /^\s*mdump\s*$/) {
              $mon_offset = $mdump_mon_offset;
            } elsif ($jobname =~ /^\s*mdelete\s*$/) {
              $mon_offset = $mdelete_mon_offset;
            } elsif ($jobname =~ /^\s*load_list\s*$/) {
              $mon_offset = $load_list_mon_offset;
            } elsif ($jobname =~ /^\s*dump_list\s*$/) {
              $mon_offset = $dump_list_mon_offset;
            } elsif ($jobname =~ /^\s*del_list\s*$/) {
              $mon_offset = $del_list_mon_offset;
            } elsif ($jobname =~ /^\s*custom\d*\s*$/) {
              $mon_offset = $custom_mon_offset;
            } elsif ($jobname =~ /^\s*cloop\s*$/) {
              $mon_offset = $cloop_mon_offset;
            };

            my ($curr_mon, $curr_year) =
                new_mon_year($mon, $year, $mon_offset);
            my ($prev_mon, $prev_year) =
                new_mon_year($mon, $year, $mon_offset - 2);

            if ( $force_first_sea ) {
              # Reset the local variable first_mon_included to
              # indicate March so that the first MAM of the
              # string will be included
              $first_mon_included = 3;
            }

            # Do not include this job if it would appear before
            # the start of the include range determined above
            next if ($prev_year < $first_year_included);
            if ($prev_year == $first_year_included) {
              next if ($prev_mon < $first_mon_included);
            }

            # Do not include this job if it would appear after
            # the end of the include range determined above
            next if ($curr_year > $last_year_included);
            if ($curr_year == $last_year_included) {
              next if ($curr_mon > $last_mon_included);
            }

            # Count the number of times this job gets used at this frequency
            my $job_id = "${jobname}_MAM";
            if ( exists $JOB_COUNT{$job_id} ) {
              # Increment the count for this job at this frequency
              $JOB_COUNT{$job_id} += 1;
            } else {
              # Initialize the count
              $JOB_COUNT{$job_id} = 1;
            }
            my $curr_job_count = $JOB_COUNT{$job_id};

            # Add parallel job delimiters to specified jobs
            my $OPTref = {CURR_MON        => $curr_mon,
                          CURR_YEAR       => $curr_year,
                          PREV_MON        => $prev_mon,
                          PREV_YEAR       => $prev_year,
                          RUN_START_YEAR  => $run_start_year,
                          RUN_START_MONTH => $run_start_month,
                          RUN_END_YEAR    => $run_end_year,
                          RUN_END_MONTH   => $run_end_month};
            my $pre_print = '';
            my $post_print = '';
            ($thisjob, $pre_print, $post_print) =
               insert_ext_delim($thisjob,$jobname,$OPTref);

            if ($main::Verbose == 1) {
              my $jinfo = " ${jobname}:M";
              if ($split_thisjob) {$jinfo = " ${jobname}:M:${split_thisjob}m"}
              my $strng = $pre_print . $jinfo . $post_print;
              if ( $line_out{pfx} ) {
                $strng = $line_out{pfx} . $pre_print . $jinfo . $post_print;
                $line_out{pfx} = "";
              }
              $line_out{wrote} = 1;
              print "$strng";
            }

            # replace variable values in $thisjob
            ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,"mm01","m03");
            ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,"mm02","m04");
            ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,"mm03","m05");
            ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,"mn01","mar");
            ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,"mn02","apr");
            ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,"mn03","may");
            ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,"ml01","   31");
            ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,"ml02","   30");
            ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,"ml03","   31");
            ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,"season","mam");
            ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,"days","MAM");
            ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,"obsday","MAM");
            ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,"year",$year);
            ($thisjob,$rv2) = shell_var_subs::var_replace($thisjob,"mon", $mon);

            ($thisjob) = shell_var_subs::var_replace($thisjob,"current_year",
                             sprintf("%3.3d",$curr_year));
            ($thisjob) = shell_var_subs::var_replace($thisjob,"current_month",
                             sprintf("%2.2d",$curr_mon));
            ($thisjob) = shell_var_subs::var_replace($thisjob,"previous_year",
                             sprintf("%3.3d",$prev_year));
            ($thisjob) = shell_var_subs::var_replace($thisjob,"previous_month",
                             sprintf("%2.2d",$prev_mon));

            if ($split_thisjob) {
              # Insert multiple jobs with differences in only certain parameters
              # These are currently:
              # current_year, current_month, previous_year, previous_month
              my $mdelt = int($split_thisjob);
              my $year_a = $prev_year;
              my $mon_a  = $prev_mon;
              my ($mon_b, $year_b) = new_mon_year($mon_a, $year_a, $mdelt-1);
              my $delt_mon = diff_in_months($year_b,$mon_b,$curr_year,$curr_mon);
              if ($delt_mon < 0) {
                print "\nAttempting to split $jobname into $mdelt month intervals.\n";
                my $rmf = 1 + diff_in_months($prev_year,$prev_mon,$curr_year,$curr_mon);
                print "This is greater than the current interval of $rmf months.\n";
                die "Stopped";
              }
              while ( $delt_mon >= 0 ) {
                if ($main::Verbose > 3) {
                  print "\n$jobname   ";
                  print "year_a=$year_a mon_a=$mon_a ... year_b=$year_b mon_b=$mon_b\n";
                }
                ($thisjob) = shell_var_subs::var_replace($thisjob,"previous_year",
                             sprintf("%3.3d",$year_a));
                ($thisjob) = shell_var_subs::var_replace($thisjob,"previous_month",
                             sprintf("%2.2d",$mon_a));
                ($thisjob) = shell_var_subs::var_replace($thisjob,"current_year",
                             sprintf("%3.3d",$year_b));
                ($thisjob) = shell_var_subs::var_replace($thisjob,"current_month",
                             sprintf("%2.2d",$mon_b));
                $str .= $thisjob;
                # delt_mon=0 means mon_b,year_b is equal to curr_mon,curr_year
                last if $delt_mon == 0;
                ($mon_a, $year_a) = new_mon_year($mon_b, $year_b, 1);
                ($mon_b, $year_b) = new_mon_year($mon_a, $year_a, $mdelt-1);
                $delt_mon = diff_in_months($year_b,$mon_b,$curr_year,$curr_mon);
                if ($delt_mon < 0) {
                  # mon_b, year_b is later than curr_mon, curr_year
                  ($mon_b, $year_b) = ($curr_mon, $curr_year);
                  $delt_mon = 0;
                }
              }
            } else {
              $str .= $thisjob;
            }
          }

          # Append to the job string
          $job .= $str;
          if ($main::Verbose > $Vtrip) {
            print "=== MAM === year=$year  mon=$mon ================\n";
            print "$str";
          };
          if ($main::Verbose > $Vtrip) {
            print "***WARNING*** MAM: Failed to replace year\n" unless $rv1;
          };
        };
      };

      #==========  JJA section --- insert at end of JJA ==========
      if ( $mon == 8 ) {
        # copy the JJA section template to a temporary string
        ($str) = @{$TEMPLATE{JJA}};
        if ($str) {
          # split the current string into individual jobs
          # delimited by #end_of_job at the bottom
          my @splitjob = split_job_string($str);

          # perform section specific substitutions
          $str = '';
          foreach my $thisjob (@splitjob) {
            # Ignore empty job strings
            next if ($thisjob =~ /^\s*$/s);

            # get the name of the current job in jobname and
            # remove the job tag from the current job string
            my ($jobname) = read_jobtag($thisjob,{NAME_TAG=>1});

            # Extract info about splitting this job into multiple jobs
            my ($split_thisjob) = read_jobtag($thisjob,{SPLIT_TAG=>1});

	    # Get a hash ref to internal job specific options
            my $iopt_href = read_jobtag($thisjob,{IOPT_TAG=>1});

            # determine a year/month range, outside of which this job
            # should not be included in the final job string
            my $first_year_included = $run_start_year;
            my $first_mon_included  = $run_start_month;
            my $last_year_included  = $run_end_year;
            my $last_mon_included   = $run_end_month;
            $first_year_included = $iopt_href->{start_year} if $iopt_href->{start_year};
            $first_mon_included  = $iopt_href->{start_mon}  if $iopt_href->{start_mon};
            $last_year_included  = $iopt_href->{stop_year}  if $iopt_href->{stop_year};
            $last_mon_included   = $iopt_href->{stop_mon}   if $iopt_href->{stop_mon};

            my $mon_offset = 0;
            if ($jobname =~ /^\s*mload\s*$/) {
              $mon_offset = $mload_mon_offset;
            } elsif ($jobname =~ /^\s*mdump\s*$/) {
              $mon_offset = $mdump_mon_offset;
            } elsif ($jobname =~ /^\s*mdelete\s*$/) {
              $mon_offset = $mdelete_mon_offset;
            } elsif ($jobname =~ /^\s*load_list\s*$/) {
              $mon_offset = $load_list_mon_offset;
            } elsif ($jobname =~ /^\s*dump_list\s*$/) {
              $mon_offset = $dump_list_mon_offset;
            } elsif ($jobname =~ /^\s*del_list\s*$/) {
              $mon_offset = $del_list_mon_offset;
            } elsif ($jobname =~ /^\s*custom\d*\s*$/) {
              $mon_offset = $custom_mon_offset;
            } elsif ($jobname =~ /^\s*cloop\s*$/) {
              $mon_offset = $cloop_mon_offset;
            };

            my ($curr_mon, $curr_year) =
                new_mon_year($mon, $year, $mon_offset);
            my ($prev_mon, $prev_year) =
                new_mon_year($mon, $year, $mon_offset - 2);

            if ( $force_first_sea ) {
              # Reset the local variable first_mon_included to
              # indicate June so that the first JJA of the
              # string will be included
              $first_mon_included = 6;
            }

            # Do not include this job if it would appear before
            # the start of the include range determined above
            next if ($prev_year < $first_year_included);
            if ($prev_year == $first_year_included) {
              next if ($prev_mon < $first_mon_included);
            }

            # Do not include this job if it would appear after
            # the end of the include range determined above
            next if ($curr_year > $last_year_included);
            if ($curr_year == $last_year_included) {
              next if ($curr_mon > $last_mon_included);
            }

            # Count the number of times this job gets used at this frequency
            my $job_id = "${jobname}_JJA";
            if ( exists $JOB_COUNT{$job_id} ) {
              # Increment the count for this job at this frequency
              $JOB_COUNT{$job_id} += 1;
            } else {
              # Initialize the count
              $JOB_COUNT{$job_id} = 1;
            }
            my $curr_job_count = $JOB_COUNT{$job_id};

            # Add parallel job delimiters to specified jobs
            my $OPTref = {CURR_MON        => $curr_mon,
                          CURR_YEAR       => $curr_year,
                          PREV_MON        => $prev_mon,
                          PREV_YEAR       => $prev_year,
                          RUN_START_YEAR  => $run_start_year,
                          RUN_START_MONTH => $run_start_month,
                          RUN_END_YEAR    => $run_end_year,
                          RUN_END_MONTH   => $run_end_month};
            my $pre_print = '';
            my $post_print = '';
            ($thisjob, $pre_print, $post_print) =
               insert_ext_delim($thisjob,$jobname,$OPTref);

            if ($main::Verbose == 1) {
              my $jinfo = " ${jobname}:J";
              if ($split_thisjob) {$jinfo = " ${jobname}:J:${split_thisjob}m"}
              my $strng = $pre_print . $jinfo . $post_print;
              if ( $line_out{pfx} ) {
                $strng = $line_out{pfx} . $pre_print . $jinfo . $post_print;
                $line_out{pfx} = "";
              }
              $line_out{wrote} = 1;
              print "$strng";
            }

            # replace variable values in $thisjob
            ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,"mm01","m06");
            ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,"mm02","m07");
            ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,"mm03","m08");
            ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,"mn01","jun");
            ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,"mn02","jul");
            ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,"mn03","aug");
            ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,"ml01","   30");
            ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,"ml02","   31");
            ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,"ml03","   31");
            ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,"season","jja");
            ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,"days","JJA");
            ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,"obsday","JJA");
            ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,"year",$year);
            ($thisjob,$rv2) = shell_var_subs::var_replace($thisjob,"mon", $mon);

            ($thisjob) = shell_var_subs::var_replace($thisjob,"current_year",
                             sprintf("%3.3d",$curr_year));
            ($thisjob) = shell_var_subs::var_replace($thisjob,"current_month",
                             sprintf("%2.2d",$curr_mon));
            ($thisjob) = shell_var_subs::var_replace($thisjob,"previous_year",
                             sprintf("%3.3d",$prev_year));
            ($thisjob) = shell_var_subs::var_replace($thisjob,"previous_month",
                             sprintf("%2.2d",$prev_mon));

            if ($split_thisjob) {
              # Insert multiple jobs with differences in only certain parameters
              # These are currently:
              # current_year, current_month, previous_year, previous_month
              my $mdelt = int($split_thisjob);
              my $year_a = $prev_year;
              my $mon_a  = $prev_mon;
              my ($mon_b, $year_b) = new_mon_year($mon_a, $year_a, $mdelt-1);
              my $delt_mon = diff_in_months($year_b,$mon_b,$curr_year,$curr_mon);
              if ($delt_mon < 0) {
                print "\nAttempting to split $jobname into $mdelt month intervals.\n";
                my $rmf = 1 + diff_in_months($prev_year,$prev_mon,$curr_year,$curr_mon);
                print "This is greater than the current interval of $rmf months.\n";
                die "Stopped";
              }
              while ( $delt_mon >= 0 ) {
                if ($main::Verbose > 3) {
                  print "\n$jobname   ";
                  print "year_a=$year_a mon_a=$mon_a ... year_b=$year_b mon_b=$mon_b\n";
                }
                ($thisjob) = shell_var_subs::var_replace($thisjob,"previous_year",
                             sprintf("%3.3d",$year_a));
                ($thisjob) = shell_var_subs::var_replace($thisjob,"previous_month",
                             sprintf("%2.2d",$mon_a));
                ($thisjob) = shell_var_subs::var_replace($thisjob,"current_year",
                             sprintf("%3.3d",$year_b));
                ($thisjob) = shell_var_subs::var_replace($thisjob,"current_month",
                             sprintf("%2.2d",$mon_b));
                $str .= $thisjob;
                # delt_mon=0 means mon_b,year_b is equal to curr_mon,curr_year
                last if $delt_mon == 0;
                ($mon_a, $year_a) = new_mon_year($mon_b, $year_b, 1);
                ($mon_b, $year_b) = new_mon_year($mon_a, $year_a, $mdelt-1);
                $delt_mon = diff_in_months($year_b,$mon_b,$curr_year,$curr_mon);
                if ($delt_mon < 0) {
                  # mon_b, year_b is later than curr_mon, curr_year
                  ($mon_b, $year_b) = ($curr_mon, $curr_year);
                  $delt_mon = 0;
                }
              }
            } else {
              $str .= $thisjob;
            }
          }

          # Append to the job string
          $job .= $str;
          if ($main::Verbose > $Vtrip) {
            print "=== JJA === year=$year  mon=$mon ================\n";
            print "$str";
          };
          if ($main::Verbose > $Vtrip) {
            print "***WARNING*** JJA: Failed to replace year\n" unless $rv1;
          };
        };
      };

      #==========  SON section --- insert at end of SON ==========
      if ( $mon == 11 ) {
        # copy the SON section template to a temporary string
        ($str) = @{$TEMPLATE{SON}};
        if ($str) {
          # split the current string into individual jobs
          # delimited by #end_of_job at the bottom
          my @splitjob = split_job_string($str);

          # perform section specific substitutions
          $str = '';
          foreach my $thisjob (@splitjob) {
            # Ignore empty job strings
            next if ($thisjob =~ /^\s*$/s);

            # get the name of the current job in jobname and
            # remove the job tag from the current job string
            my ($jobname) = read_jobtag($thisjob,{NAME_TAG=>1});

            # Extract info about splitting this job into multiple jobs
            my ($split_thisjob) = read_jobtag($thisjob,{SPLIT_TAG=>1});

	    # Get a hash ref to internal job specific options
            my $iopt_href = read_jobtag($thisjob,{IOPT_TAG=>1});

            # determine a year/month range, outside of which this job
            # should not be included in the final job string
            my $first_year_included = $run_start_year;
            my $first_mon_included  = $run_start_month;
            my $last_year_included  = $run_end_year;
            my $last_mon_included   = $run_end_month;
            $first_year_included = $iopt_href->{start_year} if $iopt_href->{start_year};
            $first_mon_included  = $iopt_href->{start_mon}  if $iopt_href->{start_mon};
            $last_year_included  = $iopt_href->{stop_year}  if $iopt_href->{stop_year};
            $last_mon_included   = $iopt_href->{stop_mon}   if $iopt_href->{stop_mon};

            my $mon_offset = 0;
            if ($jobname =~ /^\s*mload\s*$/) {
              $mon_offset = $mload_mon_offset;
            } elsif ($jobname =~ /^\s*mdump\s*$/) {
              $mon_offset = $mdump_mon_offset;
            } elsif ($jobname =~ /^\s*mdelete\s*$/) {
              $mon_offset = $mdelete_mon_offset;
            } elsif ($jobname =~ /^\s*load_list\s*$/) {
              $mon_offset = $load_list_mon_offset;
            } elsif ($jobname =~ /^\s*dump_list\s*$/) {
              $mon_offset = $dump_list_mon_offset;
            } elsif ($jobname =~ /^\s*del_list\s*$/) {
              $mon_offset = $del_list_mon_offset;
            } elsif ($jobname =~ /^\s*custom\d*\s*$/) {
              $mon_offset = $custom_mon_offset;
            } elsif ($jobname =~ /^\s*cloop\s*$/) {
              $mon_offset = $cloop_mon_offset;
            };

            my ($curr_mon, $curr_year) =
                new_mon_year($mon, $year, $mon_offset);
            my ($prev_mon, $prev_year) =
                new_mon_year($mon, $year, $mon_offset - 2);

            if ( $force_first_sea ) {
              # Reset the local variable first_mon_included to
              # indicate Sept so that the first SON of the
              # string will be included
              $first_mon_included = 9;
            }

            if ($main::Verbose > 10) {
              print "\nSON: first_year_included=$first_year_included";
              print "   first_mon_included=$first_mon_included\n";
              print "SON:  last_year_included=$last_year_included";
              print "    last_mon_included=$last_mon_included\n";
              print "SON: prev_year=$prev_year   prev_mon=$prev_mon";
              print "   curr_year=$curr_year   curr_mon=$curr_mon\n";
              while (my ($var,$val) = each %{$iopt_href}) {
                print "SON iopt: $var=$val\n";
              }
            }

            # Do not include this job if it would appear before
            # the start of the include range determined above
            next if ($prev_year < $first_year_included);
            if ($prev_year == $first_year_included) {
              next if ($prev_mon < $first_mon_included);
            }

            # Do not include this job if it would appear after
            # the end of the include range determined above
            next if ($curr_year > $last_year_included);
            if ($curr_year == $last_year_included) {
              next if ($curr_mon > $last_mon_included);
            }

            # Count the number of times this job gets used at this frequency
            my $job_id = "${jobname}_SON";
            if ( exists $JOB_COUNT{$job_id} ) {
              # Increment the count for this job at this frequency
              $JOB_COUNT{$job_id} += 1;
            } else {
              # Initialize the count
              $JOB_COUNT{$job_id} = 1;
            }
            my $curr_job_count = $JOB_COUNT{$job_id};

            # Add parallel job delimiters to specified jobs
            my $OPTref = {CURR_MON        => $curr_mon,
                          CURR_YEAR       => $curr_year,
                          PREV_MON        => $prev_mon,
                          PREV_YEAR       => $prev_year,
                          RUN_START_YEAR  => $run_start_year,
                          RUN_START_MONTH => $run_start_month,
                          RUN_END_YEAR    => $run_end_year,
                          RUN_END_MONTH   => $run_end_month};
            my $pre_print = '';
            my $post_print = '';
            ($thisjob, $pre_print, $post_print) =
               insert_ext_delim($thisjob,$jobname,$OPTref);

            if ($main::Verbose == 1) {
              my $jinfo = " ${jobname}:S";
              if ($split_thisjob) {$jinfo = " ${jobname}:S:${split_thisjob}m"}
              my $strng = $pre_print . $jinfo . $post_print;
              if ( $line_out{pfx} ) {
                $strng = $line_out{pfx} . $pre_print . $jinfo . $post_print;
                $line_out{pfx} = "";
              }
              $line_out{wrote} = 1;
              print "$strng";
            }

            # replace variable values in $thisjob
            ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,"mm01","m09");
            ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,"mm02","m10");
            ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,"mm03","m11");
            ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,"mn01","sep");
            ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,"mn02","oct");
            ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,"mn03","nov");
            ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,"ml01","   30");
            ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,"ml02","   31");
            ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,"ml03","   30");
            ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,"season","son");
            ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,"days","SON");
            ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,"obsday","SON");
            ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,"year",$year);
            ($thisjob,$rv2) = shell_var_subs::var_replace($thisjob,"mon", $mon);

            ($thisjob) = shell_var_subs::var_replace($thisjob,"current_year",
                             sprintf("%3.3d",$curr_year));
            ($thisjob) = shell_var_subs::var_replace($thisjob,"current_month",
                             sprintf("%2.2d",$curr_mon));
            ($thisjob) = shell_var_subs::var_replace($thisjob,"previous_year",
                             sprintf("%3.3d",$prev_year));
            ($thisjob) = shell_var_subs::var_replace($thisjob,"previous_month",
                             sprintf("%2.2d",$prev_mon));

            if ($split_thisjob) {
              # Insert multiple jobs with differences in only certain parameters
              # These are currently:
              # current_year, current_month, previous_year, previous_month
              my $mdelt = int($split_thisjob);
              my $year_a = $prev_year;
              my $mon_a  = $prev_mon;
              my ($mon_b, $year_b) = new_mon_year($mon_a, $year_a, $mdelt-1);
              my $delt_mon = diff_in_months($year_b,$mon_b,$curr_year,$curr_mon);
              if ($delt_mon < 0) {
                print "\nAttempting to split $jobname into $mdelt month intervals.\n";
                my $rmf = 1 + diff_in_months($prev_year,$prev_mon,$curr_year,$curr_mon);
                print "This is greater than the current interval of $rmf months.\n";
                die "Stopped";
              }
              while ( $delt_mon >= 0 ) {
                if ($main::Verbose > 3) {
                  print "\n$jobname   ";
                  print "year_a=$year_a mon_a=$mon_a ... year_b=$year_b mon_b=$mon_b\n";
                }
                ($thisjob) = shell_var_subs::var_replace($thisjob,"previous_year",
                             sprintf("%3.3d",$year_a));
                ($thisjob) = shell_var_subs::var_replace($thisjob,"previous_month",
                             sprintf("%2.2d",$mon_a));
                ($thisjob) = shell_var_subs::var_replace($thisjob,"current_year",
                             sprintf("%3.3d",$year_b));
                ($thisjob) = shell_var_subs::var_replace($thisjob,"current_month",
                             sprintf("%2.2d",$mon_b));
                $str .= $thisjob;
                # delt_mon=0 means mon_b,year_b is equal to curr_mon,curr_year
                last if $delt_mon == 0;
                ($mon_a, $year_a) = new_mon_year($mon_b, $year_b, 1);
                ($mon_b, $year_b) = new_mon_year($mon_a, $year_a, $mdelt-1);
                $delt_mon = diff_in_months($year_b,$mon_b,$curr_year,$curr_mon);
                if ($delt_mon < 0) {
                  # mon_b, year_b is later than curr_mon, curr_year
                  ($mon_b, $year_b) = ($curr_mon, $curr_year);
                  $delt_mon = 0;
                }
              }
            } else {
              $str .= $thisjob;
            }
          }

          # Append to the job string
          $job .= $str;
          if ($main::Verbose > $Vtrip) {
            print "=== SON === year=$year  mon=$mon ================\n";
            print "$str";
          };
          if ($main::Verbose > $Vtrip) {
            print "***WARNING*** SON: Failed to replace year\n" unless $rv1;
          };
        };
      };

      #========== multi monthly section --- insert at the end of every N months ==========
      foreach my $Nkey (@Nmonthskey) {
        my ($N,$curr_iopt) = $Nkey =~ /every(-?\d+)months(.*)/;
        next unless $N > 0;
        # insert at the end of every N months for positive values of N

        my $curr_iopt_href = str2ref_iopts( $curr_iopt, {ID=>"${N}m"} );
        $curr_iopt_href = {} unless $curr_iopt_href;

        my $mm_start_year = $run_start_year;
        my $mm_start_mon  = $run_start_month;
        my $mm_stop_year  = $run_end_year;
        my $mm_stop_mon   = $run_end_month;
        my $resize_last_chunk = 0;
        $mm_start_year = $curr_iopt_href->{start_year} if $curr_iopt_href->{start_year};
        $mm_start_mon  = $curr_iopt_href->{start_mon}  if $curr_iopt_href->{start_mon};
        $mm_stop_year  = $curr_iopt_href->{stop_year}  if $curr_iopt_href->{stop_year};
        $mm_stop_mon   = $curr_iopt_href->{stop_mon}   if $curr_iopt_href->{stop_mon};
        if ( $curr_iopt_href->{resize_last_chunk} ) {
          $resize_last_chunk = $curr_iopt_href->{resize_last_chunk}
        }

        my $mm_mon_offset = 0;
        $mm_mon_offset = $curr_iopt_href->{mon_offset} if $curr_iopt_href->{mon_offset};

        # determine the total number of months since the
        # start of the current multi month cycle
        my $months_this_cycle = $mm_mon_offset + 1 +
            diff_in_months($mm_start_year,$mm_start_mon,$next_year,$next_month);

        if ($main::Verbose > 10) {
          print "\n ====== ${N}mon: months_this_cycle=$months_this_cycle ";
          print "   next_year=$next_year   next_month=$next_month\n";
          print " ====== ${N}mon: months_this_cycle % $N = ",$months_this_cycle%$N,"\n";
          print " ====== ${N}mon: mm_mon_offset = $mm_mon_offset\n";
          print " ====== ${N}mon: next_year=$next_year   next_month=$next_month\n";
          print " ====== ${N}mon: mm_start_year=$mm_start_year";
          print "   mm_start_mon=$mm_start_mon\n";
          print " ====== ${N}mon:  mm_stop_year=$mm_stop_year";
          print "    mm_stop_mon=$mm_stop_mon\n";
          print " ====== ${N}mon: ",$curr_iopt ? "$curr_iopt" : "curr_iopt is not defined","\n";
          while (my ($var,$val) = each %{$curr_iopt_href}) {
            print " ====== ${N}mon mm_opt: $var=$val\n";
          }
        }

        if ($main::Verbose > 10) {
          my $count = 0;
          foreach my $jobstr ( @{$TEMPLATE{$Nkey}} ) {
            $count++;
            print " ====== $count $Nkey\n";
            unless ($jobstr) {
              print " ====== $count No job string for ${N}m\n";
            }
            foreach ( split_job_string($jobstr) ) {
              my ($jobname) = read_jobtag($_,{NAME_TAG=>1});
              print " ====== $count ${N}m   jobname: $jobname\n";
            }
          }
        }

        if ( $main::Verbose > 10 ) {
          print "\n====== ${N}mon: mm_mon_offset = $mm_mon_offset\n";
          print "====== ${N}mon: months_this_cycle=$months_this_cycle ";
          print "   next_year=$next_year   next_month=$next_month\n";
          print "====== ${N}mon: months_this_cycle % N ",$months_this_cycle % $N,"\n";
        }

        if ($curr_iopt_href->{full_first_interval}) {
          next if $months_this_cycle <= 0;
        } else {
          next if $months_this_cycle < 0;
        }

        if ( $next_year == $mm_stop_year and $next_month == $mm_stop_mon ) {
          # Special processing when the end year/month is encountered
          unless ( $resize_last_chunk ) {
            next unless ($months_this_cycle % $N == 0);
          }
        } else {
          # Unset resize_last_chunk unless this is actually the last chunk
          $resize_last_chunk = 0;
          next unless ($months_this_cycle % $N == 0);
        }

        # copy the monthly section template to a temporary string
        ($str) = @{$TEMPLATE{$Nkey}};

        if ($str) {
          # split the current string into individual jobs
          # delimited by #end_of_job at the bottom
          my @splitjob = split_job_string($str);

          # define current values for parameters to be substituted
          my $obsday = $MONname[$mon-1];
          my $monn = $obsday;
          $monn =~ tr/A-Z/a-z/;   # lower case month name

          # perform section specific substitutions
          $str = '';
          foreach my $thisjob (@splitjob) {
            # Ignore empty job strings
            next if ($thisjob =~ /^\s*$/s);

            # get the name of the current job in jobname and
            # remove the job tag from the current job string
            my ($jobname) = read_jobtag($thisjob,{NAME_TAG=>1});

            # Determine the number of times this job has been used up to this point
            my $job_id = "${jobname}_${N}m";
            my $prev_job_count = 0;
            if ( exists $JOB_COUNT{$job_id} ) {
              # Set the count for this job up to this point
              # $JOB_COUNT{$job_id} will be incremented below if this job is used
              $prev_job_count = $JOB_COUNT{$job_id};
            }

            if ($main::Verbose > 10) {
              print "\n ====== ${N}m  jobname = $jobname  prev_job_count = $prev_job_count\n";
            }

            # Extract info about splitting this job into multiple jobs
            my ($split_thisjob) = read_jobtag($thisjob,{SPLIT_TAG=>1});

	    # Get a hash ref to internal job specific options
            my $iopt_href = read_jobtag($thisjob,{IOPT_TAG=>1});

            # determine a year/month range, outside of which this job
            # should not be included in the final job string
            my $first_year_included = $run_start_year;
            my $first_mon_included  = $run_start_month;
            my $last_year_included  = $run_end_year;
            my $last_mon_included   = $run_end_month;
            my $insert_at_start = 0;
            my $insert_at_end = 0;
            $first_year_included = $iopt_href->{start_year} if $iopt_href->{start_year};
            $first_mon_included  = $iopt_href->{start_mon}  if $iopt_href->{start_mon};
            $last_year_included  = $iopt_href->{stop_year}  if $iopt_href->{stop_year};
            $last_mon_included   = $iopt_href->{stop_mon}   if $iopt_href->{stop_mon};
            $insert_at_start = $iopt_href->{insert_at_start} if $iopt_href->{insert_at_start};
            $insert_at_end   = $iopt_href->{insert_at_end}   if $iopt_href->{insert_at_end};

            if ($N%$months != 0 ) {
              warn "\n*** WARNING *** ";
              warn "  The \"every $N months\" increment found in ${jobname}:${N}m is not a multiple of months=$months.\n\n";
            }

            # current_(year|month) and previous_(year|month) are used by the modules
            # mdump, mdelete and mload. These modules generate file names based on
            # these values. Since mdump and mdelete are typically run after a model
            # run of $months months we set current_(year|month) to correspond to the
            # end of the model run and previous_(year|month) to correspond to the
            # start of the model run.

#xxx            my $mon_offset = 0;
#xxx            if ($jobname =~ /^\s*mload\s*$/) {
#xxx              $mon_offset = $mload_mon_offset;
#xxx            } elsif ($jobname =~ /^\s*mdump\s*$/) {
#xxx              $mon_offset = $mdump_mon_offset;
#xxx            } elsif ($jobname =~ /^\s*mdelete\s*$/) {
#xxx              $mon_offset = $mdelete_mon_offset;
#xxx            } elsif ($jobname =~ /^\s*load_list\s*$/) {
#xxx              $mon_offset = $load_list_mon_offset;
#xxx            } elsif ($jobname =~ /^\s*dump_list\s*$/) {
#xxx              $mon_offset = $dump_list_mon_offset;
#xxx            } elsif ($jobname =~ /^\s*del_list\s*$/) {
#xxx              $mon_offset = $del_list_mon_offset;
#xxx            } elsif ($jobname =~ /^\s*custom\d*\s*$/) {
#xxx              $mon_offset = $custom_mon_offset;
#xxx            } elsif ($jobname =~ /^\s*cloop\s*$/) {
#xxx              $mon_offset = $cloop_mon_offset;
#xxx            };

            my ($curr_mon, $curr_year) = ($next_month, $next_year);
            my ($prev_mon, $prev_year) =
                new_mon_year($curr_mon, $curr_year, 1 - $N);
            if ( $resize_last_chunk ) {
              # resize_last_chunk is only set when this is the last chunk before
              # the end of the run and the user has requested that the last chunk
              # be made to fit regardless of the number of months left in the run
              # In this case we need to use the prevmonth and prevyear hashes to
              # determine where that penultimite chunk stopped. This could be
              # different from $N months before next_(year/month)
              ($prev_mon, $prev_year) = ($prevmonth{$Nkey}, $prevyear{$Nkey});
            }
            if ( $main::Verbose > 10 ) {
              print "\n$jobname ${N}mon: prev_year=$prev_year   prev_mon=$prev_mon";
              print "   curr_year=$curr_year  curr_mon=$curr_mon";
              print "   next_year=$next_year  next_month=$next_month  mm_mon_offset=$mm_mon_offset";
            }

            # Check that prev_(year/mon) is not before the start of the run
            my $dmon = diff_in_months($run_start_year,$run_start_month,
                                           $prev_year,     $prev_mon);

	    if ($dmon < 0 and ($iopt_href->{full_first_interval} or $mm_mon_offset > 0)) {
              # If prev_(year/mon) is a date before the start of the run
              # and the user has requested that the first interval contains
              # the entire $N months
              # then reset prev_(year/mon) to the run start date
              $prev_year = $run_start_year;
              $prev_mon  = $run_start_month;
	    }

	    if ($dmon < 0 and $insert_at_start) {
              # If prev_(year/mon) is a date before the start of the run
              # and the first interval is always included then
              # reset prev_(year/mon) to run_start_(year/mon)
              $prev_year = $run_start_year;
              $prev_mon  = $run_start_month;
            }

            if ( $mm_mon_offset > 0  ) {

              # my $dmon = diff_in_months($curr_year,    $curr_mon,
              #                           $run_end_year, $run_end_month);

              # Determine the difference in months between the start of the current
              # insert interval and the end of the run
              # If this difference is less than the length of a single insert
              # interval then reset the curr_year and curr_mon to coinside with
              # the end of the run
              my $dmon = diff_in_months($prev_year,    $prev_mon,
                                        $run_end_year, $run_end_month);
              if ( $dmon < $N ) {
                # If curr_(year/mon) is a date after the end of the run
                # then reset curr_(year/mon) to the run end date
                $curr_year = $run_end_year;
                $curr_mon  = $run_end_month;
              }
            }

            if ( $main::Verbose > 10 ) {
              print "\n$jobname ${N}mon: first_year_included=$first_year_included";
              print "   first_mon_included=$first_mon_included\n";
              print "$jobname ${N}mon:  last_year_included=$last_year_included";
              print "    last_mon_included=$last_mon_included\n";
              print "$jobname ${N}mon: prev_year=$prev_year   prev_mon=$prev_mon";
              print "   curr_year=$curr_year   curr_mon=$curr_mon\n";
              while (my ($var,$val) = each %{$iopt_href}) {
                print "$jobname ${N}mon iopt: $var=$val\n";
              }
            }

            # Do not include this job if it would appear before
            # the start of the include range determined above
            next if ($prev_year < $first_year_included);
            if ($prev_year == $first_year_included) {
              next if ($prev_mon < $first_mon_included);
            }

            unless ($insert_at_end) {
              # Do not include this job if it would appear after
              # the end of the include range determined above
              next if ($curr_year > $last_year_included);
              if ($curr_year == $last_year_included) {
                next if ($curr_mon > $last_mon_included);
              }
            }

            if ( $mm_mon_offset < 0 ) {
              if ( $prev_job_count == 0 ) {
                # Reset prev_(year/mon) to the run start date
                # for the first job in the string
                $prev_year = $run_start_year;
                $prev_mon  = $run_start_month;
              }
            }

            if ($jobname =~ /^\s*pool_ann\s*$/) {
              # Do not insert the annual pooling job unless there are a
              # full 12 months in the pooling interval prior to this point
              # (ie the first year of pooling does not start from JAN)
              my $pool_start_year = $run_start_year;
              my $pool_start_mon  = $run_start_month;
              my $pool_stop_year  = $run_end_year;
              my $pool_stop_mon   = $run_end_month;
              $pool_start_year = $iopt_href->{pool_start_year} if $iopt_href->{pool_start_year};
              $pool_start_mon  = $iopt_href->{pool_start_mon}  if $iopt_href->{pool_start_mon};
              $pool_stop_year  = $iopt_href->{pool_stop_year}  if $iopt_href->{pool_stop_year};
              $pool_stop_mon   = $iopt_href->{pool_stop_mon}   if $iopt_href->{pool_stop_mon};
              next if (($curr_year == $pool_start_year) and ($pool_start_mon >  1));
              next if (($curr_year == $pool_stop_year)  and ($pool_stop_mon  < 12));
            }

            if ($jobname =~ /^\s*pdiag/ and $prev_job_count == 0) {
              ($prev_mon, $prev_year) = ($run_start_month, $run_start_year)
            }

            # Count the number of times this job gets used at this frequency
            if ( exists $JOB_COUNT{$job_id} ) {
              # Increment the count for this job at this frequency
              $JOB_COUNT{$job_id} += 1;
            } else {
              # Initialize the count
              $JOB_COUNT{$job_id} = 1;
            }
            my $curr_job_count = $JOB_COUNT{$job_id};

            if ($main::Verbose > 10) {
              print "\n ====== ${N}m  jobname = $jobname  curr_job_count = $curr_job_count\n";
            }

            # Add parallel job delimiters to specified jobs
            my $OPTref = {CURR_MON        => $curr_mon,
                          CURR_YEAR       => $curr_year,
                          PREV_MON        => $prev_mon,
                          PREV_YEAR       => $prev_year,
                          RUN_START_YEAR  => $run_start_year,
                          RUN_START_MONTH => $run_start_month,
                          RUN_END_YEAR    => $run_end_year,
                          RUN_END_MONTH   => $run_end_month};
            my $pre_print = '';
            my $post_print = '';
            ($thisjob, $pre_print, $post_print) =
                insert_ext_delim($thisjob,$jobname,$OPTref);

            if ($main::Verbose == 1) {
              my $jinfo = " ${jobname}:${N}m";
              if ($split_thisjob) {$jinfo = " ${jobname}:${N}m:${split_thisjob}m"}
              my $strng = $pre_print . $jinfo . $post_print;
              if ( $line_out{pfx} ) {
                $strng = $line_out{pfx} . $pre_print . $jinfo . $post_print;
                $line_out{pfx} = "";
              }
              $line_out{wrote} = 1;
              print "$strng";
            }

            ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,"monn",   $monn);
            ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,"year",   $next_year);
            ($thisjob,$rv2) = shell_var_subs::var_replace($thisjob,"mon",    $next_month);
            ($thisjob,$rv3) = shell_var_subs::var_replace($thisjob,"days",   $obsday);
            ($thisjob,$rv4) = shell_var_subs::var_replace($thisjob,"obsday", $obsday);
            ($thisjob,$rv5) = shell_var_subs::var_replace($thisjob,"kfinal", $kfinal);
            ($thisjob,$rv6) = shell_var_subs::var_replace($thisjob,"year_restart",
                                  $year_restart);
            ($thisjob,$rv7) = shell_var_subs::var_replace($thisjob,"mon_restart",
                                  $mon_restart);
            ($thisjob,$rv8) = shell_var_subs::var_replace($thisjob,"samerun",$samerun);
            ($thisjob,$rv9) = shell_var_subs::var_replace($thisjob,"initsp", $initsp);
            ($thisjob,$rv10)= shell_var_subs::var_replace($thisjob,"cpl_new_rtd", $cpl_new_rtd);
            ($thisjob,$rv11)= shell_var_subs::var_replace($thisjob,"nemo_new_rtd",$nemo_new_rtd);
            if ($first_loop_iter and $main::Restart_fname) {
              # If a restart file name is available then set start in the string
              my $start = $main::Restart_fname;
              # strip any three digit numerical extension as well as a trailing "rs"
              # A gcm model job will use ${start}rs as the restart file name
              $start  =~ s/(.*)[cor]s(\.[0-9]{3})?$/$1/;
              # print "\nstart = $start  Restart_fname = $main::Restart_fname\n";
              ($thisjob) = shell_var_subs::var_replace($thisjob,"start",$start);
            };

#            if ($curr_job_count == 1 and $jobname eq "xnemo") {
            if ( $curr_job_count == 1 ) {
              if (exists $main::VARS{nemo_restart}) {
                # If a nemo restart file name is available then
                # set it only on the first job of the string
                my $nemo_restart = $main::VARS{nemo_restart}{value};
                # Remove nemo_restart from the VARS hash so that it does not
                # get set in any other jobs in this string
                delete $main::VARS{nemo_restart};
                if ( $nemo_restart ) {
                  # Redefine nemo_restart only when it has a non null value
                  # strip any three digit numerical extension
                  $nemo_restart  =~ s/(.*)(\.[0-9]{3})?$/$1/;
                  ($thisjob) = shell_var_subs::var_replace($thisjob,"nemo_restart",$nemo_restart);
                }
              }
              if (exists $main::VARS{nemo_exec}) {
                # If a nemo_exec is set by the user then
                # set it only on the first job of the string
                my $nemo_exec = $main::VARS{nemo_exec}{value};
                # Remove nemo_exec from the VARS hash so that it does not
                # get set in any other jobs in this string
                delete $main::VARS{nemo_exec};
                if ( $nemo_exec ) {
                  # Redefine nemo_exec only when it has a non null value
                  ($thisjob) = shell_var_subs::var_replace($thisjob,"nemo_exec",$nemo_exec);
                }
              }
              if (exists $main::VARS{nemo_nn_it000}) {
                # If nemo_nn_it000 is set by the user then
                # set it only on the first job of the string
                my $nemo_nn_it000 = $main::VARS{nemo_nn_it000}{value};
                # Remove nemo_nn_it000 from the VARS hash so that it does not
                # get set in any other jobs in this string
                delete $main::VARS{nemo_nn_it000};
                if ( $nemo_nn_it000 ) {
                  # Redefine nemo_nn_it000 only when it has a non null value
                  ($thisjob) = shell_var_subs::var_replace($thisjob,"nemo_nn_it000",$nemo_nn_it000);
		}
              }
              if (exists $main::VARS{nemo_from_rest}) {
                # If nemo_from_rest is set by the user then
                # set it only on the first job of the string
                my $nemo_from_rest = $main::VARS{nemo_from_rest}{value};
                # Remove nemo_from_rest from the VARS hash so that it does not
                # get set in any other jobs in this string
                delete $main::VARS{nemo_from_rest};
                # Redefine nemo_from_rest in the current job only
                ($thisjob) = shell_var_subs::var_replace($thisjob,"nemo_from_rest",$nemo_from_rest);
              }
              if (exists $main::VARS{nemo_force_namelist}) {
                # If nemo_force_namelist is set by the user then
                # set it only on the first job of the string
                my $nemo_force_namelist = $main::VARS{nemo_force_namelist}{value};
                # Remove nemo_force_namelist from the VARS hash so that it does not
                # get set in any other jobs in this string
                delete $main::VARS{nemo_force_namelist};
                # Redefine nemo_force_namelist in the current job only
                ($thisjob) = shell_var_subs::var_replace($thisjob,"nemo_force_namelist",$nemo_force_namelist);
              }
	    }

            ($thisjob) = shell_var_subs::var_replace($thisjob,"current_year",
                              sprintf("%3.3d",$curr_year));
            ($thisjob) = shell_var_subs::var_replace($thisjob,"current_month",
                              sprintf("%2.2d",$curr_mon));
            ($thisjob) = shell_var_subs::var_replace($thisjob,"previous_year",
                               sprintf("%3.3d",$prev_year));
            ($thisjob) = shell_var_subs::var_replace($thisjob,"previous_month",
                               sprintf("%2.2d",$prev_mon));

            if ($split_thisjob) {
              # Insert multiple jobs with differences in only certain parameters
              # These are currently:
              # current_year, current_month, previous_year, previous_month
              my $mdelt = int($split_thisjob);
              my $year_a = $prev_year;
              my $mon_a  = $prev_mon;
              my ($mon_b, $year_b) = new_mon_year($mon_a, $year_a, $mdelt-1);
              my $delt_mon = diff_in_months($year_b,$mon_b,$curr_year,$curr_mon);
              if ($delt_mon < 0) {
                print "\nAttempting to split $jobname into $mdelt month intervals.\n";
                my $rmf = 1 + diff_in_months($prev_year,$prev_mon,$curr_year,$curr_mon);
                print "This is greater than the current interval of $rmf months.\n";
                die "Stopped";
              }
              while ( $delt_mon >= 0 ) {
                if ($main::Verbose > 3) {
                  print "\n$jobname   ";
                  print "year_a=$year_a mon_a=$mon_a ... year_b=$year_b mon_b=$mon_b\n";
                }
                ($thisjob) = shell_var_subs::var_replace($thisjob,"previous_year",
                             sprintf("%3.3d",$year_a));
                ($thisjob) = shell_var_subs::var_replace($thisjob,"previous_month",
                             sprintf("%2.2d",$mon_a));
                ($thisjob) = shell_var_subs::var_replace($thisjob,"current_year",
                             sprintf("%3.3d",$year_b));
                ($thisjob) = shell_var_subs::var_replace($thisjob,"current_month",
                             sprintf("%2.2d",$mon_b));
                $str .= $thisjob;
                # delt_mon=0 means mon_b,year_b is equal to curr_mon,curr_year
                last if $delt_mon == 0;
                ($mon_a, $year_a) = new_mon_year($mon_b, $year_b, 1);
                ($mon_b, $year_b) = new_mon_year($mon_a, $year_a, $mdelt-1);
                $delt_mon = diff_in_months($year_b,$mon_b,$curr_year,$curr_mon);
                if ($delt_mon < 0) {
                  # mon_b, year_b is later than curr_mon, curr_year
                  ($mon_b, $year_b) = ($curr_mon, $curr_year);
                  $delt_mon = 0;
                }
              }
            } else {
              $str .= $thisjob;
            }
          }

          # set previous_(year/month) to the last time the section was inserted
          $prevmonth{$Nkey} = sprintf("%2.2d",$next_month + 1);
          if ($prevmonth{$Nkey} > 12) {
            $prevmonth{$Nkey} = sprintf("%2.2d",1);
            $prevyear{$Nkey} = sprintf("%3.3d",$next_year + 1);
          } else {
            $prevyear{$Nkey} = $next_year;
          }

          # Append to the job string
          $job .= $str;
        }
      }

      #========== yearly section --- insert at end of year ==========
      if ( $next_month == 12 ) {
        # copy the yearly section template to a temporary string
        ($str) = @{$TEMPLATE{yearly}};
        if ($str) {
          # split the current string into individual jobs
          # delimited by #end_of_job at the bottom
          my @splitjob = split_job_string($str);

          # perform section specific substitutions
          $str = '';
          foreach my $thisjob (@splitjob) {
            # Ignore empty job strings
            next if ($thisjob =~ /^\s*$/s);

            # get the name of the current job in jobname and
            # remove the job tag from the current job string
            my ($jobname) = read_jobtag($thisjob,{NAME_TAG=>1});

            # Extract info about splitting this job into multiple jobs
            my ($split_thisjob) = read_jobtag($thisjob,{SPLIT_TAG=>1});

	    # Get a hash ref to internal job specific options
            my $iopt_href = read_jobtag($thisjob,{IOPT_TAG=>1});

            # determine a year/month range, outside of which this job
            # should not be included in the final job string
            my $first_year_included = $run_start_year;
            my $first_mon_included  = $run_start_month;
            my $last_year_included  = $run_end_year;
            my $last_mon_included   = $run_end_month;
            $first_year_included = $iopt_href->{start_year} if $iopt_href->{start_year};
            $first_mon_included  = $iopt_href->{start_mon}  if $iopt_href->{start_mon};
            $last_year_included  = $iopt_href->{stop_year}  if $iopt_href->{stop_year};
            $last_mon_included   = $iopt_href->{stop_mon}   if $iopt_href->{stop_mon};

            if ($main::Verbose > 10) {
              print "\nmon=12: first_year_included=$first_year_included";
              print "   first_mon_included=$first_mon_included\n";
              print "mon=12: last_year_included=$last_year_included";
              print "   last_mon_included=$last_mon_included\n";
              while (my ($var,$val) = each %{$iopt_href}) {
                print "mon=12 iopt: $var=$val\n";
              }
            }

            # current_(year|month) and previous_(year|month) are used by the modules
            # mdump, mdelete and mload. These modules generate file names based on
            # these values. Since mdump and mdelete are typically run after a model
            # run of $months months we set current_(year|month) to correspond to the
            # end of the model run and previous_(year|month) to correspond to the
            # start of the model run.

            my $mon_offset = 0;
            if ($jobname =~ /^\s*mload\s*$/) {
              $mon_offset = $mload_mon_offset;
            } elsif ($jobname =~ /^\s*mdump\s*$/) {
              $mon_offset = $mdump_mon_offset;
            } elsif ($jobname =~ /^\s*mdelete\s*$/) {
              $mon_offset = $mdelete_mon_offset;
            } elsif ($jobname =~ /^\s*load_list\s*$/) {
              $mon_offset = $load_list_mon_offset;
            } elsif ($jobname =~ /^\s*dump_list\s*$/) {
              $mon_offset = $dump_list_mon_offset;
            } elsif ($jobname =~ /^\s*del_list\s*$/) {
              $mon_offset = $del_list_mon_offset;
            } elsif ($jobname =~ /^\s*custom\d*\s*$/) {
              $mon_offset = $custom_mon_offset;
            } elsif ($jobname =~ /^\s*cloop\s*$/) {
              $mon_offset = $cloop_mon_offset;
            };

            my ($curr_mon, $curr_year) =
                new_mon_year($next_month, $next_year, $mon_offset);
            my ($prev_mon, $prev_year) =
                new_mon_year($yearly{previous_month},$yearly{previous_year},$mon_offset);

            # Do not include this job if it would appear before
            # the start of the include range determined above
            next if ($curr_year < $first_year_included);
            if ($curr_year == $first_year_included) {
              next if ($curr_mon < $first_mon_included);
            }

            # Do not include this job if it would appear after
            # the end of the include range determined above
            next if ($curr_year > $last_year_included);
            if ($curr_year == $last_year_included) {
              next if ($curr_mon > $last_mon_included);
            }

            if ($jobname =~ /^\s*pool_ann\s*$/) {
              # Do not insert the annual pooling job unless there are a
              # full 12 months in the pooling interval prior to this point
              # (ie the first year does not start from JAN)
              next if (($curr_year == $first_year_included) and ($first_mon_included >  1));
              next if (($curr_year == $last_year_included)  and ($last_mon_included  < 12));
            }

            # Count the number of times this job gets used at this frequency
            my $job_id = "${jobname}_y";
            if ( exists $JOB_COUNT{$job_id} ) {
              # Increment the count for this job at this frequency
              $JOB_COUNT{$job_id} += 1;
            } else {
              # Initialize the count
              $JOB_COUNT{$job_id} = 1;
            }
            my $curr_job_count = $JOB_COUNT{$job_id};

            # Add parallel job delimiters to specified jobs
            my $OPTref = {CURR_MON        => $curr_mon,
                          CURR_YEAR       => $curr_year,
                          PREV_MON        => $prev_mon,
                          PREV_YEAR       => $prev_year,
                          RUN_START_YEAR  => $run_start_year,
                          RUN_START_MONTH => $run_start_month,
                          RUN_END_YEAR    => $run_end_year,
                          RUN_END_MONTH   => $run_end_month};
            my $pre_print = '';
            my $post_print = '';
            ($thisjob, $pre_print, $post_print) =
               insert_ext_delim($thisjob,$jobname,$OPTref);

            if ($main::Verbose == 1) {
              my $jinfo = " ${jobname}:y";
              if ($split_thisjob) {$jinfo = " ${jobname}:y:${split_thisjob}m"}
              my $strng = $pre_print . $jinfo . $post_print;
              if ( $line_out{pfx} ) {
                $strng = $line_out{pfx} . $pre_print . $jinfo . $post_print;
                $line_out{pfx} = "";
              }
              $line_out{wrote} = 1;
              print "$strng";
            }

            # replace variable values in $thisjob
            # next_year and next_month are used since this should be inserted at
            # the end of $months months after current $year,$mon counter values
            ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,
                                                "year",$next_year);
            ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,
                                                "mon",$next_month);

            ($thisjob,$rv2) = shell_var_subs::var_replace($thisjob,"current_year",
                                  sprintf("%3.3d",$curr_year));
            ($thisjob,$rv3) = shell_var_subs::var_replace($thisjob,"current_month",
                                  sprintf("%2.2d",$curr_mon));
            ($thisjob,$rv3) = shell_var_subs::var_replace($thisjob,"previous_year",
                                  sprintf("%3.3d",$prev_year));
            ($thisjob,$rv3) = shell_var_subs::var_replace($thisjob,"previous_month",
                                  sprintf("%2.2d",$prev_mon));

#            if ($curr_job_count == 1 and $jobname eq "xnemo") {
            if ( $curr_job_count == 1 ) {
              if (exists $main::VARS{nemo_restart}) {
                # If a nemo restart file name is available then
                # set it only on the first job of the string
                my $nemo_restart = $main::VARS{nemo_restart}{value};
                # Remove nemo_restart from the VARS hash so that it does not
                # get set in any other jobs in this string
                delete $main::VARS{nemo_restart};
                if ( $nemo_restart ) {
                  # Redefine nemo_restart only when it has a non null value
                  # strip any three digit numerical extension
                  $nemo_restart  =~ s/(.*)(\.[0-9]{3})?$/$1/;
                  ($thisjob) = shell_var_subs::var_replace($thisjob,"nemo_restart",$nemo_restart);
                }
              }
              if (exists $main::VARS{nemo_exec}) {
                # If a nemo_exec is set by the user then
                # set it only on the first job of the string
                my $nemo_exec = $main::VARS{nemo_exec}{value};
                # Remove nemo_exec from the VARS hash so that it does not
                # get set in any other jobs in this string
                delete $main::VARS{nemo_exec};
                if ( $nemo_exec ) {
                  # Redefine nemo_exec only when it has a non null value
                  ($thisjob) = shell_var_subs::var_replace($thisjob,"nemo_exec",$nemo_exec);
                }
              }
              if (exists $main::VARS{nemo_nn_it000}) {
                # If nemo_nn_it000 is set by the user then
                # set it only on the first job of the string
                my $nemo_nn_it000 = $main::VARS{nemo_nn_it000}{value};
                # Remove nemo_nn_it000 from the VARS hash so that it does not
                # get set in any other jobs in this string
                delete $main::VARS{nemo_nn_it000};
                if ( $nemo_nn_it000 ) {
                  # Redefine nemo_nn_it000 only when it has a non null value
                  ($thisjob) = shell_var_subs::var_replace($thisjob,"nemo_nn_it000",$nemo_nn_it000);
		}
              }
              if (exists $main::VARS{nemo_from_rest}) {
                # If nemo_from_rest is set by the user then
                # set it only on the first job of the string
                my $nemo_from_rest = $main::VARS{nemo_from_rest}{value};
                # Remove nemo_from_rest from the VARS hash so that it does not
                # get set in any other jobs in this string
                delete $main::VARS{nemo_from_rest};
                # Redefine nemo_from_rest in the current job only
                ($thisjob) = shell_var_subs::var_replace($thisjob,"nemo_from_rest",$nemo_from_rest);
              }
              if (exists $main::VARS{nemo_force_namelist}) {
                # If nemo_force_namelist is set by the user then
                # set it only on the first job of the string
                my $nemo_force_namelist = $main::VARS{nemo_force_namelist}{value};
                # Remove nemo_force_namelist from the VARS hash so that it does not
                # get set in any other jobs in this string
                delete $main::VARS{nemo_force_namelist};
                # Redefine nemo_force_namelist in the current job only
                ($thisjob) = shell_var_subs::var_replace($thisjob,"nemo_force_namelist",$nemo_force_namelist);
              }
	    }

            if ($split_thisjob) {
              # Insert multiple jobs with differences in only certain parameters
              # These are currently:
              # current_year, current_month, previous_year, previous_month
              my $mdelt = int($split_thisjob);
              my $year_a = $prev_year;
              my $mon_a  = $prev_mon;
              my ($mon_b, $year_b) = new_mon_year($mon_a, $year_a, $mdelt-1);
              my $delt_mon = diff_in_months($year_b,$mon_b,$curr_year,$curr_mon);
              if ($delt_mon < 0) {
                print "\nAttempting to split $jobname into $mdelt month intervals.\n";
                my $rmf = 1 + diff_in_months($prev_year,$prev_mon,$curr_year,$curr_mon);
                print "This is greater than the current interval of $rmf months.\n";
                die "Stopped";
              }
              while ( $delt_mon >= 0 ) {
                if ($main::Verbose > 3) {
                  print "\n$jobname   ";
                  print "year_a=$year_a mon_a=$mon_a ... year_b=$year_b mon_b=$mon_b\n";
                }
                ($thisjob) = shell_var_subs::var_replace($thisjob,"previous_year",
                             sprintf("%3.3d",$year_a));
                ($thisjob) = shell_var_subs::var_replace($thisjob,"previous_month",
                             sprintf("%2.2d",$mon_a));
                ($thisjob) = shell_var_subs::var_replace($thisjob,"current_year",
                             sprintf("%3.3d",$year_b));
                ($thisjob) = shell_var_subs::var_replace($thisjob,"current_month",
                             sprintf("%2.2d",$mon_b));
                $str .= $thisjob;
                # delt_mon=0 means mon_b,year_b is equal to curr_mon,curr_year
                last if $delt_mon == 0;
                ($mon_a, $year_a) = new_mon_year($mon_b, $year_b, 1);
                ($mon_b, $year_b) = new_mon_year($mon_a, $year_a, $mdelt-1);
                $delt_mon = diff_in_months($year_b,$mon_b,$curr_year,$curr_mon);
                if ($delt_mon < 0) {
                  # mon_b, year_b is later than curr_mon, curr_year
                  ($mon_b, $year_b) = ($curr_mon, $curr_year);
                  $delt_mon = 0;
                }
              }
            } else {
              $str .= $thisjob;
            }
          }

          # set previous_(year/month) to the month after the the last time
          # the section was inserted
          $yearly{previous_month} = sprintf("%2.2d",$next_month + 1);
          if ($yearly{previous_month} > 12) {
            $yearly{previous_month} = sprintf("%2.2d",1);
            $yearly{previous_year} = sprintf("%3.3d",$next_year + 1);
          } else {
            $yearly{previous_year} = $next_year;
          }

          # Append to the job string
          $job .= $str;

          if ($main::Verbose > $Vtrip) {
            print "=== yearly === year=$year  mon=$mon ================\n";
            print "$str";
          };
          if ($main::Verbose > $Vtrip) {
            print "***WARNING*** yearly: Failed to replace year\n" unless $rv1;
          };
        };
      };

      #========== multi year section --- insert at end of every N years ==========
      if ( $next_month == 12 ) {
        foreach my $Nkey (@Nyearskey) {
          my ($N,$curr_iopt) = $Nkey =~ /every(-?\d+)years(.*)/;
          # insert at the end of every N years for positive values of N
          next unless $N > 0;

          my $curr_iopt_href = str2ref_iopts( $curr_iopt, {ID=>"${N}y"} );
          $curr_iopt_href = {} unless $curr_iopt_href;

          my $my_start_year = $run_start_year;
          my $my_start_mon  = $run_start_month;
          my $my_stop_year  = $run_end_year;
          my $my_stop_mon   = $run_end_month;
          $my_start_year = $curr_iopt_href->{start_year} if $curr_iopt_href->{start_year};
          $my_start_mon  = $curr_iopt_href->{start_mon}  if $curr_iopt_href->{start_mon};
          $my_stop_year  = $curr_iopt_href->{stop_year}  if $curr_iopt_href->{stop_year};
          $my_stop_mon   = $curr_iopt_href->{stop_mon}   if $curr_iopt_href->{stop_mon};

          my $my_mon_offset = 0;
          $my_mon_offset = $curr_iopt_href->{mon_offset} if $curr_iopt_href->{mon_offset};

          my $first_ychunk = 0;
          $first_ychunk = $curr_iopt_href->{first_ychunk} if $curr_iopt_href->{first_ychunk};

          my $addnow = $next_year - $my_start_year + 1;
          next unless ($addnow%$N == 0);

          # copy the multi year section template to a temporary string
          ($str) = @{$TEMPLATE{$Nkey}};
          if ($str) {
            # split the current string into individual jobs
            # delimited by #end_of_job at the bottom
            my @splitjob = split_job_string($str);

            # perform section specific substitutions
            $str = '';
            foreach my $thisjob (@splitjob) {
              # Ignore empty job strings
              next if ($thisjob =~ /^\s*$/s);

              # get the name of the current job in jobname and
              # remove the job tag from the current job string
              my ($jobname) = read_jobtag($thisjob,{NAME_TAG=>1});

              # Extract info about splitting this job into multiple jobs
              my ($split_thisjob) = read_jobtag($thisjob,{SPLIT_TAG=>1});

	      # Get a hash ref to internal job specific options
              my $iopt_href = read_jobtag($thisjob,{IOPT_TAG=>1});

              # determine a year/month range, outside of which this job
              # should not be included in the final job string
              my $first_year_included = $run_start_year;
              my $first_mon_included  = $run_start_month;
              my $last_year_included  = $run_end_year;
              my $last_mon_included   = $run_end_month;
              $first_year_included = $iopt_href->{start_year} if $iopt_href->{start_year};
              $first_mon_included  = $iopt_href->{start_mon}  if $iopt_href->{start_mon};
              $last_year_included  = $iopt_href->{stop_year}  if $iopt_href->{stop_year};
              $last_mon_included   = $iopt_href->{stop_mon}   if $iopt_href->{stop_mon};

              # current_(year|month) and previous_(year|month) are used by the modules
              # mdump, mdelete and mload. These modules generate file names based on
              # these values. Since mdump and mdelete are typically run after a model
              # run of $months months we set current_(year|month) to correspond to the
              # end of the model run and previous_(year|month) to correspond to the
              # start of the model run.

              my $mon_offset = 0;
              if ($jobname =~ /^\s*mload\s*$/) {
                $mon_offset = $mload_mon_offset;
              } elsif ($jobname =~ /^\s*mdump\s*$/) {
                $mon_offset = $mdump_mon_offset;
              } elsif ($jobname =~ /^\s*mdelete\s*$/) {
                $mon_offset = $mdelete_mon_offset;
              } elsif ($jobname =~ /^\s*load_list\s*$/) {
                $mon_offset = $load_list_mon_offset;
              } elsif ($jobname =~ /^\s*dump_list\s*$/) {
                $mon_offset = $dump_list_mon_offset;
              } elsif ($jobname =~ /^\s*del_list\s*$/) {
                $mon_offset = $del_list_mon_offset;
              } elsif ($jobname =~ /^\s*custom\d*\s*$/) {
                $mon_offset = $custom_mon_offset;
              } elsif ($jobname =~ /^\s*cloop\s*$/) {
                $mon_offset = $cloop_mon_offset;
              };

              my ($curr_mon, $curr_year) =
                  new_mon_year($next_month, $next_year, $mon_offset);
              my ($prev_mon, $prev_year) =
                  new_mon_year($curr_mon, $curr_year, 1 - 12*$N);

              # Check that prev_(year/mon) is not before the start of the run
              my $dmon = diff_in_months($run_start_year,$run_start_month,
                                             $prev_year,     $prev_mon);
              if ($dmon < 0) {
                # If prev_(year/mon) is a date before the start of the run
                # then reset prev_(year/mon) to the run start date
                $prev_year = $run_start_year;
                $prev_mon  = $run_start_month;
              }

              if ($main::Verbose > 10) {
                print "\n${N}year: first_year_included=$first_year_included";
                print "   first_mon_included=$first_mon_included\n";
                print "${N}year:  last_year_included=$last_year_included";
                print "    last_mon_included=$last_mon_included\n";
                print "${N}year: prev_year=$prev_year   prev_mon=$prev_mon";
                print "   curr_year=$curr_year   curr_mon=$curr_mon\n";
                while (my ($var,$val) = each %{$iopt_href}) {
                  print "${N}year iopt: $var=$val\n";
                }
              }

              # Do not include this job if it would appear before
              # the start of the include range determined above
              next if ($prev_year < $first_year_included);
              if ($prev_year == $first_year_included) {
                next if ($prev_mon < $first_mon_included);
              }

              # Do not include this job if it would appear after
              # the end of the include range determined above
              next if ($curr_year > $last_year_included);
              if ($curr_year == $last_year_included) {
                next if ($curr_mon > $last_mon_included);
              }

              # Count the number of times this job gets used at this frequency
              my $job_id = "${jobname}_${N}y";
              if ( exists $JOB_COUNT{$job_id} ) {
                # Increment the count for this job at this frequency
                $JOB_COUNT{$job_id} += 1;
              } else {
                # Initialize the count
                $JOB_COUNT{$job_id} = 1;
              }
              my $curr_job_count = $JOB_COUNT{$job_id};

              # Add parallel job delimiters to specified jobs
              my $OPTref = {CURR_MON        => $curr_mon,
                            CURR_YEAR       => $curr_year,
                            PREV_MON        => $prev_mon,
                            PREV_YEAR       => $prev_year,
                            RUN_START_YEAR  => $run_start_year,
                            RUN_START_MONTH => $run_start_month,
                            RUN_END_YEAR    => $run_end_year,
                            RUN_END_MONTH   => $run_end_month};
              my $pre_print = '';
              my $post_print = '';
              ($thisjob, $pre_print, $post_print) =
                 insert_ext_delim($thisjob,$jobname,$OPTref);

              if ($main::Verbose == 1) {
                my $jinfo = " ${jobname}:${N}y";
                if ($split_thisjob) {$jinfo = " ${jobname}:${N}y:${split_thisjob}m"}
                my $strng = $pre_print . $jinfo . $post_print;
                if ( $line_out{pfx} ) {
                  $strng = $line_out{pfx} . $pre_print . $jinfo . $post_print;
                  $line_out{pfx} = "";
                }
                $line_out{wrote} = 1;
                print "$strng";
              }

              # replace variable values in $thisjob
              # next_year and next_month are used since this should be inserted at
              # the end of $months months after current $year,$mon counter values
              ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,"year",$next_year);
              ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,"mon",$next_month);

              ($thisjob,$rv2) = shell_var_subs::var_replace($thisjob,"current_year",
                                    sprintf("%3.3d",$curr_year));
              ($thisjob,$rv3) = shell_var_subs::var_replace($thisjob,"current_month",
                                    sprintf("%2.2d",$curr_mon));
              ($thisjob,$rv3) = shell_var_subs::var_replace($thisjob,"previous_year",
                                    sprintf("%3.3d",$prev_year));
              ($thisjob,$rv3) = shell_var_subs::var_replace($thisjob,"previous_month",
                                    sprintf("%2.2d",$prev_mon));

#              if ($curr_job_count == 1 and $jobname eq "xnemo") {
              if ( $curr_job_count == 1 ) {
                if (exists $main::VARS{nemo_restart}) {
                  # If a nemo restart file name is available then
                  # set it only on the first job of the string
                  my $nemo_restart = $main::VARS{nemo_restart}{value};
                  # Remove nemo_restart from the VARS hash so that it does not
                  # get set in any other jobs in this string
                  delete $main::VARS{nemo_restart};
                  if ( $nemo_restart ) {
                    # Redefine nemo_restart only when it has a non null value
                    # strip any three digit numerical extension
                    $nemo_restart  =~ s/(.*)(\.[0-9]{3})?$/$1/;
                    ($thisjob) = shell_var_subs::var_replace($thisjob,"nemo_restart",$nemo_restart);
                  }
                }
                if (exists $main::VARS{nemo_exec}) {
                  # If a nemo_exec is set by the user then
                  # set it only on the first job of the string
                  my $nemo_exec = $main::VARS{nemo_exec}{value};
                  # Remove nemo_exec from the VARS hash so that it does not
                  # get set in any other jobs in this string
                  delete $main::VARS{nemo_exec};
                  if ( $nemo_exec ) {
                    # Redefine nemo_exec only when it has a non null value
                    ($thisjob) = shell_var_subs::var_replace($thisjob,"nemo_exec",$nemo_exec);
                  }
                }
                if (exists $main::VARS{nemo_nn_it000}) {
                  # If nemo_nn_it000 is set by the user then
                  # set it only on the first job of the string
                  my $nemo_nn_it000 = $main::VARS{nemo_nn_it000}{value};
                  # Remove nemo_nn_it000 from the VARS hash so that it does not
                  # get set in any other jobs in this string
                  delete $main::VARS{nemo_nn_it000};
                  if ( $nemo_nn_it000 ) {
                    # Redefine nemo_nn_it000 only when it has a non null value
                    ($thisjob) = shell_var_subs::var_replace($thisjob,"nemo_nn_it000",$nemo_nn_it000);
		  }
                }
                if (exists $main::VARS{nemo_from_rest}) {
                  # If nemo_from_rest is set by the user then
                  # set it only on the first job of the string
                  my $nemo_from_rest = $main::VARS{nemo_from_rest}{value};
                  # Remove nemo_from_rest from the VARS hash so that it does not
                  # get set in any other jobs in this string
                  delete $main::VARS{nemo_from_rest};
                  # Redefine nemo_from_rest in the current job only
                  ($thisjob) = shell_var_subs::var_replace($thisjob,"nemo_from_rest",$nemo_from_rest);
                }
                if (exists $main::VARS{nemo_force_namelist}) {
                  # If nemo_force_namelist is set by the user then
                  # set it only on the first job of the string
                  my $nemo_force_namelist = $main::VARS{nemo_force_namelist}{value};
                  # Remove nemo_force_namelist from the VARS hash so that it does not
                  # get set in any other jobs in this string
                  delete $main::VARS{nemo_force_namelist};
                  # Redefine nemo_force_namelist in the current job only
                  ($thisjob) = shell_var_subs::var_replace($thisjob,"nemo_force_namelist",$nemo_force_namelist);
                }
	      }

              if ($split_thisjob) {
                # Insert multiple jobs with differences in only certain parameters
                # These are currently:
                # current_year, current_month, previous_year, previous_month
                my $mdelt = int($split_thisjob);
                my $year_a = $prev_year;
                my $mon_a  = $prev_mon;
                my ($mon_b, $year_b) = new_mon_year($mon_a, $year_a, $mdelt-1);
                my $delt_mon = diff_in_months($year_b,$mon_b,$curr_year,$curr_mon);
                if ($delt_mon < 0) {
                  print "\nAttempting to split $jobname into $mdelt month intervals.\n";
                  my $rmf = 1 + diff_in_months($prev_year,$prev_mon,$curr_year,$curr_mon);
                  print "This is greater than the current interval of $rmf months.\n";
                  die "Stopped";
                }
                while ( $delt_mon >= 0 ) {
                  if ($main::Verbose > 3) {
                    print "\n$jobname   ";
                    print "year_a=$year_a mon_a=$mon_a ... year_b=$year_b mon_b=$mon_b\n";
                  }
                  ($thisjob) = shell_var_subs::var_replace($thisjob,"previous_year",
                               sprintf("%3.3d",$year_a));
                  ($thisjob) = shell_var_subs::var_replace($thisjob,"previous_month",
                               sprintf("%2.2d",$mon_a));
                  ($thisjob) = shell_var_subs::var_replace($thisjob,"current_year",
                               sprintf("%3.3d",$year_b));
                  ($thisjob) = shell_var_subs::var_replace($thisjob,"current_month",
                               sprintf("%2.2d",$mon_b));
                  $str .= $thisjob;
                  # delt_mon=0 means mon_b,year_b is equal to curr_mon,curr_year
                  last if $delt_mon == 0;
                  ($mon_a, $year_a) = new_mon_year($mon_b, $year_b, 1);
                  ($mon_b, $year_b) = new_mon_year($mon_a, $year_a, $mdelt-1);
                  $delt_mon = diff_in_months($year_b,$mon_b,$curr_year,$curr_mon);
                  if ($delt_mon < 0) {
                    # mon_b, year_b is later than curr_mon, curr_year
                    ($mon_b, $year_b) = ($curr_mon, $curr_year);
                    $delt_mon = 0;
                  }
                }
              } else {
                $str .= $thisjob;
              }
            }

            # set previous_(year/month) to the month after the the last time
            # the section was inserted
            $prevmonth{$Nkey} = sprintf("%2.2d",$next_month + 1);
            if ($prevmonth{$Nkey} > 12) {
              $prevmonth{$Nkey} = sprintf("%2.2d",1);
              $prevyear{$Nkey} = sprintf("%3.3d",$next_year + 1);
            } else {
              $prevyear{$Nkey} = $next_year;
            }

            # Append to the job string
            $job .= $str;
          };
        };
      };

      # Unset the flag that indicates the first time through the loop
      $first_loop_iter=0;

      if ( $main::Verbose == 1 and $line_out{wrote} ) {print "\n"};

      # When kfinal is set on the command line (and put into vlist) it becomes
      # the stopping criterion for the job rather than run_end_year and end_mon
      if ($kfinal == $kount_at_end) {last YEARLOOP};

    }; # end of MONTHLOOP

    # reset $mon to the value it had on the last "mon" loop iteration
    # This value is used at the top of the "year" loop to set start_mon
    $mon = sprintf("%2.2d",$mon-$months);

  }; # end of YEARLOOP

  if ($main::Verbose > 1) {
    print "After YEARLOOP: year=$year  mon=$mon\n";
    print "                year_last_iter=$year_last_iter   mon_last_iter=$mon_last_iter";
    print "  kount_last_iter=",1*$kount_last_iter,"\n";
  };
  if ($main::Verbose > 3) {print "kount_list = @kount_list\n"};

  # set $template_last_year and $template_last_mon to the last year and month
  # of any job that was generated by the template
  my $template_last_year = $year_last_iter;
  my $template_last_mon = $mon_last_iter + $months - 1;
  if ($template_last_mon > 12) {
    $template_last_year = $template_last_year + int($template_last_mon/12);
    $template_last_mon  = $template_last_mon%12;
    if ($template_last_mon == 0) {
      $template_last_mon = 12;
      $template_last_year = $template_last_year - 1;
    };
  };
  if ($main::Override_total_months_check) {
    # If the total month check is not done above then
    # these are simply run_end_year and run_end_month
    $template_last_year = $run_end_year;
    $template_last_mon = $run_end_month;
  }
  $template_last_year = sprintf("%3.3d",$template_last_year);
  $template_last_mon  = sprintf("%2.2d",$template_last_mon);

  if ($main::Verbose > 3) {
    print "template_last_year = $template_last_year";
    print "   template_last_mon = $template_last_mon\n";
  };

  #========== special section ==========
  # copy the special section template to a temporary string
  ($str) = @{$TEMPLATE{special}};
  if ($str) {
    # split the current string into individual jobs
    # delimited by #end_of_job at the bottom
    my @splitjob = split_job_string($str);

    if (scalar(@splitjob) and $main::Verbose == 1) {
      print "Appended to job string:\n";
    };

    # perform section specific substitutions
    $str = '';
    foreach my $thisjob (@splitjob) {
      # Ignore empty job strings
      next if ($thisjob =~ /^\s*$/s);

      # get the name of the current job in jobname and
      # remove the job tag from the current job string
      my ($jobname) = read_jobtag($thisjob,{NAME_TAG=>1});

      # Extract info about splitting this job into multiple jobs
      my ($split_thisjob) = read_jobtag($thisjob,{SPLIT_TAG=>1});

      # Get a hash ref to internal job specific options
      my $iopt_href = read_jobtag($thisjob,{IOPT_TAG=>1});

      # determine a year/month range, outside of which this job
      # should not be included in the final job string
      my $first_year_included = $run_start_year;
      my $first_mon_included  = $run_start_month;
      my $last_year_included  = $run_end_year;
      my $last_mon_included   = $run_end_month;
      $first_year_included = $iopt_href->{start_year} if $iopt_href->{start_year};
      $first_mon_included  = $iopt_href->{start_mon}  if $iopt_href->{start_mon};
      $last_year_included  = $iopt_href->{stop_year}  if $iopt_href->{stop_year};
      $last_mon_included   = $iopt_href->{stop_mon}   if $iopt_href->{stop_mon};

      my $mon_offset = 0;
      if ($jobname =~ /^\s*mload\s*$/) {
        $mon_offset = $mload_mon_offset;
      } elsif ($jobname =~ /^\s*mdump\s*$/) {
        $mon_offset = $mdump_mon_offset;
      } elsif ($jobname =~ /^\s*mdelete\s*$/) {
        $mon_offset = $mdelete_mon_offset;
      } elsif ($jobname =~ /^\s*load_list\s*$/) {
        $mon_offset = $load_list_mon_offset;
      } elsif ($jobname =~ /^\s*dump_list\s*$/) {
        $mon_offset = $dump_list_mon_offset;
      } elsif ($jobname =~ /^\s*del_list\s*$/) {
        $mon_offset = $del_list_mon_offset;
      } elsif ($jobname =~ /^\s*custom\d*\s*$/) {
        $mon_offset = $custom_mon_offset;
      } elsif ($jobname =~ /^\s*cloop\s*$/) {
        $mon_offset = $cloop_mon_offset;
      };

      my ($curr_mon, $curr_year) =
          new_mon_year($template_last_mon, $template_last_year, $mon_offset);
      my ($prev_mon, $prev_year) =
          new_mon_year($special{previous_month}, $special{previous_year}, $mon_offset);

      # Do not include this job if it would appear before
      # the start of the include range determined above
      next if ($curr_year < $first_year_included);
      if ($curr_year == $first_year_included) {
        next if ($curr_mon < $first_mon_included);
      }

      # Do not include this job if it would appear after
      # the end of the include range determined above
      next if ($curr_year > $last_year_included);
      if ($curr_year == $last_year_included) {
        next if ($curr_mon > $last_mon_included);
      }

#      # do not include this job if mon_offset would make
#      # the next year|month go past the end of the run
#      next if ($curr_year > $run_end_year);
#      if ($curr_year == $run_end_year) {
#        next if ($curr_mon > $run_end_month);
#      };

      if ($main::Verbose == 1) {
        my $jinfo = "    ${jobname}:s\n";
        if ($split_thisjob) {$jinfo = "    ${jobname}:s:${split_thisjob}m\n"}
        print " $jinfo";
      }

      # replace variable values in $thisjob
      ($thisjob,$rv1) = shell_var_subs::var_replace($thisjob,"year",
                            sprintf("%3.3d",$template_last_year));
      ($thisjob,$rv3) = shell_var_subs::var_replace($thisjob,"mon",
                            sprintf("%2.2d",$template_last_mon));

      ($thisjob,$rv2) = shell_var_subs::var_replace($thisjob,"current_year",
                            sprintf("%3.3d",$curr_year));
      ($thisjob,$rv3) = shell_var_subs::var_replace($thisjob,"current_month",
                            sprintf("%2.2d",$curr_mon));
      ($thisjob,$rv3) = shell_var_subs::var_replace($thisjob,"previous_year",
                            sprintf("%3.3d",$prev_year));
      ($thisjob,$rv3) = shell_var_subs::var_replace($thisjob,"previous_month",
                            sprintf("%2.2d",$prev_mon));

      if ($split_thisjob) {
        # Insert multiple jobs with differences in only certain parameters
        # These are currently:
        # current_year, current_month, previous_year, previous_month
        my $mdelt = int($split_thisjob);
        my $year_a = $prev_year;
        my $mon_a  = $prev_mon;
        my ($mon_b, $year_b) = new_mon_year($mon_a, $year_a, $mdelt-1);
        my $delt_mon = diff_in_months($year_b,$mon_b,$curr_year,$curr_mon);
        if ($delt_mon < 0) {
          print "\nAttempting to split $jobname into $mdelt month intervals.\n";
          my $rmf = 1 + diff_in_months($prev_year,$prev_mon,$curr_year,$curr_mon);
          print "This is greater than the current interval of $rmf months.\n";
          die "Stopped";
        }
        while ( $delt_mon >= 0 ) {
          if ($main::Verbose > 3) {
            print "\n$jobname   ";
            print "year_a=$year_a mon_a=$mon_a ... year_b=$year_b mon_b=$mon_b\n";
          }
          ($thisjob) = shell_var_subs::var_replace($thisjob,"previous_year",
                       sprintf("%3.3d",$year_a));
          ($thisjob) = shell_var_subs::var_replace($thisjob,"previous_month",
                       sprintf("%2.2d",$mon_a));
          ($thisjob) = shell_var_subs::var_replace($thisjob,"current_year",
                       sprintf("%3.3d",$year_b));
          ($thisjob) = shell_var_subs::var_replace($thisjob,"current_month",
                       sprintf("%2.2d",$mon_b));
          $str .= $thisjob;
          # delt_mon=0 means mon_b,year_b is equal to curr_mon,curr_year
          last if $delt_mon == 0;
          ($mon_a, $year_a) = new_mon_year($mon_b, $year_b, 1);
          ($mon_b, $year_b) = new_mon_year($mon_a, $year_a, $mdelt-1);
          $delt_mon = diff_in_months($year_b,$mon_b,$curr_year,$curr_mon);
          if ($delt_mon < 0) {
            # mon_b, year_b is later than curr_mon, curr_year
            ($mon_b, $year_b) = ($curr_mon, $curr_year);
            $delt_mon = 0;
          }
        }
      } else {
        $str .= $thisjob;
      }
    }
    print "\n";

    # Append to the job string
    $job .= $str;

    if ($main::Verbose > $Vtrip) {
      print "=== special === year=$template_last_year  mon=$template_last_mon ================\n";
      print "$str";
    };
    if ($main::Verbose > $Vtrip) {
      print "***WARNING*** special: Failed to replace year\n" unless $rv1;
    };
  };

  return $job;
}

sub parse_job_here_docs { my ($job) = @_;
  use strict;
  # Split the job into sections by looking for "standard" here documents.
  # The here documents recognized are Model_Input, Execute_Script and Input_Cards.
  # Each of these must begin with a line of the form e.g.
  # cat > Model_Input << end_of_data
  # where the end_of_data delimiter may be any string and the blanks may be
  # any white space.
  # Each document is assigned to an element of the @parsed array and groups of
  # intervening non-document lines are assigned to other elements in order.
  # Names are given to each section in the section_id array. These names are
  # either "Model_Input", "Execute_Script", "Input_Cards" or "Global_n" where
  # the n in Global_n is a sequence number (starting from 0) to identify the
  # sections that are not part of one of the here documents.
  my @parsed;
  my @section_id;
  my @section_end;
  my $Vtrip=5;
  my $section = '';
  my $line = '';
  my $ecat = '';
  my $ecat_m;
  my $ecat_e;
  my $ecat_i;
  my $section_name = 'Global_0';
  my $gcount = 0;
  my %RET = ();
  my @job_line = split '\n', $job;
  if ($main::Verbose > $Vtrip) {print "Length of job :: ",scalar(@job_line)," lines\n"};
  foreach $line (@job_line) {
    ($ecat) = $line =~ /^ *cat *> *Model_Input *<<[ '"]*(\w+)/ and do {
      # beginning of Model_Input here document
      $ecat_m = $ecat;
      if ($main::Verbose > $Vtrip) {
        print "parse_job_here_docs :: start of Model_Input ecat_m = $ecat_m\n";
      };
      push @parsed, $section;
      push @section_id, $section_name;
      push @section_end, '';
      $section_name = "Model_Input";
      $section = "$line\n";
      next;
    };
    defined $ecat_m and $line =~ /^$ecat_m/ and do {
      # end of Model_Input here document
      if ($main::Verbose > $Vtrip) {
        print "parse_job_here_docs :: end of Model_Input ecat_m = $ecat_m\n";
      };
      $section .= "$line\n";
      push @parsed, $section;
      push @section_id, $section_name;
      push @section_end, $ecat_m;
      $section_name = "Global_" . ++$gcount;
      $section = '';
      undef $ecat_m;
      next;
    };
    ($ecat) = $line =~ /^ *cat *> *Execute_Script *<<[ '"]*(\w+)/ and do {
      # beginning of Execute_Script here document
      $ecat_e = $ecat;
      if ($main::Verbose > $Vtrip) {
        print "parse_job_here_docs :: start of Execute_Script ecat_e = $ecat_e\n";
      };
      push @parsed, $section;
      push @section_id, $section_name;
      push @section_end, '';
      $section_name = "Execute_Script";
      $section = "$line\n";
      next;
    };
    defined $ecat_e and $line =~ /^$ecat_e/ and do {
      # end of Execute_Script here document
      if ($main::Verbose > $Vtrip) {
        print "parse_job_here_docs :: end of Execute_Script ecat_e = $ecat_e\n";
      };
      $section .= "$line\n";
      push @parsed, $section;
      push @section_id, $section_name;
      push @section_end, $ecat_e;
      $section_name = "Global_" . ++$gcount;
      $section = '';
      undef $ecat_e;
      next;
    };
    ($ecat) = $line =~ /^ *cat *> *Input_Cards *<<[ '"]*(\w+)/ and do {
      # beginning of Input_Cards here document
      $ecat_i = $ecat;
      if ($main::Verbose > $Vtrip) {
        print "parse_job_here_docs :: start of Input_Cards ecat_i = $ecat_i\n";
      };
      push @parsed, $section;
      push @section_id, $section_name;
      push @section_end, '';
      $section_name = "Input_Cards";
      $section = "$line\n";
      next;
    };
    defined $ecat_i and $line =~ /^$ecat_i/ and do {
      # end of Input_Cards here document
      if ($main::Verbose > $Vtrip) {
        print "parse_job_here_docs :: end of Input_Cards ecat_i = $ecat_i\n";
      };
      $section .= "$line\n";
      push @parsed, $section;
      push @section_id, $section_name;
      push @section_end, $ecat_i;
      $section_name = "Global_" . ++$gcount;
      $section = '';
      undef $ecat_i;
      next;
    };
    $section .= "$line\n";
  };
  if ($section) {
    push @parsed, $section;
    push @section_id, $section_name;
    push @section_end, '';
  };
  if ($main::Verbose > $Vtrip) {
    foreach (@parsed) {
      print "="x20," parse_job_here_docs ","="x20,"\n$_";
    };
    print "Length of parsed job :: ",scalar(@parsed),"\n";
  };
  # return a hash of 2 arrays of equal length, one containing the
  # sections and the other containing section ids
  $RET{job} = $job;
  $RET{section} = [ @parsed ];
  $RET{id} = [ @section_id ];
  $RET{doc_end} = [ @section_end ];
  return %RET;
}

sub parse_job_sections { my ($job, $doc_end) = @_;
  use strict;
  # split the first argument passed in, into sections that begin with a line
  # of the form /^ *###.*$regex/ and end with either one of these header
  # lines or a line of the form /^ *end_of_data *$/ or /^ *# *end_of_job *$/.
  # Here regex is one of gcmparm, parmsub, condef, update.*script,
  # update.*model, update.*sub, update.*ocean or update.*ocean.*sub
  #
  # $job is a "section" string returned from parse_job_here_docs
  # $doc_end is a string containing the end delimiter of the current here doc
  # that was found in parse_job_here_docs or the null string indicating a
  # "global" section (ie: not part of a here doc).
  my @parsed;
  my @section_id;
  my $Vtrip=5;
  my $section_name='';
  my $section;
  my $sname;
  my %RET = ();
  foreach (split '\n', $job) {
    if (($sname) = /^ *(###.*gcmparm.*)/ or
        ($sname) = /^ *(###.*parmsub.*)/ or
        ($sname) = /^ *(###.*condef.*)/ or
        ($sname) = /^ *(###.*update.*script.*)/ or
        ($sname) = /^ *(###.*update.*model.*)/ or
        ($sname) = /^ *(###.*update.*sub.*)/ or
        ($sname) = /^ *(###.*update.*ocean.*)/ or
        ($sname) = /^ *(###.*update.*ocean.*sub.*)/) {
      push @parsed, $section;
      push @section_id, $section_name;
      $section_name = $sname;
      $section = "$_\n";
    } elsif ($doc_end and /^ *$doc_end *$/) {
      push @parsed, $section;
      push @section_id, $section_name;
      $section_name = '';
      $section = "$_\n";
    } elsif (/^ *# *end_of_job *$/) {
      push @parsed, $section;
      push @section_id, $section_name;
      $section_name = 'END OF JOB';
      $section = "$_\n";
    } else {
      $section .= "$_\n";
    };
  };
  if ($section) {
    push @parsed, $section;
    push @section_id, $section_name;
  };
  if ($main::Verbose > $Vtrip) {
    my $lcount=0;
    my $tcount = scalar(@parsed);
    foreach (@parsed) {
      print "="x20," ",++$lcount," of $tcount  parse_job_sections ","="x20,"\n$_";
    };
  };
  # return a hash of 2 arrays of equal length, one containing the
  # subsections and the other containing subsection ids
  $RET{job} = $job;
  $RET{section} = [ @parsed ];
  $RET{id} = [ @section_id ];
  return %RET;
}

sub substitute_job { my ($JOBSref, $SUBSref) = @_;
  use strict;
  # Modify the job string by performing substitutions found in SUBS.
  # A list containing the modified job string is returned.
  # The section values in the JOBS hash are also modified upon return.
  my $patt;
  my $repl;
  my $sopt;
  my $subbed = 0;
  my $curr_subs = 0;
  my $href = {};
  my @RET = ();
  my $subexp;
  my $delim;
  my $job;
  my $id;
  my $subid;
  my $seqno;
  my $i;

  foreach $subexp (@$SUBSref) {
    # Each element of SUBS must be of the form
    # /pattern/replacement/
    # optionally followed by the "i" (case insensitive) option.
    # The delimiter can be any single char not in either the pattern
    # or replacement string. Any perl regex is allowed for the pattern.
    ($delim) = $subexp =~ /^(.)/;   # determine delimiter as first char
    ($patt)  = $subexp =~ /^$delim([^$delim]*)/;
    ($repl)  = $subexp =~ /^$delim[^$delim]*$delim([^$delim]*)/;
    ($sopt)  = $subexp =~ /^$delim[^$delim]*$delim[^$delim]*$delim([^$delim]*)$/;
    # The "i" (case insensitive) option for the substitution operator will be
    # recognized and appended to the "gm" options if it is supplied in SUBS,
    # otherwise "gm" will be the only options allowed.
    if ($sopt) {$sopt = "gm".$sopt} else {$sopt = "gm"};
    # $patt =~ s/(\W)/\\$1/g;       # quote all non-alphanumeric chars
    # $repl =~ s/(\W)/\\$1/g;       # quote all non-alphanumeric chars
    if ($main::Verbose > 3) {print "patt=$patt   repl=$repl   sopt=$sopt\n"};
    foreach $href (@$JOBSref) {
      $job   = $href->{section};
      $id    = $href->{id};
      $subid = $href->{subid};
      $seqno = $href->{sequence};
      # do not do any substitutions in an update section unless the
      # Sub_in_update_section flag is set
      if (not $main::Sub_in_update_section and $subid =~ /^ *###.*update/) {next};
      if ($sopt eq "gmi") {$curr_subs = $href->{section} =~ s/$patt/$repl/gmi}
      else {$curr_subs = $href->{section} =~ s/$patt/$repl/gm};
      $subbed += $curr_subs;
    }
  }
  if ($main::Verbose > 0) {print "\nNumber of substitutions done $subbed\n"};

  # Rebuild the "job" entry in the JOBS hash so that any
  # changes made in this subroutine are reflected there
  my @jobs = ();
  foreach $href (@$JOBSref) {
    $i = $href->{sequence} - 1;
    $jobs[$i] .= $href->{section};
  };
  foreach $href (@$JOBSref) {
    $i = $href->{sequence} - 1;
    $href->{job} = $jobs[$i];
  };

  foreach $href (@$JOBSref) {push @RET, $href->{section}};
  return @RET;
}

sub change_var_vals { my ($JOBSref, $VARSref) = @_;
  use strict;
  # Modify the job string according to the variable definitions found in VARS
  # A list containing the modified job string is returned.
  # The section values in the JOBS hash are also modified upon return.
  my $vname;
  my $vval;
  my $replaced = 0;
  my $removed = 0;
  my $added = 0;
  my $href = {};
  my @RET = ();
  my $job;
  my $apat;
  my $bpat;
  my $cpat;
  my $zpat;
  my $rval;
  my $i;

  # Replace all occurrences in JOBS of variables defined in VARS.
  # Add variables that do not exist in JOBS and remove variables
  # that have a null value.

  # loop over variable replacements in VARS by numeric order of the element rank
  foreach $vname (sort {$VARSref->{$a}{rank} <=>
                        $VARSref->{$b}{rank}} keys %$VARSref) {
    $vval = $VARSref->{$vname}{value};
    if ($main::Verbose > 5) {
      printf "change_var_vals: processing vname=%-10s vval=%-20s\n",$vname,$vval;
    };
    # loop over job sections in JOBS replacing or removing vname=value pairs
    foreach $href (@$JOBSref) {
      my $section = $href->{section};
      my $id      = $href->{id};
      my $subid   = $href->{subid};
      # my $seqno   = $href->{sequence};

      # If this is an update section then move on,
      # unless the Rep_in_update_section flag is set
      if (not $main::Rep_in_update_section and $subid =~ /^ *###.*update/) {next};

      # If this variable is not present in the current section
      # then  skip the rest of the loop
      next unless ($section =~ /\b${vname}=/);

      # if remove flag is set and the value of $vname is undefined or null
      # then remove all definitions of $vname from $section
      if ($VARSref->{$vname}{remove} and (not defined $vval or $vval eq '')) {
        ($section, $rval) =
          shell_var_subs::var_remove($section, $vname,{VERBOSE=>$main::Verbose});
        if ($rval) {$removed++};
      }
      # otherwise redefine $vname in $section
      else {
        ($section, $rval) = shell_var_subs::var_replace($section, $vname, $vval);
        if ($rval) {$replaced++};
      };

      # put the modified section back into the JOBS hash
      $href->{section} = $section;
    }
  }

  # Rebuild the "job" entry in the JOBS hash so that any
  # changes made in this subroutine are reflected there
  my @jobs = ();
  foreach $href (@$JOBSref) {
    $i = $href->{sequence} - 1;
    $jobs[$i] .= $href->{section};
  };
  foreach $href (@$JOBSref) {
    $i = $href->{sequence} - 1;
    $href->{job} = $jobs[$i];
  };

  # Insert variables as required
  foreach $href (@$JOBSref) {
    my $job     = $href->{job};
    my $section = $href->{section};
    my $subid   = $href->{subid};
    my $id      = $href->{id};
    # my $seqno   = $href->{sequence};
    # print "seqno = $seqno  id = $id  subid = $subid\n";
    # If this is an update section then move on,
    # unless the Rep_in_update_section flag is set
    if (not $main::Rep_in_update_section and $subid =~ /^ *###.*update/) {next};

    # loop over variables in VARS by reverse numeric order of the element "rank"
    # The order is reversed because variables will be always be inserted at the
    # same point in the $section string which will always be just before the
    # previous insert.
    foreach $vname (reverse sort {$VARSref->{$a}{rank} <=>
                        $VARSref->{$b}{rank}} keys %$VARSref) {
      $vval = $VARSref->{$vname}{value};

      # Ignore unless this variables "add" flag is set
      next unless ($VARSref->{$vname}{add});

      # Ignore if this variable already exists in the string
      unless ($job =~ /\b${vname}=/) {
        # There is no variable vname in the current job ...add one

        # quote values that contain shell meta characters
        if ($vval =~ /[\Q][*)(?!\E]/) {
          # If the value is already quoted, do nothing
          unless ($vval =~ /^".*"$/ or $vval =~ /^'.*'$/) {
            if ($vval !~ /'/) {$vval = "'" . "$vval" . "'"}
            elsif ($vval !~ /"/) {$vval = '"' . "$vval" . '"'}
            else {print "change_var_vals: Unquoted shell meta chars in $vname=$vval\n"};
          }
        }

        # ensure that white space in the value is quoted
        if ($vval =~ /\s/) {
          # If the value is already quoted, do nothing
          unless ($vval =~ /^".*"$/ or $vval =~ /^'.*'$/) {
            if ($vval !~ /"/) {$vval = '"' . "$vval" . '"'}
            elsif ($vval !~ /'/) {$vval = "'" . "$vval" . "'"}
            else {print "change_var_vals: Unquoted  white space in $vname=$vval\n"}
          }
        }

        # call var_insert once using $job as the string to find out where the variable
        # should be inserted. The insertion point is returned as $rval which is then
        # supplied with the TAG option to indicate the insertion point on a second
        # call to var_insert using the $section string
        (my $j, $rval) = shell_var_subs::var_insert($job, "$vname=$vval");

        # A return value of -6 means the insertion point is at the top of the string.
        # This should only happen on the first $section in the $job string. This first
        # $section is identified by $id=Global_0.
        if ($rval == -6 and ($id ne "Global_0")) {next};

        # Avoid adding any variables after the end of the job string
        next if $subid =~ /^ *END *OF *JOB *$/;

        # Insert the variable into the $section string and replace the section
        # entry in the JOBS hash with this modified string.
        # Note that the job section of the JOBS hash is rewritten below
        # to reflect these changes.
        ($section, $rval) =
          shell_var_subs::var_insert($section, "$vname=$vval",{TAG=>$rval});
        if ($rval) {$added++};
        # put the modified section back into the JOBS hash
        $href->{section} = $section;
      };
    };
  };

  if ($main::Verbose > 0) {
    print "\nVariables reset $replaced, removed $removed, added $added\n";
  };

  # Once again rebuild the "job" entry in the JOBS hash so that any
  # changes made in the variable insertion phase are reflected there
  @jobs = ();
  foreach $href (@$JOBSref) {
    $i = $href->{sequence} - 1;
    $jobs[$i] .= $href->{section};
  };
  foreach $href (@$JOBSref) {
    $i = $href->{sequence} - 1;
    $href->{job} = $jobs[$i];
  };

  foreach $href (@$JOBSref) {push @RET, $href->{section}};
  return @RET;
}

sub insert_updates { my ($JOBSref, $UPDATEref) = @_;
  use strict;
  no strict qw(refs);
  # Insert updates provided in the UPDATE hash into the job string.
  # A list containing the modified job string is returned.
  # The section values in the JOBS hash are also modified upon return.
  my $upkey;
  my $job;
  my $id;
  my $subid;
  my $seqno = 0;
  my $seqno_p = 0;
  my $pflag;
  my $href = {};
  my @RET = ();
  my $section;
  my $samerun;
  my $i;
  my %updates_added;

  # loop over all keys in the UPDATE hash
  # The current list of possible keys is script, model, sub, ocean or non_standard
  foreach $upkey (keys %$UPDATEref) {
    if ($UPDATEref->{$upkey}) {
      # Initialize a flag that will indicate whether updates are added or not added
      $updates_added{$upkey} = 0;
      # loop over all sections in the JOBS array looking for an update section
      foreach $href (@$JOBSref) {
        $job     = $href->{job};
        $section = $href->{section};
        $id      = $href->{id};
        $subid   = $href->{subid};
        $seqno   = $href->{sequence};
        if ($seqno == $seqno_p) {$pflag = 0}
        else {$pflag = 1};
        # Do not print warnings for script updates
        if ($upkey eq "script") {$pflag = 0};
        $seqno_p = $seqno;
        # look for the value of samerun in the current job
        ($samerun) = shell_var_subs::var_find($job, "samerun");
        $samerun = '' unless defined $samerun;
        if ($samerun ne "" and $samerun ne "on" and $samerun ne "off") {
          if ($pflag) {
            print "***WARNING*** The value of samerun=$samerun found ";
            print "in the current job is neither \"on\" nor \"off\"\n";
          };
          $samerun='';
        };
        unless ($samerun) {
          # if samerun is not present then set it to on (do not add updates)
          if ($main::Verbose > 2 and $pflag and not $main::Always_add_updates) {
            print "***WARNING*** $upkey updates are supplied but samerun ";
            print "is not present or is invalid in the current job.\n";
            print "              No updates are added.\n";
            print "              To add updates ensure that there is a ";
            print "\"### update $upkey ###\" \n";
            print "              line in your job and add \":f\" to the";
            print " --updates=fname:f command line arg\n";
            print " current job:\n";
            $i = 0; foreach (split '\n',$job) {
              $i++;
              if ($i > 5) {last};
              print "$_\n";
            };
          };
          $samerun = "on";
        };
        # When $Always_add_updates is set then always add updates provided
        # the /^ *###.*update.*$upkey/ line is present
        if ($main::Always_add_updates) {$samerun = "off"};
        # Always add script updates regardless of the value of samerun
        if ($upkey eq "script") {$samerun = "off"};
        my $topline = '^ *###.*update.*' . $upkey;
        if ($samerun eq "off" and $subid =~ /$topline/) {
          # If these are script updates simply append them to the current
          # script updates found in the "### update script" section
          # If these are not script updates then replace any existing
          # updates with those from the UPDATES hash.
          # TODO: This needs to be examined more closely. Perhaps set a flag
          # to determine if one wants to append or replace an update section.
#xxx Treat script updates the same as all others
#xxx          unless ($upkey eq "script") {
            # Remove anything currently in this update section and
            # replace it with the user supplied set of updates
            my ($section_head) = $section =~ /($topline)/;
            $section = "$section_head\n";
#xxx          }
          my $tmpupdt = $section . $UPDATEref->{$upkey};

          if ($main::Reorder_updates) {
            # my $linecount = 0;
            # foreach (split "\n", $tmpupdt) {
            #   print "$_\n";
            #   $linecount++;
            #   last if ($linecount > 10);
            # }
            # strip all "%c" update directives from the updates and prepend them
            # to the current section, after any /^ *###.*update.*$upkey/ line and
            # after removing any duplicate directives
            my $cregex = '^\s*%c\s+.*';
            my @upclines = ($tmpupdt =~ /($cregex)/mig);
            $tmpupdt =~ s/$cregex//mig;
            # remove duplicate lines
            my @c2 = ();
            foreach (@upclines) {
              # strip any whitespace around % and ensure a single space follows
              s/^\s*%\s*(.*)/%$1/;
              push @c2,$_ unless grep /^$_$/i,@c2;
            }
            my $cdefs = join "\n",@c2;
            if ($cdefs) {
              $tmpupdt =~ s/($topline.*)/$1\n$cdefs/mi;
            }

            # remove all %df update directives from these updates and prepend them
            # to the current section, after any /^ *###.*update.*$upkey/ line and
            # after removing any duplicate directives
            my $dfregex = '^\s*%\s*df\s+.*';
            my @updefs = ($tmpupdt =~ /($dfregex)/mig);
            $tmpupdt =~ s/$dfregex//mig;
            # remove duplicate lines
            my @d2 = ();
            foreach (@updefs) {
              # strip any whitespace around % and ensure a single space follows
              s/^\s*%\s*(.*)/%$1/;
              push @d2,$_ unless grep /^$_$/i,@d2;
            }
            my $defs = join "\n",@d2;
            if ($defs) {
              $tmpupdt =~ s/($topline.*)/$1\n$defs/mi;
            }
	  }

          # append to the current section
          $href->{section} = $tmpupdt;
          # set a flag to indicate that updates were added
          $updates_added{$upkey} = 1;
        };
        if ($main::Verbose > 5) {
          printf "%s\n", '=' x 80;
          printf "%s update key = %s\n", '.' x 20, $upkey;
          printf "%s samerun    = %s\n", '.' x 20, $samerun;
          printf "%s    id = %s\n", '.' x 20, $id;
          printf "%s subid = %s\n", '.' x 20, $subid;
          printf "%s seqno = %s\n", '.' x 20, $seqno;
          if ($main::Verbose > 10) {print "section:\n",$href->{section},"\n"};
          if ($main::Verbose > 10) {print "job:\n",$href->{job}};
        };
      };
      unless ($UPDATEref->{$upkey}->{ignore_error}) {
        unless ($updates_added{$upkey}) {
          die "*** ERROR *** $upkey updates were supplied but never added to the job string.
    Updates will never be added unless samerun=off is present in at least one job.
    Updates will never be added unless an appropriate \"### update $upkey ###\"
    line is found in the job string.
    Make sure there is an \"### update $upkey ###\" line in the input job then try
    rerunning $main::Invoked_name with samerun=off on the command line.
    Stopped";
        };
      };
    };
  };

  # Rebuild the "job" entry in the JOBS hash so that any
  # changes made in this subroutine are reflected there
  my @jobs = ();
  foreach $href (@$JOBSref) {
    $i = $href->{sequence} - 1;
    $jobs[$i] .= $href->{section};
  };
  foreach $href (@$JOBSref) {
    $i = $href->{sequence} - 1;
    $href->{job} = $jobs[$i];
  };

  foreach $href (@$JOBSref) {push @RET, $href->{section}};
  return @RET;
}

sub filter_updates {
  # Filter out any model or sub updates from jobs that
  # do not contain the parameter samerun=off
  use strict;
  no strict qw(refs);
  my $JOBSref = shift;
  my $OPTS    = shift;
  my $href;

  my $verbose = 0;
  if (defined $OPTS->{VERBOSE}) {$verbose = $OPTS->{VERBOSE}};

  foreach $href (@$JOBSref) {
    my $job     = $href->{job};
    my $section = $href->{section};
    my $subid   = $href->{subid};
    my $id      = $href->{id};

    # look for the value of samerun in the current job
    my ($samerun) = shell_var_subs::var_find($job, "samerun");
    # if samerun is not present set it to "on" (no updates)
    $samerun = 'on' unless defined $samerun;

    # filter updates unless samerun=off
    unless ($samerun eq "off") {
      # model or sub updates may be filtered
      if ($subid =~ /^ *###.*update.*model/) {
        my $fsec='';
        foreach my $line (split '\n',$section) {
          # remove any model updates that do not begin with "%c"
          unless ($line =~ /^ *###.*update.*model/ or
                   $line =~ /^\s*%[cC]/) {
            next;
          };
          $fsec .= $line . "\n";
        };
        $section = $fsec . "\n";
      };
      if ($subid =~ /^ *###.*update.*sub/) {
        # remove all sub updates
        my $fsec='';
        foreach my $line (split '\n',$section) {
          next unless $line =~ /^ *###.*update.*sub/;
          $fsec .= $line . "\n";
        };
        $section = $fsec . "\n";
      };
      $href->{section} = $section;
    };
  };

  # Rebuild the "job" entry in the JOBS hash so that any
  # changes made in this subroutine are reflected there
  my @jobs = ();
  foreach $href (@$JOBSref) {
    my $i = $href->{sequence} - 1;
    $jobs[$i] .= $href->{section};
  };
  foreach $href (@$JOBSref) {
    my $i = $href->{sequence} - 1;
    $href->{job} = $jobs[$i];
  };

  my @RET;
  foreach $href (@$JOBSref) {push @RET, $href->{section}};
  return @RET;
}

sub define_job {my @jobkey = @_;
  use strict;
  # Define a job string template using info from a list of keywords
  # Each keyword must be a key in the JOBDEF hash or a filename.
  # The JOBDEF hash is defined in the CCCjobs module which is accessed
  # via a use statement at the top of the cccjob driver.
  #
  # Each keyword may optionally be followed by a semicolon and a string
  # of alphanumeric characters. Each character in this option list will
  # indicate a position at which the current job is to be inserted into the
  # job string (e.g. monthly, yearly etc.)
  my $thisjob   = '';
  my $monthly   = '';
  my @Nmonthly  = ();
  my $yearly    = '';
  my @Nyearly   = ();
  my $special   = '';
  my $annual    = '';
  my $quarterly = '';
  my $DJF = '';
  my $MAM = '';
  my $JJA = '';
  my $SON = '';
  my $passthru = '';
  my $key;
  my $opt;
  my $str;
  my $Ldelim;
  my $Rdelim;
  my $i;

  foreach $key (@jobkey) {
    if ($main::Verbose > 10) {print "jobkey: $key\n"};

    # Check for internal options ...anything enclosed in <...> at the end of the string
    my ($int_opt) = $key =~ /(<.*>)$/;
    if ($int_opt) {
      # Strip the internal options from key
      $key =~ s/(<.*>)$//;
      # Strip the angle brackets from int_opt
      $int_opt =~ s/^\s*<\s*(.*)\s*>\s*$/$1/;
    }

    my $opt = '';
    # check for options ...anything following a ":" in the keyword
    ($opt) = $key =~ /(:.*)$/;
    $opt = '' unless $opt;
    if ($opt =~ /^:+$/) {
      print "Empty option list on keyword $key\n";
      die "Stopped";
    }
    $opt =~ s/^://;
    $opt = '' unless $opt;
    # strip any options from the end of key
    $key =~ s/:.*$//;
    # look for an equal sign in $key and put whatever is right of that equal
    # sign into $keymod
    my $keymod = '';
    ($key, $keymod) = split '=',$key;
    $keymod = '' unless $keymod;
    if ($main::Verbose > 10) {print "key=$key   keymod=$keymod   opt=$opt\n"};

    # Look for a file named $key first
    # If the file is not found then look in the JOBDEF hash
    my $fname = '';
    if (-f $key) {
      $fname = $key;
    } else {
      ($fname) = on_cccjob_path($key);
    }
    if ($fname) {
      # This is actually a file, so read it
      $thisjob = '';
      open(THISJOB, "<$fname") || die "cannot open $fname, stopped";
      while (<THISJOB>) {$thisjob .= $_};
      close(THISJOB);
      unless ($thisjob) {
        die "The file $fname is empty.\nStopped";
      }

      # Add this file to the JOBDEF hash
      if (exists $main::JOBDEF{job}{$key}) {
        if ($main::Verbose > 1) {
          print "*** WARNING *** Overwriting internal module $key";
          print " with a user supplied version found in file $fname\n";
        }
        push @main::warnings,
          "*** WARNING *** Overwriting internal module $key
         with a user supplied version found in file $fname";
      }
      $main::JOBDEF{job}{$key} = $thisjob;
      $main::JOBDEF{path}{$key} = $fname;
      $main::JOBDEF{clone}{$key} = 0;
      $main::JOBDEF{user_supplied}{$key} = 1;
      $main::JOBDEF{description}{$key} = '';
      $main::JOBDEF{hide}{$key} = 0;

    } elsif (defined $main::JOBDEF{job}{$key}) {
      # extract this job from the JOBDEF hash
      $thisjob = $main::JOBDEF{job}{$key};
      unless ($thisjob) {
        die "The $key module found in JOBDEF is empty.\n  Stopped";
      }

    } else {
      show_JOBDEF "An invalid entry \"$key\" was found in the job list. Valid entries are:";
      die "Stopped";
    }

    # Determine if this job contains any CPP_I section and, if so, modify it
    # according to user supplied macro defines/undefs
    my ($new_job, $modified) = update_cpp_i( $main::JOBDEF{job}{$key},
                                             {JOBNAME => $key} );
    if ($modified) {
      # Replace this job string with the updated version
      $main::JOBDEF{job}{$key} = $new_job;
      $thisjob = $new_job;
    }
    undef $new_job;

    if ($key eq "pool_mul") {
      # pool_mul should have a key modifier supplied
      my $pool_mul_id = "jja";
      if ($keymod) {
        $pool_mul_id = $keymod;
      } else {
        print "Keyword pool_mul was specified without a modifier (ie: pool_mul=modifier)\n";
        print "Valid modifiers are ann,djf,mam,jja,son,m01..m12,jan...dec\n";
        die "  Stopped";
      }
      if ($main::Verbose > 10) {print "pool_mul_id=$pool_mul_id\n"};
      # replace pool_mul_id in this job only
      ($thisjob) = shell_var_subs::var_replace($thisjob, "pool_mul_id", $pool_mul_id);
      # set a default value for opt st pool_mul will be appended to the job string
      # Note: a typical opt might be :5y but :s is safest as a default
      $opt = ':s' unless $opt;
    }

    # If there are MARKER lines already in this job
    # then parse it and append to the appropriate sections
    if ($thisjob =~ /(^|\n)#MARKER:/s) {
      my %TEMPLATE = parse_job_template($thisjob);
      if (($str) = @{$TEMPLATE{monthly}})   {
        ($str) = add_jobtag($str, $key);
        # Add an internal options tag if any internal options are present
        ($str) = add_jobtag($str, $key, {IOPT_TAG=>$int_opt}) if $int_opt;
        $monthly .= $str;
      }
      if (($str) = @{$TEMPLATE{yearly}})    {
        ($str) = add_jobtag($str, $key);
        # Add an internal options tag if any internal options are present
        ($str) = add_jobtag($str, $key, {IOPT_TAG=>$int_opt}) if $int_opt;
        $yearly .= $str;
      }
      if (($str) = @{$TEMPLATE{special}})   {
        ($str) = add_jobtag($str, $key);
        # Add an internal options tag if any internal options are present
        ($str) = add_jobtag($str, $key, {IOPT_TAG=>$int_opt}) if $int_opt;
        $special .= $str;
      }
      if (($str) = @{$TEMPLATE{annual}})    {
        ($str) = add_jobtag($str, $key);
        # Add an internal options tag if any internal options are present
        ($str) = add_jobtag($str, $key, {IOPT_TAG=>$int_opt}) if $int_opt;
        $annual .= $str;
      }
      if (($str) = @{$TEMPLATE{DJF}})       {
        ($str) = add_jobtag($str, $key);
        # Add an internal options tag if any internal options are present
        ($str) = add_jobtag($str, $key, {IOPT_TAG=>$int_opt}) if $int_opt;
        $DJF .= $str;
      }
      if (($str) = @{$TEMPLATE{MAM}})       {
        ($str) = add_jobtag($str, $key);
        # Add an internal options tag if any internal options are present
        ($str) = add_jobtag($str, $key, {IOPT_TAG=>$int_opt}) if $int_opt;
        $MAM .= $str;
      }
      if (($str) = @{$TEMPLATE{JJA}})       {
        ($str) = add_jobtag($str, $key);
        # Add an internal options tag if any internal options are present
        ($str) = add_jobtag($str, $key, {IOPT_TAG=>$int_opt}) if $int_opt;
        $JJA .= $str;
      }
      if (($str) = @{$TEMPLATE{SON}})       {
        ($str) = add_jobtag($str, $key);
        # Add an internal options tag if any internal options are present
        ($str) = add_jobtag($str, $key, {IOPT_TAG=>$int_opt}) if $int_opt;
        $SON .= $str;
      }
      if (($str) = @{$TEMPLATE{quarterly}}) {
        ($str) = add_jobtag($str, $key);
        # Add an internal options tag if any internal options are present
        ($str) = add_jobtag($str, $key, {IOPT_TAG=>$int_opt}) if $int_opt;
        $quarterly .= $str;
      }
      next;
    }

    # Add a line to $thisjob identifying it with a keyword
    ($thisjob) = add_jobtag($thisjob, $key);

    # Add an internal options tag if any internal options are present
    ($thisjob) = add_jobtag($thisjob, $key, {IOPT_TAG=>$int_opt}) if $int_opt;

    if ($opt) {
      # do the right thing for each option
      # Groups of options are separated by a ':'
      # Currently only the first and second colon separated groups are recognized.
      # All other groups (after the third colon) are ignored.
      my ($optA, $optB) = split /:/, $opt;
      $optA = '' unless $optA;
      $optB = '' unless $optB;
      if ($main::Verbose > 10) {
        print "key=$key   opt=$opt   optA=$optA    optB=$optB\n";
      }
      # Deal with optB first in case the loop is short cycled
      # by a "next" command while processing optA
      if ($optB) {
        # optB should contain only digits followed by the letter "m" or "y"
        unless ($optB =~ /^(\d*([mM]?|[yY]?))$/) {
          print "Invalid second modifier -->$optB<-- on keyword ${key}:$opt";
          print "  ...require a positive integer optionally followed by m or y\n";
          die "Stopped";
        }
        my $split = $optB;
        ($split) = $split =~ /(.*)[mM]$/ if $split =~ /[mM]$/;
        $split = 1 unless $split;
        if ($split =~ /[yY]$/) {
          my ($y) = $split =~ /(.*)[yY]$/;
          $y = 1 unless $y;
          $split = 12*$y;
        }
        # Add a tag line to $thisjob to indicate the split value
        # This will also add a Job Name tag line which may or may not
        # get overwritten later on.
        ($thisjob) = add_jobtag($thisjob, $key, {SPLIT_TAG=>$split});
      } else {
        die "Empty second modifier on keyword ${key}:$opt" if $opt =~ /^.*:/;
      }
      if (not $optA or $optA =~ /n/) {
        # If there is one or more n options ignore the rest
        $passthru  .= $thisjob;
        next;
      } else {
        # split opt into elements, with each element after the colon consisting
        # of a single letter that is optionally preceeded by an integer that may
        # be in turn preceeded by a negative sign.
        my @optel = $optA =~ /((?:-?\d+)*[a-zA-Z]{1}|:)/g;
        if ($main::Verbose > 10) {print "optel=@optel\n"};
        # If there is anything that doesn't match an integer or letter then we have a problem
        my ($extra) = $optA =~ /[^-\da-zA-Z:]+/;
        if ($main::Verbose > 10) {print "extra=$extra\n"};
        if ($extra) {
          die "Invalid modifier $extra on keyword ${key}:$opt in joblist\n Stopped";
        }
        foreach (@optel) {
          /:/ and do {next};

          # define N as the integer that (possibly) preceeds the option
          # then strip that integer to ensure the option is a single letter
          my $N = 1; # set a default for N
          /-?\d+/ and do { ($N) = /(-?\d+)/;  s/-?\d+// };
          if ($main::Verbose > 10) {print "N=$N\n"};
          if ($N == 0) {
            print "Numeric modifier ($N) to flag $_ on --joblist=${key}:$opt";
            print " command line option is out of range\n";
            die " Stopped";
          };

          # Currently only the monthly and yearly sections will recognize
          # the integer modifier N. It will be ignored for all other options.
          /m/ and do {
            if ($N == 1) {
              $monthly .= $thisjob;
            } else {
              my $href = {};
              $href->{interval} = $N;
              $href->{options}  = $int_opt;
              $href->{section}  = $thisjob;
              # append to an existing element with the same interval value
              # or push onto the end of the list if this is a new interval
              my $found=0;
              foreach (@Nmonthly) {
                if ($_->{interval} == $N) {
                  # The intervals are equivalent when
                  # the interval length is the same
                  $_->{section} .= $thisjob;
                  $found = 1;
#xxx                  if ($_->{options} or $int_opt) {
#xxx                    # If one set of internal options is missing but the other
#xxx                    # is present then the intervals are not equivalent
#xxx                    next unless $_->{options};
#xxx                    next unless $int_opt;
#xxx                    # Check for matching option strings
#xxx                    if ($_->{options} eq $int_opt) {
#xxx                      $_->{section} .= $thisjob;
#xxx                      $found = 1;
#xxx#DBGprint "=========  Option strings match\n";
#xxx                    }
#xxx                  } else {
#xxx                    $_->{section} .= $thisjob;
#xxx                    $found = 1;
#xxx                  }
                }
#DBG
#DBGmy ($jobname) = read_jobtag($_->{section},{NAME_TAG=>1}) if $_->{section};
#DBGprint "N=$N  interval=$_->{interval}  found=$found  jobname=$jobname\n";
#DBGprint "=========  int_opt=$int_opt\n";
#DBGprint "=========  options=$_->{options}\n";
              }

#xxx#DBG
#xxxunless ($found) {
#xxx  my ($jobname) = read_jobtag($href->{section},{NAME_TAG=>1});
#xxx  print "*** ADDING $jobname\n";
#xxx}
              push @Nmonthly, $href unless $found;
            }
            next;
          };
          /y/ and do {
            if ($N == 1) {
              $yearly .= $thisjob;
            } else {
              my $href = {};
              $href->{interval} = $N;
              $href->{options}  = $int_opt;
              $href->{section}  = $thisjob;
              # append to an existing element with the same interval value
              # or push onto the end of the list if this is a new interval
              my $found=0;
              foreach (@Nyearly) {
                if ($_->{interval} == $N) {
                  # If neither set of internal options exist then the intervals are
                  # equivalent when the interval length is the same
                  if ($_->{options} or $int_opt) {
                    # If one set of internal options is missing but the other
                    # is present then the intervals are not equivalent
                    next unless $_->{options};
                    next unless $int_opt;
                    # Check for matching option strings
                    if ($_->{options} eq $int_opt) {
                      $_->{section} .= $thisjob;
                      $found = 1;
                    }
                  } else {
                    $_->{section} .= $thisjob;
                    $found = 1;
                  }
                }
              }
              push @Nyearly, $href unless $found;
            }
            next;
          };
          /a/ and do {$annual    .= $thisjob; next};
          /q/ and do {$quarterly .= $thisjob; next};
          /s/ and do {$special   .= $thisjob; next};
          /D/ and do {$DJF       .= $thisjob; next};
          /M/ and do {$MAM       .= $thisjob; next};
          /J/ and do {$JJA       .= $thisjob; next};
          /S/ and do {$SON       .= $thisjob; next};
          die "Unknown option $_ on keyword $key in joblist\n Stopped";
        }
      }
    } else {
      # default: append to the passthru section
      $passthru .= $thisjob;
    }
  }

  my $job = '';
  my @RET = ();

  # Return the passthru section, if any, as the first element of @RET
  if ($passthru) {
    push @RET, $passthru;
  }

  # Concatenate the rest of the sections (bracketed by appropriate delimiters)
  # and push them onto @RET
  if ($monthly) { # insert at start of month
    $Ldelim = "\n#MARKER: Monthly Section\n";
    $Rdelim = "\n#MARKER: End of Monthly\n";
    $job .= $Ldelim . $monthly . $Rdelim;
  }
  foreach my $href (@Nmonthly) {
    my $N    = $href->{interval};
    my $opts = $href->{options};
    if ($opts) {
      substr($opts,0,0) = '<';
      $opts .= '>';
    }
    if ($main::Verbose > 10) {
#DBG
      print "*** ${N}:m ", $opts ? "$opts\n" : "No internal options\n";
      foreach ( split_job_string($href->{section}) ) {
        my ($jobname) = read_jobtag($_,{NAME_TAG=>1});
        print "***   jobname: $jobname\n";
      }
    }
    my $Section = $href->{section};
    if ($Section) { # insert at end of every N months
      if ($opts) {
        $Ldelim = "\n#MARKER: Every $N Months Section$opts\n";
        $Rdelim = "\n#MARKER: End of $N Months$opts\n";
      } else {
        $Ldelim = "\n#MARKER: Every $N Months Section\n";
        $Rdelim = "\n#MARKER: End of $N Months\n";
      }
      $job .= $Ldelim . $Section . $Rdelim;
    }
  }
  if ($yearly) {  # insert at end of year
    $Ldelim = "\n#MARKER: Yearly Section\n";
    $Rdelim = "\n#MARKER: End of Yearly\n";
    $job .= $Ldelim . $yearly . $Rdelim;
  }
  foreach my $href (@Nyearly) {
    my $N    = $href->{interval};
    my $opts = $href->{options};
    if ($opts) {
      substr($opts,0,0) = '<';
      $opts .= '>';
    }
    if ($main::Verbose > 10) {
      print "*** ${N}:y ",$opts ? "$opts" : "No internal options","\n";
      foreach ( split_job_string($href->{section}) ) {
        my ($jobname) = read_jobtag($_,{NAME_TAG=>1});
        print "***   jobname: $jobname\n";
      }
    }
    my $Section = $href->{section};
    if ($Section) { # insert at end of every N years
      if ($opts) {
        $Ldelim = "\n#MARKER: Every $N Years Section$opts\n";
        $Rdelim = "\n#MARKER: End of $N Years$opts\n";
      } else {
        $Ldelim = "\n#MARKER: Every $N Years Section\n";
        $Rdelim = "\n#MARKER: End of $N Years\n";
      }
      $job .= $Ldelim . $Section . $Rdelim;
    }
  }
  if ($special) {  # insert at end of job
    $Ldelim = "\n#MARKER: Special Section\n";
    $Rdelim = "\n#MARKER: End of Special\n";
    $job .= $Ldelim . $special . $Rdelim;
  }
  if ($annual) {  # insert at start of year
    $Ldelim = "\n#MARKER: Annually Section\n";
    $Rdelim = "\n#MARKER: End of Annually\n";
    $job .= $Ldelim . $annual . $Rdelim;
  }
  if ($quarterly) {  # insert at start of each quarter
    $Ldelim = "\n#MARKER: Quarterly Section\n";
    $Rdelim = "\n#MARKER: End of Quarterly\n";
    $job .= $Ldelim . $quarterly . $Rdelim;
  }
  if ($DJF) {  # insert at the end of DJF (excluding first year)
    $Ldelim = "\n#MARKER: Seasonal DJF Section\n";
    $Rdelim = "\n#MARKER: End of DJF Seasonal\n";
    $job .= $Ldelim . $DJF . $Rdelim;
  }
  if ($MAM) {  # insert at the end of MAM
    $Ldelim = "\n#MARKER: Seasonal MAM Section\n";
    $Rdelim = "\n#MARKER: End of MAM Seasonal\n";
    $job .= $Ldelim . $MAM . $Rdelim;
  }
  if ($JJA) {  # insert at the end of JJA
    $Ldelim = "\n#MARKER: Seasonal JJA Section\n";
    $Rdelim = "\n#MARKER: End of JJA Seasonal\n";
    $job .= $Ldelim . $JJA . $Rdelim;
  }
  if ($SON) {  # insert at the end of SON
    $Ldelim = "\n#MARKER: Seasonal SON Section\n";
    $Rdelim = "\n#MARKER: End of SON Seasonal\n";
    $job .= $Ldelim . $SON . $Rdelim;
  }
  if ($job) {push @RET, $job};
  if ($main::Verbose > 10) {
    $i=1;
    foreach (@RET) {print '-'x40,$i++,'-'x40,"\n",$_};
  }
  return @RET;
}

# This is a common prefix used to identify lines containing job specific
# information that is written by add_jobtag and read by read_jobtag
$ID_jobtag = "#:#:#:#:#:#:#:#:#:#";

sub add_jobtag {
  # Add a tag line to a job string
  use strict;
  use vars qw($ID_jobtag);
  my ($jobstr, $jobname, $OPTS) = @_;

  # Set option defaults
  my $verbose = 0;
  my $remove_tag = 0;
  my $name_tag = 1;
  my $split_tag = 0;
  my $iopt_tag = '';

  # Assign any user supplied option values
  $verbose    = $OPTS->{VERBOSE}    if defined $OPTS->{VERBOSE};
  $remove_tag = $OPTS->{REMOVE_TAG} if defined $OPTS->{REMOVE_TAG};
  $name_tag   = $OPTS->{NAME_TAG}   if defined $OPTS->{NAME_TAG};
  $split_tag  = $OPTS->{SPLIT_TAG}  if defined $OPTS->{SPLIT_TAG};
  $iopt_tag   = $OPTS->{IOPT_TAG}   if defined $OPTS->{IOPT_TAG};

  # error checking
  unless ($jobstr) {die "add_jobtag: job string is empty.\n  Stopped"};
  unless ($jobname) {print "add_jobtag: job name is not defined\n"};

  if ($verbose > 0) {
    print "\nadd_jobtag: Job Name = $jobname\n";
    print "add_jobtag: remove_tag=$remove_tag  name_tag=$name_tag  split_tag=$split_tag\n";
  }

  if ($remove_tag) {
    # Remove any existing tag lines from the job string
    $jobstr =~ s/[ \t]*$ID_jobtag[^\n]*\n//gs;
  }

  if ($name_tag) {
    # Define a tag line to be used to identify this job
    # when it is part of a larger string

    # $ID must be exactly the same as in the subroutine read_jobtag
    my $ID = "$ID_jobtag Job Name:";
    my $jobtag = "$ID $jobname\n";

    # Remove any existing Job Name tag lines from the string
    $jobstr =~ s/[ \t]*$ID[^\n]*\n//gs;

    # Insert the tag line after any shebang lines
    $jobstr =~ s@(^|\n)[ \t]*(#! */[^\n]*\n)@$1$2$jobtag@gs;

    unless ($jobstr =~ /[ \t]*$ID[^\n]*\n/s) {
      # if no tag lines were inserted prepend one together with a shebang line
      $jobstr = "#!/bin/sh\n" . $jobtag . $jobstr;
    }
  }

  if ($split_tag) {
    # Define a tag line to be used to indicate if this job should be split
    # into 2 or more jobs

    # $ID must be exactly the same as in the subroutine read_jobtag
    my $ID = "$ID_jobtag Split:";
    my $jobtag = "$ID $split_tag\n";

    # Remove any existing Split tag lines from the string
    $jobstr =~ s/[ \t]*$ID[^\n]*\n//gs;

    # Insert the tag line after any shebang lines
    $jobstr =~ s@(^|\n)[ \t]*(#! */[^\n]*\n)@$1$2$jobtag@gs;

    unless ($jobstr =~ /[ \t]*$ID[^\n]*\n/s) {
      # if no tag lines were inserted prepend one together with a shebang line
      $jobstr = "#!/bin/sh\n" . $jobtag . $jobstr;
    }
  }

  if ($iopt_tag) {
    # Define a tag line to be used to pass internal options

    # $ID must be exactly the same as in the subroutine read_jobtag
    my $ID = "$ID_jobtag Iopt:";
    my $jobtag = "$ID $iopt_tag\n";

    # Remove any existing Iopt tag lines from the string
    $jobstr =~ s/[ \t]*$ID[^\n]*\n//gs;

    # Insert the tag line after any shebang lines
    $jobstr =~ s@(^|\n)[ \t]*(#! */[^\n]*\n)@$1$2$jobtag@gs;

    unless ($jobstr =~ /[ \t]*$ID[^\n]*\n/s) {
      # if no tag lines were inserted prepend one together with a shebang line
      $jobstr = "#!/bin/sh\n" . $jobtag . $jobstr;
    }
  }

  if ($verbose > 0) {
    my $line = 0;
    print "\n\nadd_jobtag: head\n";
    for (split "\n",$jobstr) {
      $line++;
      if ($line < 6) {print "$_\n"}
    }
    print "\ntotal lines: $line\n";
    my ($call_package, $call_filename, $call_line) = caller;
    print "called from $call_filename  line $call_line\n";
  }

  return $jobstr;
}

sub read_jobtag {
  # Read a tag line written by add_jobtag from a job string
  use strict;
  use vars qw($ID_jobtag);
  my ($jobstr, $OPTS) = @_;
  my @RET = ();

  # Set option defaults
  my $verbose    = 0;
  my $remove_tag = 0;
  my $name_tag   = 0;
  my $split_tag  = 0;
  my $iopt_tag   = 0;

  # Assign user supplied option values
  $verbose    = $OPTS->{VERBOSE}    if defined $OPTS->{VERBOSE};
  $remove_tag = $OPTS->{REMOVE_TAG} if defined $OPTS->{REMOVE_TAG};
  $name_tag   = $OPTS->{NAME_TAG}   if defined $OPTS->{NAME_TAG};
  $split_tag  = $OPTS->{SPLIT_TAG}  if defined $OPTS->{SPLIT_TAG};
  $iopt_tag   = $OPTS->{IOPT_TAG}   if defined $OPTS->{IOPT_TAG};

  # error checking
  unless ($jobstr) {die "read_jobtag: job string is empty.\n  Stopped"};
  if ( ($name_tag and $split_tag) or
       ($name_tag and $iopt_tag) or
       ($split_tag and $iopt_tag) ) {
    print "read_jobtag: Only one of NAME_TAG, SPLIT_TAG or IOPT_TAG may be set\n";
    print "read_jobtag: ";
    print "NAME_TAG=$name_tag" if $name_tag;
    print "   SPLIT_TAG=$split_tag" if $split_tag;
    print "   IOPT_TAG=$iopt_tag\n" if $iopt_tag;
    die "Stopped";
  }
  unless ($name_tag or $split_tag or $iopt_tag or $remove_tag) {
    print "read_jobtag: One of NAME_TAG, SPLIT_TAG or IOPT_TAG must be set\n";
    die "Stopped";
  }

  if ($verbose > 0) {
    my $line = 0;
    print "\n\nread_jobtag: head\n";
    for (split "\n",$jobstr) {
      $line++;
      if ($line < 6) {print "$_\n"}
    }
    print "\ntotal lines: $line\n";
    my ($call_package, $call_filename, $call_line) = caller;
    print "called from $call_filename  line $call_line\n";
  }

  if ($name_tag) {
    # $ID must be exactly the same as in the subroutine add_jobtag
    my $ID = "$ID_jobtag Job Name:";

    # return a list containing all job names found in the job string
    @RET = $jobstr =~ /[ \t]*$ID([^\n]*)\n/gs;

    # strip leading and trailing white space from each job name found
    foreach (@RET) {s/^\s*//; s/\s*$//};

    if ($verbose > 0) {
      print "read_jobtag: Job Names ",join(" ",@RET),"\n";
    }
  }

  if ($split_tag) {
    # $ID must be exactly the same as in the subroutine add_jobtag
    my $ID = "$ID_jobtag Split:";

    # return a list containing all split values found in the job string
    @RET = $jobstr =~ /[ \t]*$ID([^\n]*)\n/gs;

    # strip leading and trailing white space from each split value found
    foreach (@RET) {s/^\s*//; s/\s*$//};

    # If no Split tag lines were found then return zero
    unless (scalar(@RET)) {push @RET,0}
    if ($verbose > 0) {
      print "read_jobtag: Split values ",join(" ",@RET),"\n";
    }
  }

  if ($iopt_tag) {
    # $ID must be exactly the same as in the subroutine add_jobtag
    my $ID = "$ID_jobtag Iopt:";

    # Populate a hash containing all internal option definitions
    my $iopt_href = {};

    # This will loop over all Iopt lines (prefix removed) in the job string
    foreach ( $jobstr =~ /[ \t]*$ID\s*([^\n]*)\n/gs ) {
      # Options found on a single line are separated by commas (optional
      # whitespace may surround a comma separator)
      foreach (split /\s*,\s*/) {
        next unless $_;
        if (/=/) {
          my ($var,$val) = split(/=/);
          die "read_jobtag: Invalid internal option definition --> $_ <--\n"
            unless $var;
          $iopt_href->{$var} = $val;
	} else {
	  $iopt_href->{$_} = 1;
	}
      }
    }

    if ($verbose > 0) {
      print "\nread_jobtag: Iopt values ",
        join("=",each %{$iopt_href}),"\n";
    }

    # Return only the options hash ref
    return $iopt_href;
  }

  if ($remove_tag) {
    # remove all tag lines from the input string
    $_[0] =~ s/[ \t]*$ID_jobtag[^\n]*\n//gs;
  }

  my ($ret) = @RET;
  return wantarray ? @RET : $ret;
}

sub str2ref_iopts {
  use strict;
  my $iopts_string  = shift;
  my $OPTS          = shift;
  # Given a string containing internal options, return a hash of all those options
  # The string must be of the form
  #    < var1=val1,var2=val2,... >
  # where the enclosing angle brackets are optional and if any value is missing
  # along with the "=" sign then that variable is assigned the valus of 1 (ie true)

  my $id = 'str2ref_iopts';
  $id = $OPTS->{ID} if defined $OPTS->{ID};
  my $verbose = 0;
  $verbose = $OPTS->{VERBOSE} if defined $OPTS->{VERBOSE};

  my $iopts_href = {};
  my $nopts = 0;
  if ($iopts_string) {
    # Strip any enclosing angle brackets
    if ($iopts_string =~ /[><]/) {
      my $errst = "Internal option string is missing opening \"<\"";
      die "${id}: $errst --> $iopts_string <--\n  Stopped"
        unless ($iopts_string =~ /^\s*</);
      $errst = "Internal option string is missing closing \">\"";
      die "${id}: $errst --> $iopts_string <--\n  Stopped"
        unless ($iopts_string =~ />\s*$/);
    }
    my ($iopts) = $iopts_string =~ /^\s*<?\s*(.*)$/;
    $iopts =~ s/\s*>?\s*$//;
    # The remaining string must be a comma separated list (optional white space
    # is allowed around the commas)
    foreach (split(/\s*,\s*/,$iopts)) {
      next unless $_;
      if (/=/) {
        my ($var,$val) = split(/=/);
        unless ($var) {
          warn "${id}: Invalid internal option definition --> $_ <--\n";
          die "options: $iopts\n   Stopped";
        }
        $iopts_href->{$var} = $val;
        $nopts++;
      } else {
        $iopts_href->{$_} = 1;
        $nopts++;
      }
    }
  }

  if ($verbose > 0) {
    if ($nopts) {
      print "\n${id}: $iopts_string\n";
      while (my ($var,$val) = each %{$iopts_href}) {
        print "${id}: $var=$val\n";
      }
    } else {
      print "${id}: No internal options found in --> ";
      print $iopts_string ? $iopts_string : ''," <--\n";
    }
  }

  # Return undef unless there are options found
  return $nopts ? $iopts_href : undef;
}

sub ref2str_iopts {
  use strict;
  my $iopts_href  = shift;
  my $OPTS        = shift;
  # Given a hash reference containing internal options, return a single string
  # containing all the options
  # The output string will be of the form
  #    < var1=val1,var2=val2,... >
  # where the enclosing angle brackets are optionally returned

  my $id = 'ref2str_iopts';
  $id = $OPTS->{ID} if defined $OPTS->{ID};
  my $verbose = 0;
  $verbose = $OPTS->{VERBOSE} if defined $OPTS->{VERBOSE};
  my $with_delim = 1;
  $with_delim = $OPTS->{WITH_DELIM} if defined $OPTS->{WITH_DELIM};

  my $iopts_string = '';

  # Create the string, adding options in lexagraphical order of hash keys
  foreach my $var ( sort keys %{$iopts_href} ) {
    my $val = $iopts_href->{$var};
    $iopts_string .= ",${var}=$val";
  }

  # Strip the leading comma (if any) from this option list
  $iopts_string =~ s/\s*,// if $iopts_string;

  if ( $with_delim ) {
    # Add enclosing angle brackets to the option string
    if ($iopts_string) {
      substr($iopts_string,0,0) = '<';
      $iopts_string .= '>';
    }
  }

  return $iopts_string;
}

sub set_iopts {
  use strict;
  my $iopts_string   = shift;
  my $set_iopts_href = shift;
  my $OPTS           = shift;
  # Given a string containing internal options, and a hash containing a group
  # of new options, set (or reset) each new option and return the new string
  # The string must be of the form
  #    < var1=val1,var2=val2,... >
  # where the enclosing angle brackets are optional and if any RHS value
  # is missing then the associated "=" sign must also be missing

  my $id = 'set_iopts';
  $id = $OPTS->{ID} if defined $OPTS->{ID};
  my $verbose = 0;
  $verbose = $OPTS->{VERBOSE} if defined $OPTS->{VERBOSE};
  my $with_delim = 1;
  my $test_string = $iopts_string;
  $test_string =~ s/\s*//g if $test_string;
  if ($test_string) {
    $with_delim = 0 unless $test_string =~ /[><]/;
  }
  $with_delim = $OPTS->{WITH_DELIM} if defined $OPTS->{WITH_DELIM};

  # Convert the input string to a hash
  my $iopts_in_href = str2ref_iopts($iopts_string, {ID=>$id});

  # If str2ref_iopts returns null, redefine this as an empty hash ref
  # so that empty input does not imply empty output
  $iopts_in_href = {} unless $iopts_in_href;

  foreach ( keys %{$set_iopts_href} ) {
    $iopts_in_href->{$_} = $set_iopts_href->{$_};
  }

  # Transform the (possibly modified) hash back into a string
  my $OPTS_out = {ID=>$id, WITH_DELIM=>$with_delim};
  my $iopts_out_string = ref2str_iopts($iopts_in_href, $OPTS_out);
  return $iopts_out_string;
}

sub split_job_string {
  use strict;
  my $jobstr  = shift;
  my $OPTS    = shift;
  # return a list of jobs in a job string
  # a single job is delimited at the bottom by a line
  # of the form /^\s*#\s*end_of_job\s*$/

  my $verbose = 0;
  if (defined $OPTS->{VERBOSE}) {$verbose = $OPTS->{VERBOSE}};

  # error checking
  unless ($jobstr) {die "split_job_string: job string is empty.\n  Stopped"};

  # read the file line by line breaking it into individual jobs delimited
  # at the end by a line of the form /^ *# *end_of_job */
  my @RET;
  my $job = "";
  foreach (split /\n/,$jobstr) {
    $job .= $_ . "\n";
    if ($_ =~ /^\s*#\s*end_of_job\s*/ ) {
      push @RET, $job;
      $job = "";
    }
  }
  # push whatever is left over onto the end of job_list
  if ($job) {push @RET, $job};

  # strip leading and trailing white space from each job name found
  foreach (@RET) {s/^\s*//s; s/\s*$//s; $_ .= "\n"};

  return @RET;
};

sub new_mon_year {
  use strict;
  # input current month, year and an offset in months
  my $mon  = shift;
  my $year = shift;
  my $mon_offset  = shift;
  my $OPTS = shift;

  my $verbose = 0;
  if (defined $OPTS->{VERBOSE}) {$verbose = $OPTS->{VERBOSE}};

  # error checking
  if ($mon<1 or $mon>12) {die "new_mon_year: month=$mon is out of range.\n  Stopped"};

  my $new_mon  = $mon + $mon_offset;
  my $new_year = $year + int(($new_mon-1)/12);
  if ($new_mon < 1) {$new_year = $year - 1 + int(($new_mon)/12)};
  if ($new_mon < 1 or $new_mon > 12) {$new_mon = 1 + ($new_mon-1)%12};

  if ($verbose>0) {
    print "mon=$mon  year=$year  mon_offset=$mon_offset";
    print "     new_mon=$new_mon  new_year=$new_year\n"
  };

  return ($new_mon,$new_year);
};

sub diff_in_months {
  # Calculate the difference in months: y2,m2 minus y1,m1
  # where y1,m1,y2,m2 are input in that order.
  # If y1,m1 is the same date as y2,m2 then the return value is zero.
  # If y1,m1 is later than y2,m2 then the return value will be negative.
  use strict;
  my ($year1_in,$mon1_in,$year2_in,$mon2_in,$OPTS) = @_;

  my $verbose = 0;
  if (defined $OPTS->{VERBOSE}) {$verbose = $OPTS->{VERBOSE}};

  die "\ndiff_in_months: year1 is not a positive integer.\n  Stopped"
     unless $year1_in =~ /^\d+$/;
  die "\ndiff_in_months: year2 is not a positive integer.\n  Stopped"
     unless $year2_in =~ /^\d+$/;
  die "\ndiff_in_months: mon1 is not a positive integer.\n  Stopped"
     unless $mon1_in =~ /^\d+$/;
  die "\ndiff_in_months: mon2 is not a positive integer.\n  Stopped"
     unless $mon2_in =~ /^\d+$/;

  my $mon1 = 1*$mon1_in;
  my $mon2 = 1*$mon2_in;
  my $year1 = 1*$year1_in;
  my $year2 = 1*$year2_in;

  if ($verbose>0) {
    print "\nyear1=$year1 mon1=$mon1 ... year2=$year2 mon2=$mon2\n";
  }

  # error checking
  if ($mon1<1 or $mon1>12 or $mon2<1 or $mon2>12) {
    print "\ndiff_in_months: year1=$year1 mon1=$mon1 ... year2=$year2 mon2=$mon2\n";
    print "diff_in_months: One of mon1=$mon1 or mon2=$mon2 is out of range.\n";
    die "Stopped";
  }

  my $mon_delt = -999;
  if ($year1 == $year2) {
    $mon_delt = $mon2 - $mon1;
  } elsif ($year1 > $year2) {
    $mon_delt = -(12*($year1-$year2-1) + 12-$mon2 + $mon1);
  } else {
    $mon_delt = 12*($year2-$year1-1) + 12-$mon1 + $mon2;
  }

  if ($verbose>0) {
    print "     mon_delt=$mon_delt\n"
  }

  return wantarray ? ($mon_delt) : $mon_delt;
}

sub on_cccjob_path {
  use strict;
  # If a file is found in the colon separated directory list defined in the
  # environment variable CCCJOB_PATH then return the full path name of that
  # file, otherwise return false.
  #
  # Only the first ocurrence will be returned.
  #
  # If a list of files is passed in then a list of full path names is returned.
  # This list may be shorter than the input list if some of the input files
  # are not found in CCCJOB_PATH
  my @RET = ();
  return @RET unless ($ENV{CCCJOB_PATH});
  foreach my $f (@_) {
    foreach my $d (split ':',$ENV{CCCJOB_PATH}) {
      if (-r "$d/$f") {
        push @RET, "$d/$f";
        last;
      };
    };
  };
  return @RET;
}

sub file_body {
  # Find a file whose name is passed in and return its contents as a
  # string (also return the full pathname of the file if invoked in
  # list context).
  # The file is searched for in a list of dirs determined herein.
  use strict;
  use File::Basename;
  use Text::Tabs;
  my ($fname) = shift;
  my ($OPTS)  = shift;

  my $verbose = 0;
  if (defined $OPTS->{VERBOSE}) {$verbose = $OPTS->{VERBOSE}};

  if ($verbose > 1) {print "file_body: fname = $fname\n"}

  # CURR_PATH is a colon separated list of directories to be searched
  chomp(my $cwd = `pwd`);
  my $CURR_PATH = "$cwd:$ENV{HOME}/cccjobs";

  # Append any dirs found in the environment variable CCCJOB_PATH
  # CCCJOB_PATH should be a colon separated list of directories
  $CURR_PATH = "${CURR_PATH}:$ENV{CCCJOB_PATH}" if $ENV{CCCJOB_PATH};

  # Append the default jobdefs path
  $CURR_PATH = "${CURR_PATH}:$main::CCCJOB_LIB/jobdefs";

  if ($verbose > 1) {print "file_body: CURR_PATH = $CURR_PATH\n"}

  # Look for the file in dirs found in CURR_PATH
  my $fbody = '';
  my $fpath = '';
  foreach my $dn (split '\s*:\s*',$CURR_PATH) {
    if ($verbose > 3) {print "file_body: dn = $dn\n"}
    # Silently ignore anything that is not a directory
    next unless (-d $dn);
    # ensure a trailing slash
    $dn .= '/' unless $dn =~ m'/$';
    unless (opendir DIR, $dn) {
      warn "*** WARNING *** Cannot access directory $dn\n";
      next;
    }
    # list all files except .*
    my @files = grep !/^\./, readdir DIR;
    # prepend dir name to each file
    @files = map "$dn$_", @files;
    foreach my $full_path (@files) {
      if ($verbose > 4) {print "file_body: full_path = $full_path\n"}
      # Ignore anything that is not a regular file
      next unless (-f "$full_path");
      my ($name, $path, $suffix) = fileparse("$full_path",'');
      if ($verbose > 5) {
        print "file_body:  name=$name  path=$path  suffix=$suffix\n";
      }
      next unless "$name$suffix" eq "$fname";

      # read the file contents and get out of the loop
      die "$full_path is not a regular file.\n  Stopped"
        unless (-f $full_path);
      if (open(FSRC, "<$full_path")) {
        # expand tabs as each line is read
        while (<FSRC>) {$fbody .= expand($_)};
        close(FSRC);
        $fpath = $full_path;
        if ($verbose > 0) {
          print "file_body: Found $full_path\n";
        }
        last;
      } else {
        die "Cannot open file $full_path\n  Stopped";
      }
    }
    closedir DIR;
  }

  if ($verbose > 2) {
    print "file_body: Contents of $fpath\n$fbody\n\n";
  }

  return wantarray ? ($fbody, $fpath) : $fbody;
}

sub insert_ext_delim {
  use strict;
  # Insert extended section/parallel job delimiters into the current
  # job string. These delimiters are added to jobs whose names appear
  # in the global arrays @begin_ext_delim, @parallel_ext_delim
  # and @end_ext_delim.
  my $thisjob = shift;
  my $jobname = shift;
  my $OPTS    = shift;
  my $add_ext = shift;

  my $pre_print = '';
  my $post_print = '';

  my $verbose = 0;
  $verbose = $OPTS->{VERBOSE} if defined $OPTS->{VERBOSE};

  die "insert_ext_delim: jobname was not supplied.\n  "
    unless $jobname;

#  my $always_add_start = 0;
#  $always_add_start = $OPTS->{always_add_start} if defined $OPTS->{always_add_start};
#  my $always_add_end = 0;
#  $always_add_end = $OPTS->{always_add_end} if defined $OPTS->{always_add_end};
#  my $always_add_par = 0;
#  $always_add_par = $OPTS->{always_add_par} if defined $OPTS->{always_add_par};

  die "insert_ext_delim: CURR_YEAR is not defined.\n  "
    unless defined $OPTS->{CURR_YEAR};
  die "insert_ext_delim: CURR_MON is not defined.\n  "
    unless defined $OPTS->{CURR_MON};
  die "insert_ext_delim: PREV_YEAR is not defined.\n  "
    unless defined $OPTS->{PREV_YEAR};
  die "insert_ext_delim: PREV_MON is not defined.\n  "
    unless defined $OPTS->{PREV_MON};
  die "insert_ext_delim: RUN_START_YEAR is not defined.\n  "
    unless defined $OPTS->{RUN_START_YEAR};
  die "insert_ext_delim: RUN_START_MONTH is not defined.\n  "
    unless defined $OPTS->{RUN_START_MONTH};
  die "insert_ext_delim: RUN_END_YEAR is not defined.\n  "
    unless defined $OPTS->{RUN_END_YEAR};
  die "insert_ext_delim: RUN_END_MONTH is not defined.\n  "
    unless defined $OPTS->{RUN_END_MONTH};

  my $curr_year       = $OPTS->{CURR_YEAR};
  my $curr_mon        = $OPTS->{CURR_MON};
  my $prev_year       = $OPTS->{PREV_YEAR};
  my $prev_mon        = $OPTS->{PREV_MON};
  my $run_start_year  = $OPTS->{RUN_START_YEAR};
  my $run_start_month = $OPTS->{RUN_START_MONTH};
  my $run_end_year    = $OPTS->{RUN_END_YEAR};
  my $run_end_month   = $OPTS->{RUN_END_MONTH};

  my $first_job_in_string = 0;
  if ($prev_year == $run_start_year and $prev_mon == $run_start_month) {
    $first_job_in_string = 1;
  }
  my $last_job_in_string = 0;
  if ($curr_year == $run_end_year and $curr_mon == $run_end_month) {
    $last_job_in_string = 1;
  }
  # print "\nfirst_job_in_string=$first_job_in_string ... ";
  # print "$prev_year == $run_start_year and $prev_mon == $run_start_month\n";
  # print "last_job_in_string=$last_job_in_string ... ";
  # print "$curr_year == $run_end_year and $curr_mon == $run_end_month\n";

  my $add_begin_ext = 1;
  $add_begin_ext = $add_ext->{BEGIN_EXT}
    if defined $add_ext->{BEGIN_EXT};
  my $add_end_ext   = 1;
  $add_end_ext = $add_ext->{END_EXT}
    if defined $add_ext->{END_EXT};
  my $add_parallel_ext  = 1;
  $add_parallel_ext = $add_ext->{PARALLEL_EXT}
    if defined $add_ext->{PARALLEL_EXT};

  # Do not add #END_EXTENDED_SECTION delimiters to the first job in the job string
  $add_end_ext = 0      if $first_job_in_string;

  # Do not add #BEGIN_EXTENDED_SECTION delimiters to the last job in the string
  $add_begin_ext = 0 if $last_job_in_string;

  # Do not add #PARALLEL_JOB delimiters to the last job in the string
  # $add_parallel_ext = 0 if $last_job_in_string;

  # print "\nadd_begin_ext=$add_begin_ext   ";
  # print "add_end_ext=$add_end_ext   ";
  # print "add_parallel_ext=$add_parallel_ext\n";

  if ($main::add_ext_delim) {

    # Append #END_EXTENDED_SECTION delimiters to specified jobs
    if ($add_end_ext) {
      foreach (@{$main::ext_delim{add_end_delim}}) {
        s/^\s*//;
        s/\s*$//;
        next unless $_;
        if ($jobname =~ /^$_$/) {
          next if ($main::ext_delim{not_at_start}{$_} and $first_job_in_string);
          next if ($main::ext_delim{not_at_end}{$_} and $last_job_in_string);
          $thisjob =~ s/\s*$//s;
          $thisjob .= "\n#END_EXTENDED_SECTION\n";
          $post_print = "${post_print} E";
          last;
        }
      }
    }

    # Prepend #PARALLEL_JOB delimiters to specified jobs
    if ($add_parallel_ext) {
      foreach (@{$main::ext_delim{add_parallel_delim}}) {
        s/^\s*//;
        s/\s*$//;
        next unless $_;
        # print "\nparallel: $_\n";
        if ($jobname =~ /^$_$/) {
          next if ($main::ext_delim{not_at_start}{$_} and $first_job_in_string);
          next if ($main::ext_delim{not_at_end}{$_} and $last_job_in_string);
          $thisjob = "#PARALLEL_JOB\n" . $thisjob;
          $pre_print = "${pre_print} P";
          last;
        }
      }
    }

    # Prepend #BEGIN_EXTENDED_SECTION delimiters to specified jobs
    if ($add_begin_ext) {
      foreach (@{$main::ext_delim{add_begin_delim}}) {
        s/^\s*//;
        s/\s*$//;
        next unless $_;
        # print "\nbegin: $_\n";
        if ($jobname =~ /^$_$/) {
          next if ($main::ext_delim{not_at_start}{$_} and $first_job_in_string);
          next if ($main::ext_delim{not_at_end}{$_} and $last_job_in_string);
          $thisjob = "#BEGIN_EXTENDED_SECTION\n" . $thisjob;
          $pre_print = "${pre_print} S";
          last;
        }
      }
    }

    # Reverse the order of space separated strings in pre_print
    $pre_print = join ' ', reverse split /\s+/, $pre_print;
    $pre_print =~ s/\s*$//;
    $pre_print =~ s/^\s*/ / if $pre_print;

  }

  return wantarray ? ($thisjob, $pre_print, $post_print) : $thisjob;
}

sub update_cpp_i {
  # Given a job string as input update any CPP_I section found within the job
  # string by adding or changing any defines or undefs according to the
  # contents of the global hashes DEFINE and UNDEF
  use strict;
  my ($job) = shift;
  my  $OPTS = shift;

  die "Empty job string supplied to update_cpp_i\n" unless $job;

  my $verbose = 0;
  $verbose = $OPTS->{VERBOSE} if defined $OPTS->{VERBOSE};

  my $jobname = '';
  $jobname = $OPTS->{JOBNAME} if defined $OPTS->{JOBNAME};

  # modified is used to return the status of the input job string
  # modified=0 means the string was not modified
  my $modified = 0;

  # Define delimiters for the CPP_I section
  my $addr1 = qr/(?:^|\n)[ \t]*## *CPP_I_START *\n/si;
  my $addr2 = qr/[ \t]*## *CPP_I_END *\n/si;

  # Extract the first CPP_I section, if any, found in this job string
  my ($cpp_start, $CPP_I, $cpp_end) = $job =~ m?($addr1)(.*)($addr2)?si;

  # If there is no CPP_I section return the original job string with a flag
  # indicating that the string was not modified
  my $found_cpp_i = 0;
  $found_cpp_i = 1 if ( ($cpp_start and $cpp_end) or $CPP_I );
  return wantarray ? ($job, $modified) : $job unless $found_cpp_i;

  if ($verbose > 10) {print "CPP_I:\n$cpp_start$CPP_I$cpp_end\n"}

  # Use these hashes to keep track of what gets changed
  my %setDEF;
  my %setUNDEF;

  # Update this CPP_I section
  my $new_CPP_I = '';
  foreach my $line (split /\n/,$CPP_I) {
    if ($line =~ /^#\s*(?:define|undef)\s+/) {
      # This is a macro define or undef
      my ($name) = $line =~ /^#\s*(?:define|undef)\s+(\w+)/;
      die "Invalid macro name found on CPP_I line: $_\n" unless $name;
      if (exists $main::DEFINE{$name}) {
        # Replace the current line with the new user specified define
        $line =~ s/^.*$/#define $name $main::DEFINE{$name}/;
        # Set a flag to indicate that the string was modified
        $modified = 1;
        # Keep track of macros defines that are reset
        $setDEF{$name} = $main::DEFINE{$name};
      }
      if (exists $main::UNDEF{$name}) {
        # Replace the current line with an undef of this macro
        $line =~ s/^.*$/#undef $name/;
        # Set a flag to indicate that the string was modified
        $modified = 1;
        # Keep track of macros undefs that are reset
        $setUNDEF{$name} = 1;
      }
    }
    # Add the current (possibly modified) line to new_CPP_I
    $new_CPP_I .= "\n$line";
  }
  $new_CPP_I .= "\n";

  # Append lines containing define/undef for each user supplied define/undef
  # Note that undefs are added after defines, so if a macro is both defined
  # and undef'd, it will be undef'd in any source code that includes CPP_I
  foreach my $name (keys %main::DEFINE) {
    # Append this define
    $new_CPP_I .= "\n#define $name $main::DEFINE{$name}";
    # Set a flag to indicate that the string was modified
    $modified = 1;
    $setDEF{$name} = $main::DEFINE{$name};
  }
  foreach my $name (keys %main::UNDEF) {
    # Append this undef
    $new_CPP_I .= "\n#undef $name";
    # Set a flag to indicate that the string was modified
    $modified = 1;
    $setUNDEF{$name} = 1;
  }
  $new_CPP_I .= "\n" if (scalar(keys %main::DEFINE) or scalar(keys %main::UNDEF));

  if ($verbose > 10) {
    print "new_CPP_I:\n$cpp_start$new_CPP_I$cpp_end\n";
  }

  if ($modified) {
    # Replace the CPP_I section in the input job string with
    # the modified version, but only if changes were made.
    $job =~ s?$addr1.*$addr2?$cpp_start$new_CPP_I$cpp_end?si;

    # Display macro names that have been modified
    if ($main::Verbose > 0) {
      print "Updated CPP_I section";
      if ($jobname) {
        print " in --> $jobname <--";
      } else {
        print ".";
      }
      print " The following lines have been added:\n";
      foreach my $mac ( sort keys %setDEF ) {
        print "      #define $mac $setDEF{$mac}\n";
      }
      foreach my $mac ( sort keys %setUNDEF ) {
        print "      #undef $mac\n";
      }
      print "\n";
    }
  }

  # Return the modified job string along with a flag
  # indicating if the string was modified or not
  return wantarray ? ($job, $modified) : $job;

}

# Ensure a true value is returned
1;