Fix the start_time and file_duration keys in diag_manager

diag_manager/Makefile.am

@@ -69,7 +69,7 @@ diag_util_mod.$(FC_MODEXT): diag_data_mod.$(FC_MODEXT) diag_axis_mod.$(FC_MODEXT
                             diag_grid_mod.$(FC_MODEXT) fms_diag_time_utils_mod.$(FC_MODEXT) fms_diag_bbox_mod.$(FC_MODEXT)
 fms_diag_time_utils_mod.$(FC_MODEXT): diag_data_mod.$(FC_MODEXT)
 diag_table_mod.$(FC_MODEXT): diag_data_mod.$(FC_MODEXT) diag_util_mod.$(FC_MODEXT)
-fms_diag_yaml_mod.$(FC_MODEXT): diag_data_mod.$(FC_MODEXT)
+fms_diag_yaml_mod.$(FC_MODEXT): diag_data_mod.$(FC_MODEXT)  fms_diag_time_utils_mod.$(FC_MODEXT)
 fms_diag_object_mod.$(FC_MODEXT): diag_data_mod.$(FC_MODEXT) fms_diag_file_object_mod.$(FC_MODEXT) fms_diag_field_object_mod.$(FC_MODEXT) fms_diag_yaml_mod.$(FC_MODEXT) \
                                   fms_diag_time_utils_mod.$(FC_MODEXT) \
                                   fms_diag_output_buffer_mod.$(FC_MODEXT) \

diag_manager/fms_diag_file_object.F90

@@ -186,7 +186,7 @@ module fms_diag_file_object_mod
   procedure :: write_field_metadata
   procedure :: write_axis_data
   procedure :: writing_on_this_pe
-  procedure :: is_time_to_write
+  procedure :: check_file_times
   procedure :: is_time_to_close_file
   procedure :: write_time_data
   procedure :: update_next_write
@@ -200,6 +200,7 @@ module fms_diag_file_object_mod
   procedure :: close_diag_file
   procedure :: set_model_time
   procedure :: get_model_time
+  procedure :: time_to_start_doing_math
 end type fmsDiagFileContainer_type
 
 !type(fmsDiagFile_type), dimension (:), allocatable, target :: FMS_diag_file !< The array of diag files
@@ -264,8 +265,12 @@ logical function fms_diag_files_object_init (files_array)
      !! Set this to the time passed in to diag_manager_init
      !! This will be the base_time if nothing was passed in
      !! This time is appended to the filename if the prepend_date namelist is .True.
-     obj%start_time = diag_init_time
-     obj%last_output = diag_init_time
+     if (obj%has_file_start_time()) then
+       obj%start_time = obj%get_file_start_time()
+     else
+       obj%start_time = diag_init_time
+     endif
+     obj%last_output = obj%start_time
      obj%model_time = diag_init_time
      obj%next_output = diag_time_inc(obj%start_time, obj%get_file_freq(), obj%get_file_frequnit())
      obj%next_next_output = diag_time_inc(obj%next_output, obj%get_file_freq(), obj%get_file_frequnit())
@@ -274,7 +279,12 @@ logical function fms_diag_files_object_init (files_array)
        obj%next_close = diag_time_inc(obj%start_time, obj%get_file_new_file_freq(), &
                                         obj%get_file_new_file_freq_units())
      else
-       obj%next_close = diag_time_inc(obj%start_time, VERY_LARGE_FILE_FREQ, DIAG_DAYS)
+       if (obj%has_file_duration()) then
+         obj%next_close = diag_time_inc(obj%start_time, obj%get_file_duration(), &
+                                        obj%get_file_duration_units())
+       else
+         obj%next_close = diag_time_inc(obj%start_time, VERY_LARGE_FILE_FREQ, DIAG_DAYS)
+       endif
      endif
      obj%is_file_open = .false.
 
@@ -597,7 +607,7 @@ end function get_file_new_file_freq_units
 !! \return Copy of file_start_time
 pure function get_file_start_time (this) result(res)
  class(fmsDiagFile_type), intent(in) :: this !< The file object
- character (len=:), allocatable :: res
+ type(time_type):: res
   res = this%diag_yaml_file%get_file_start_time()
 end function get_file_start_time
 
@@ -639,6 +649,7 @@ pure function is_done_writing_data (this) result(res)
  class(fmsDiagFile_type), intent(in) :: this !< The file object
  logical :: res
   res = this%done_writing_data
+  if (this%is_file_open) res = .false.
 end function is_done_writing_data
 
 !> \brief Checks if file_fname is allocated in the yaml object
@@ -1014,6 +1025,11 @@ subroutine add_start_time(this, start_time)
   !! this%start_time was already set to the diag_init_time
   if (start_time .eq. diag_init_time) return
 
+  !< If the start_time sent is is greater than or equal to the start time already
+  !! in the diag file obj return because either this%start_time was already updated
+  !! or the file has start_time defined in the yaml
+  if (this%start_time >= start_time) return
+
   if (this%start_time .ne. diag_init_time) then
     !> If the this%start_time is not equal to the diag_init_time from the diag_table
     !! this%start_time was already updated so make sure it is the same for the current variable
@@ -1398,8 +1414,18 @@ logical function is_time_to_close_file (this, time_step)
   endif
 end function
 
+!> \brief Determine if it is time to start doing mathz
+!! \return .True. if it is time to start doing mathz
+logical function time_to_start_doing_math (this)
+  class(fmsDiagFileContainer_type), intent(in), target   :: this            !< The file object
+  time_to_start_doing_math = .false.
+  if (this%FMS_diag_file%model_time >= this%FMS_diag_file%start_time) then
+    time_to_start_doing_math = .true.
+  endif
+end function
+
 !> \brief Determine if it is time to "write" to the file
-logical function is_time_to_write(this, time_step, output_buffers, diag_fields, do_not_write)
+subroutine check_file_times(this, time_step, output_buffers, diag_fields, do_not_write)
   class(fmsDiagFileContainer_type), intent(inout), target   :: this              !< The file object
   TYPE(time_type),                  intent(in)              :: time_step         !< Current model step time
   type(fmsDiagOutputBuffer_type),   intent(in)              :: output_buffers(:) !< Array of output buffer.
@@ -1411,7 +1437,6 @@ logical function is_time_to_write(this, time_step, output_buffers, diag_fields,
 
   do_not_write = .false.
   if (time_step > this%FMS_diag_file%next_output) then
-    is_time_to_write = .true.
     if (this%FMS_diag_file%is_static) return
     if (time_step > this%FMS_diag_file%next_next_output) then
       if (this%FMS_diag_file%get_file_freq() .eq. 0) then
@@ -1421,9 +1446,7 @@ logical function is_time_to_write(this, time_step, output_buffers, diag_fields,
           call this%FMS_diag_file%check_buffer_times(output_buffers, diag_fields)
           this%FMS_diag_file%next_output = time_step
           this%FMS_diag_file%next_next_output = time_step
-          is_time_to_write = .true.
         endif
-        return
       elseif (this%FMS_diag_file%num_registered_fields .eq. 0) then
         !! If no variables have been registered, write a dummy time dimension for the first level
         !! At least one time level is needed for the combiner to work ...
@@ -1433,26 +1456,20 @@ logical function is_time_to_write(this, time_step, output_buffers, diag_fields,
           this%FMS_diag_file%data_has_been_written = .true.
           this%FMS_diag_file%unlim_dimension_level = 1
         endif
-        is_time_to_write =.false.
       else
         !! Only fail if send data has actually been called for at least one variable
         if (this%FMS_diag_file%has_send_data_been_called(output_buffers, .false.)) &
           call mpp_error(FATAL, this%FMS_diag_file%get_file_fname()//&
             ": diag_manager_mod: You skipped a time_step. Be sure that diag_send_complete is called at every "//&
             "time_step needed by the file.")
-        is_time_to_write =.false.
       endif
     endif
   else
-    is_time_to_write = .false.
-    if (this%FMS_diag_file%is_static) then
-      ! This is to ensure that static files get finished in the begining of the run
-      if (this%FMS_diag_file%unlim_dimension_level .eq. 1) is_time_to_write = .true.
-    else if(this%FMS_diag_file%get_file_freq() .eq. 0) then
+    if(this%FMS_diag_file%get_file_freq() .eq. 0) then
       do_not_write = .true.
     endif
   endif
-end function is_time_to_write
+end subroutine check_file_times
 
 !> \brief Determine if the current PE has data to write
 logical function writing_on_this_pe(this)
@@ -1532,7 +1549,6 @@ subroutine update_current_new_file_freq_index(this, time_step)
        diag_file%next_output = diag_file%no_more_data
        diag_file%next_next_output = diag_file%no_more_data
        diag_file%last_output = diag_file%no_more_data
-       diag_file%next_close = diag_file%no_more_data
     endif
   endif
 
@@ -1806,7 +1822,7 @@ subroutine close_diag_file(this, output_buffers, model_end_time, diag_fields)
                                         this%FMS_diag_file%get_file_new_file_freq(), &
                                         this%FMS_diag_file%get_file_new_file_freq_units())
   else
-    this%FMS_diag_file%next_close = diag_time_inc(this%FMS_diag_file%next_close, VERY_LARGE_FILE_FREQ, DIAG_DAYS)
+    this%FMS_diag_file%next_close = model_end_time
   endif
 
   if (this%FMS_diag_file%model_time >= model_end_time) &

diag_manager/fms_diag_object.F90

@@ -828,14 +828,14 @@ subroutine fms_diag_do_io(this, end_time)
 
     !< Go away if the file is a subregional file and the current PE does not have any data for it
     if (.not. diag_file%writing_on_this_pe()) cycle
-    if (diag_file%FMS_diag_file%is_done_writing_data()) cycle
 
     if (present (end_time)) then
       force_write = .true.
       model_time => end_time
     else
       model_time => diag_file%get_model_time()
     endif
+    if (diag_file%FMS_diag_file%is_done_writing_data()) cycle
 
     call diag_file%open_diag_file(model_time, file_is_opened_this_time_step)
     if (file_is_opened_this_time_step) then
@@ -849,8 +849,7 @@ subroutine fms_diag_do_io(this, end_time)
       call diag_file%write_axis_data(this%diag_axis)
     endif
 
-    !TODO this can be clean up and better documented
-    finish_writing = diag_file%is_time_to_write(model_time, this%FMS_diag_output_buffers, &
+    call diag_file%check_file_times(model_time, this%FMS_diag_output_buffers, &
       this%FMS_diag_fields, do_not_write)
     unlim_dim_was_increased = .false.
 
@@ -985,6 +984,7 @@ function fms_diag_do_reduction(this, field_data, diag_field_id, oor_mask, weight
     if (buffer_ptr%is_done_with_math()) cycle
 
     if (present(time)) call file_ptr%set_model_time(time)
+    if (.not. file_ptr%time_to_start_doing_math()) cycle
 
     bounds_out = bounds
     if (.not. using_blocking) then

diag_manager/fms_diag_time_utils.F90

@@ -27,7 +27,7 @@
 module fms_diag_time_utils_mod
 
 use time_manager_mod, only: time_type, increment_date, increment_time, get_calendar_type, NO_CALENDAR, leap_year, &
-                            get_date, get_time,  operator(>), operator(<), operator(-), set_date
+                            get_date, get_time,  operator(>), operator(<), operator(-), set_date, set_time
 use diag_data_mod,    only: END_OF_RUN, EVERY_TIME, DIAG_SECONDS, DIAG_MINUTES, DIAG_HOURS, DIAG_DAYS, DIAG_MONTHS, &
                             DIAG_YEARS, use_clock_average
 USE constants_mod,    ONLY: SECONDS_PER_DAY, SECONDS_PER_HOUR, SECONDS_PER_MINUTE
@@ -40,6 +40,7 @@ module fms_diag_time_utils_mod
 public :: diag_time_inc
 public :: get_time_string
 public :: get_date_dif
+public :: set_time_type
 
 contains
 
@@ -383,4 +384,38 @@ REAL FUNCTION get_date_dif(t2, t1, units)
        CALL mpp_error(FATAL, 'diag_util_mod::diag_date_dif illegal time units')
     END IF
   END FUNCTION get_date_dif
+
+!> @brief Sets up a time_type based on 6 member array of integers defining the
+!! [year month day hour min sec]
+subroutine set_time_type(time_int, time)
+  integer,         intent(in)    :: time_int(6) !< The time in the format [year month day hour min second]
+  type(time_type), intent(inout) :: time        !< The time converted to the time_type
+
+  integer :: year   !< Year of the time type
+  integer :: month  !< Month of the time type
+  integer :: day    !< Day of the time type
+  integer :: hour   !< Hour of the time type
+  integer :: minute !< Minute of the time type
+  integer :: second !< Second of the time type
+
+  year = time_int(1)
+  month = time_int(2)
+  day = time_int(3)
+  hour = time_int(4)
+  minute = time_int(5)
+  second = time_int(6)
+
+  ! Set up the time type for time passed in
+  IF ( get_calendar_type() /= NO_CALENDAR ) THEN
+    IF ( year==0 .OR. month==0 .OR. day==0 ) THEN
+       call mpp_error(FATAL, 'fms_diag_time_utils_mod::set_time_type'//&
+          &  'The year/month/day can not equal zero')
+    END IF
+    time = set_date(year, month, day, hour, minute, second)
+  ELSE
+    ! No calendar - ignore year and month
+    time = set_time(NINT(hour*SECONDS_PER_HOUR)+NINT(minute*SECONDS_PER_MINUTE)+second, &
+                        &  day)
+  END IF
+end subroutine set_time_type
 end module fms_diag_time_utils_mod

diag_manager/fms_diag_yaml.F90

@@ -45,6 +45,8 @@ module fms_diag_yaml_mod
                                 fms_f2c_string
 use platform_mod, only: r4_kind, i4_kind, r8_kind, i8_kind, FMS_FILE_LEN
 use fms_mod, only: lowercase
+use fms_diag_time_utils_mod, only: set_time_type
+use time_manager_mod, only: time_type, date_to_string
 
 implicit none
 
@@ -111,8 +113,9 @@ module fms_diag_yaml_mod
                                                                          !! Required if “new_file_freq” used
                                                                          !! (DIAG_SECONDS, DIAG_MINUTES, &
                                                                          !! DIAG_HOURS, DIAG_DAYS, DIAG_YEARS)
-  character (len=:),   allocatable :: file_start_time                    !< Time to start the file for the
-                                                                         !! first time. Requires “new_file_freq”
+  type(time_type)                  :: file_start_time                    !< Time to start the file for the
+                                                                         !! first time.
+  logical                          :: file_start_time_set                !< .True. if file_start_time has been set
   integer                          :: filename_time                      !< The time to use when setting the name of
                                                                          !! new files: begin, middle, or end of the
                                                                          !! time_bounds
@@ -553,6 +556,7 @@ subroutine fill_in_diag_files(diag_yaml_id, diag_file_id, yaml_fileobj)
   integer, allocatable :: key_ids(:) !< Id of the gloabl atttributes key/value pairs
   character(len=:), ALLOCATABLE :: grid_type !< grid_type as it is read in from the yaml
   character(len=:), ALLOCATABLE :: buffer      !< buffer to store any *_units as it is read from the yaml
+  integer :: start_time_int(6) !< The start_time as read in from the diag_table yaml
 
   yaml_fileobj%file_frequnit = 0
 
@@ -575,8 +579,14 @@ subroutine fill_in_diag_files(diag_yaml_id, diag_file_id, yaml_fileobj)
   call set_filename_time(yaml_fileobj, buffer)
   deallocate(buffer)
 
-  call diag_get_value_from_key(diag_yaml_id, diag_file_id, "start_time", &
-       yaml_fileobj%file_start_time, is_optional=.true.)
+  start_time_int = diag_null
+  yaml_fileobj%file_start_time_set = .false.
+  call get_value_from_key(diag_yaml_id, diag_file_id, "start_time", &
+    start_time_int, is_optional=.true.)
+  if (any(start_time_int .ne. diag_null)) then
+    yaml_fileobj%file_start_time_set = .true.
+    call set_time_type(start_time_int, yaml_fileobj%file_start_time)
+   endif
   call diag_get_value_from_key(diag_yaml_id, diag_file_id, "file_duration", buffer, is_optional=.true.)
   call parse_key(yaml_fileobj%file_fname, buffer, yaml_fileobj%file_duration, yaml_fileobj%file_duration_units, &
     "file_duration")
@@ -1062,7 +1072,7 @@ end function get_file_new_file_freq_units
 pure function get_file_start_time (this) &
 result (res)
  class (diagYamlFiles_type), intent(in) :: this !< The object being inquiried
- character (len=:), allocatable :: res !< What is returned
+ type(time_type) :: res !< What is returned
   res = this%file_start_time
 end function get_file_start_time
 !> @brief Inquiry for diag_files_obj%file_duration
@@ -1317,7 +1327,7 @@ end function has_file_new_file_freq_units
 !! @return true if diag_file_obj%file_start_time is allocated
 pure logical function has_file_start_time (this)
   class(diagYamlFiles_type), intent(in) :: this !< diagYamlFiles_type object to initialize
-  has_file_start_time = allocated(this%file_start_time)
+  has_file_start_time = this%file_start_time_set
 end function has_file_start_time
 !> @brief diag_file_obj%file_duration is allocated on th stack, so this is always true
 !! @return true
@@ -1585,7 +1595,7 @@ subroutine dump_diag_yaml_obj( filename )
       if(files(i)%has_file_new_file_freq()) write(unit_num, *) 'new_file_freq:', files(i)%get_file_new_file_freq()
       if(files(i)%has_file_new_file_freq_units()) write(unit_num, *) 'new_file_freq_units:', &
                                                          & files(i)%get_file_new_file_freq_units()
-      if(files(i)%has_file_start_time()) write(unit_num, *) 'start_time:', files(i)%get_file_start_time()
+      if(files(i)%has_file_start_time()) write(unit_num, *) 'start_time:', date_to_string(files(i)%get_file_start_time())
       if(files(i)%has_file_duration()) write(unit_num, *) 'duration:', files(i)%get_file_duration()
       if(files(i)%has_file_duration_units()) write(unit_num, *) 'duration_units:', files(i)%get_file_duration_units()
       if(files(i)%has_file_varlist()) write(unit_num, *) 'varlist:', files(i)%get_file_varlist()
@@ -1712,8 +1722,11 @@ subroutine fms_diag_yaml_out()
     enddo
     call fms_f2c_string(vals2(i)%val6, adjustl(tmpstr1))
     call fms_f2c_string(vals2(i)%val7, get_diag_unit_string(fileptr%file_new_file_freq_units))
-    call fms_f2c_string(vals2(i)%val8, trim(fileptr%get_file_start_time()))
-    st_vals(i) = fileptr%get_file_start_time()
+    if (fileptr%has_file_start_time()) then
+      call fms_f2c_string(vals2(i)%val8, trim(date_to_string(fileptr%get_file_start_time())))
+    else
+      call fms_f2c_string(vals2(i)%val8, "")
+    endif
     tmpstr1 = ''
     do k=1, SIZE(fileptr%file_duration)
         if(fileptr%file_duration(k) .eq. diag_null) exit

test_fms/diag_manager/Makefile.am

@@ -34,7 +34,7 @@ check_PROGRAMS = test_diag_manager test_diag_manager_time \
  check_time_min check_time_max check_time_sum check_time_avg test_diag_diurnal check_time_diurnal \
  check_time_pow check_time_rms check_subregional test_cell_measures test_var_masks \
  check_var_masks test_multiple_send_data test_diag_out_yaml test_output_every_freq \
- test_dm_weights test_prepend_date
+ test_dm_weights test_prepend_date test_multi_file
 
 # This is the source code for the test.
 test_output_every_freq_SOURCES = test_output_every_freq.F90
@@ -65,6 +65,7 @@ test_var_masks_SOURCES = test_var_masks.F90
 check_var_masks_SOURCES = check_var_masks.F90
 test_multiple_send_data_SOURCES = test_multiple_send_data.F90
 test_prepend_date_SOURCES = test_prepend_date.F90
+test_multi_file_SOURCES = test_multi_file.F90
 
 TEST_EXTENSIONS = .sh
 SH_LOG_DRIVER = env AM_TAP_AWK='$(AWK)' $(SHELL) \
@@ -74,15 +75,16 @@ SH_LOG_DRIVER = env AM_TAP_AWK='$(AWK)' $(SHELL) \
 TESTS = test_diag_manager2.sh test_time_none.sh test_time_min.sh test_time_max.sh test_time_sum.sh \
         test_time_avg.sh test_time_pow.sh test_time_rms.sh test_time_diurnal.sh test_cell_measures.sh \
         test_subregional.sh test_var_masks.sh test_multiple_send_data.sh test_output_every_freq.sh \
-        test_dm_weights.sh test_flush_nc_file.sh test_prepend_date.sh
+        test_dm_weights.sh test_flush_nc_file.sh test_prepend_date.sh test_multi_file.sh
 
 testing_utils.mod: testing_utils.$(OBJEXT)
 
 # Copy over other needed files to the srcdir
 EXTRA_DIST = test_diag_manager2.sh check_crashes.sh test_time_none.sh test_time_min.sh test_time_max.sh \
              test_time_sum.sh test_time_avg.sh test_time_pow.sh test_time_rms.sh test_time_diurnal.sh \
              test_cell_measures.sh test_subregional.sh test_var_masks.sh test_multiple_send_data.sh \
-             test_flush_nc_file.sh test_dm_weights.sh test_output_every_freq.sh test_prepend_date.sh
+             test_flush_nc_file.sh test_dm_weights.sh test_output_every_freq.sh test_prepend_date.sh \
+             test_multi_file.sh
 
 if USING_YAML
 skipflag=""