Eac dev (#5)

* Get metadata for butlerized exposures

* Clean up template code

* Allow for getting templates for super* files for butler

* Get first item off Butler.get('raw_filename') to use as template

* added get_filename_from_id to butler_utils

* fix typos in superdark.py and superflat.py

* Added stuff from slurm batch submission

* Fix typo in superflat

* Fix up slurm stuff

* typo in slurm stuff

* Fix up slurm stuff

* Switched to using isinstance to determine image object type

* Fix up use of batch system

* Fix up calls to extract_raft... and plot_ccd

bin/eo_plot_ccd.py

@@ -48,13 +48,13 @@ def main():
                              superbias_frame=superbias_frame,
                              vmin=args.vmin, vmax=args.vmax)
     else:
-        figs.plot_sensor("img", None, ccd,
+        figs.plot_sensor("img", ccd,
                          vmin=args.vmin, vmax=args.vmax,
                          bias=args.bias, superbias_frame=superbias_frame,
                          subtract_mean=args.subtract_mean)
 
     if args.stats_hist:
-        figs.histogram_array("hist", None, ccd,
+        figs.histogram_array("hist", ccd,
                              bias=args.bias, superbias_frame=superbias_frame,
                              xlabel="Counts", ylabel="Pixels / bin",
                              vmin=args.vmin, vmax=args.vmax,

python/lsst/eo_utils/base/analysis.py

@@ -246,9 +246,9 @@ def get_superbias_amp_image(butler, superbias_frame, amp):
         """
         if superbias_frame is not None:
             if butler is not None:
-                superbias_im = get_raw_image(None, superbias_frame, amp+1)
+                superbias_im = get_raw_image(superbias_frame, amp+1)
             else:
-                superbias_im = get_raw_image(None, superbias_frame, amp)
+                superbias_im = get_raw_image(superbias_frame, amp)
         else:
             superbias_im = None
         return superbias_im

python/lsst/eo_utils/base/batch_utils.py

@@ -13,9 +13,61 @@
 import os
 import sys
 
-#import pipes
 
-#from lsst.ctrl.pool import Batch, exportEnv, UMASK
+from lsst.eo_utils.base.defaults import BATCH_SYSTEM
+from lsst.eo_utils.base.file_utils import makedir_safe
+
+
+
+def write_slurm_batchfile(jobname, logfile, **kwargs):
+    """Dispatch a single job
+
+    Parameters
+    ----------
+    jobname : `str`
+        The command to run the job
+    logfile : `str`
+        The path to the logfile
+
+    Keywords
+    --------
+    run : `str`
+        The run number
+    batch_args : `str`
+        Arguments to pass to batch command
+    optstring : `str`
+        Additional arguments to pass to the command
+
+    Returns
+    -------
+    batchfile : `str`
+        The name of the slurm submission script file
+    """
+    run = kwargs.get('run', None)
+    batch_args = kwargs.get('batch_args', None)
+    optstring = kwargs.get('optstring', None)
+    batchfile = os.path.join('sbatch', logfile.replace('.log', '.sl'))
+
+    makedir_safe(batchfile)
+    fout = open(batchfile, 'w')
+    fout.write("#!/bin/bash -l\n")
+    tokens = batch_args.split()
+    for key, val in zip(tokens[0::2], tokens[1::2]):
+        fout.write("#SBATCH %s %s\n" % (key, val))
+    fout.write("\n")
+    sub_com = "srun --output %s" % logfile
+
+    if run is None:
+        sub_com += " %s" % jobname
+    else:
+        sub_com += " %s --run %s" % (jobname, run)
+    sub_com += " %s" % optstring
+
+    fout.write("%s\n" % sub_com)
+    fout.close()
+    return batchfile
+
+
 
 def dispatch_job(jobname, logfile, **kwargs):
     """Dispatch a single job
@@ -32,7 +84,7 @@ def dispatch_job(jobname, logfile, **kwargs):
     run : `str`
         The run number
     batch : `str`
-        Where to send the jobs
+        Send jobs to batch farm
     batch_args : `str`
         Arguments to pass to batch command
     optstring : `str`
@@ -41,25 +93,32 @@ def dispatch_job(jobname, logfile, **kwargs):
         Print command but do not run it
     """
     run = kwargs.get('run', None)
-    batch = kwargs.get('batch', 'native')
     batch_args = kwargs.get('batch_args', None)
     optstring = kwargs.get('optstring', None)
     dry_run = kwargs.get('dry_run', False)
 
-    if batch.find('bsub') >= 0:
+    disptach = 'native'
+    if kwargs.get('batch', False):
+        disptach = BATCH_SYSTEM
+
+    if disptach.find('lsf') == 0:
         sub_com = "bsub -o %s" % logfile
         if batch_args is not None:
             sub_com += " %s " % batch_args
+    elif disptach.find('slurm') == 0:
+        batchfile = write_slurm_batchfile(jobname, logfile, **kwargs)
+        sub_com = "sbatch %s" % batchfile
     else:
         sub_com = ""
 
-    if run is None:
-        sub_com += " %s" % jobname
-    else:
-        sub_com += " %s --run %s" % (jobname, run)
+    if disptach.find('slurm') < 0:
+        if run is None:
+            sub_com += " %s" % jobname
+        else:
+            sub_com += " %s --run %s" % (jobname, run)
 
-    if optstring is not None:
-        sub_com += " %s" % optstring
+        if optstring is not None:
+            sub_com += " %s" % optstring
 
     if dry_run:
         sys.stdout.write("%s\n" % sub_com)

python/lsst/eo_utils/base/butler_utils.py

@@ -32,6 +32,24 @@ def get_butler_by_repo(repo, **kwargs):
     return butler
 
 
+def get_filename_from_id(butler, data_id):
+    """Return the hardware type and hardware id for a given run
+
+    Parameters
+    ----------
+    butler : `Bulter`
+        The data Butler
+    data_id : `dict`
+        The data id in question
+
+    Returns
+    -------
+    filename : `str`
+        The filename for the assocated CCD data
+    """
+    return butler.get("raw_filename", data_id)[0][0:-3]
+
+
 def get_hardware_info(butler, run_num):
     """Return the hardware type and hardware id for a given run
 

python/lsst/eo_utils/base/defaults.py

@@ -1,20 +1,26 @@
 """This module manage the default settings for eo_utils module"""
 
-
-# Location of various butler Repos
-BUTLER_TS8_REPO = '/gpfs/slac/lsst/fs3/g/data/datasets/ts8'
-BUTLER_BOT_REPO = '/gpfs/slac/lsst/fs3/g/data/datasets/bot'
-BUTLER_TS8_NCSA_REPO = '/datasets/ts8/repo'
-BUTLER_BOT_NCSA_REPO = '/datasets/bot/repo'
-
-# Location of slac archive
-ARCHIVE_SLAC = '/gpfs/slac/lsst/fs*/g/data/jobHarness/jh_archive*'
+import os
+
+# SITE DEPENDENT STUFF
+SITE = os.environ.get('EO_UTILS_SITE', 'slac')
+
+if SITE == 'slac':
+    BUTLER_TS8_REPO = '/gpfs/slac/lsst/fs3/g/data/datasets/ts8'
+    BUTLER_BOT_REPO = '/gpfs/slac/lsst/fs3/g/data/datasets/bot'
+    ARCHIVE_DIR = '/gpfs/slac/lsst/fs*/g/data/jobHarness/jh_archive*'
+    DEFAULT_BATCH_ARGS = '-W 1200 -R bullet'
+    BATCH_SYSTEM = 'lsf'
+elif SITE == 'ncsa':
+    BUTLER_TS8_REPO = '/datasets/ts8/repo'
+    BUTLER_BOT_REPO = '/datasets/bot/repo'
+    ARCHIVE_DIR = None
+    DEFAULT_BATCH_ARGS = ""
+    BATCH_SYSTEM = 'slurm'
 
 # Map the Butler repos to simple names
 BUTLER_REPO_DICT = dict(TS8=BUTLER_TS8_REPO,
-                        BOT=BUTLER_BOT_REPO,
-                        TS8_NCSA=BUTLER_TS8_NCSA_REPO,
-                        BOT_NCSA=BUTLER_BOT_NCSA_REPO)
+                        BOT=BUTLER_BOT_REPO)
 
 # The slots
 ALL_SLOTS = ['S00', 'S01', 'S02', 'S10', 'S11', 'S12', 'S20', 'S21', 'S22']
@@ -56,12 +62,6 @@
                     FE55="cyan")
 
 
-# Template to make superbias files
-SBIAS_TEMPLATE = 'analysis/superbias/templates/sbias_template.fits'
-SFLAT_TEMPLATE = 'analysis/superflat/templates/sflat_template.fits'
-SDARK_TEMPLATE = 'analysis/superdark/templates/sdark_template.fits'
-
-
 # Some default values
 DEFAULT_OUTDIR = 'analysis/ts8'
 DEFAULT_STAT_TYPE = 'median'
@@ -70,4 +70,3 @@
 DEFAULT_SUPERBIAS_TYPE = None
 DEFAULT_LOGFILE = 'temp.log'
 DEFAULT_NBINS = 100
-DEFAULT_BATCH_ARGS = "-W 1200 -R bullet"

python/lsst/eo_utils/base/file_utils.py

@@ -18,7 +18,7 @@
     print("Warning, no datacat-utilities")
 
 
-from .defaults import ALL_RAFTS, ALL_SLOTS, ARCHIVE_SLAC
+from .defaults import ALL_RAFTS, ALL_SLOTS, ARCHIVE_DIR
 
 
 # These are the standard input filenames
@@ -289,10 +289,10 @@ def add_format(self, key, fmt, **kwargs):
                                                        bias=None, suffix='')
 TS8_FORMATTER = FILENAME_FORMATS.add_format('ts8_images',
                                             TS8_GLOB_STRING,
-                                            archive=ARCHIVE_SLAC)
+                                            archive=ARCHIVE_DIR)
 BOT_FORMATTER = FILENAME_FORMATS.add_format('bot_images',
                                             BOT_GLOB_STRING,
-                                            archive=ARCHIVE_SLAC)
+                                            archive=ARCHIVE_DIR)
 
 TS8_EORESULTSIN_FORMATTER = FILENAME_FORMATS.add_format('ts8_eoresults_in',
                                                         SLOT_FORMAT_STRING,