Eac dev (#20)

* Added data_source to called to get data

* account for different indexing in superbias image v. bulterized ccd image

* Fixed notebooks for ncsa

* Updated NCSA ts8 butler repo location

* delinting

* Skip glob based tests when not at slac

* Don't run file linking tests when not at SLAC

* Skip tests that require non-bulter data when not at SLAC

* Changes for NCSA unit tests

* whitespace cleanup

* Fixes for masking and delinting

* Fixes for BOT running with some rebs off

* fixes for BOT running at NCSA

* Allow for sending jobs to batch by raft

* fix batch job output files

* various changes

* changes to dispatch jobs over each CCD individually

* return the full VARIANCE when making FlatPair statistics

* Added null point correction in nonlinearity task

* changes for NCSA slurm

* Fix typo in raft list

* Fix up batch parallelization

* Added analysis to .gitignore

* Change cut between low and hig flats to be 20s

* Fixes for superbias_stats

* Added NAMPS header keyword to file generated by flat_pair.py

* Fixes for nine raft testing and to handle bias subtraction of butlerized data

* Fixes for nine raft testing and to handle bias subtraction of butlerized data

* fix tests/test_02_bias.py

* Fix passing teststand to get_butler function

* fix manufacturer lookup

* Fix a couple of unit tests

* Allow stacking to skip corrupted files

* Change to using LSST_NUM to identify CCD type

* fix log warning

* Changes to pass along data_source to analysis classes

* Fixes for using data_source==butler_file

* update .pylintrc to include more generated functions

* Delinting

* update .pylintrc to allow for longer functions

* Delinting

* delinting

* Slurm changes

* Fixes for running with data_source=butler_file

.pylintrc

@@ -259,7 +259,7 @@ ignored-classes=
 # List of members which are set dynamically and missed by pylint inference
 # system, and so shouldn't trigger E1101 when accessed. Python regular
 # expressions are accepted.
-generated-members=StatisticsControl,makeStatistics,MEAN,MEDIAN,STDEVCLIP,Point2I,Extent2I,Box2I,Threshold,FootprintSet,data
+generated-members=StatisticsControl,makeStatistics,MEAN,MEDIAN,STDEVCLIP,VARIANCE,Point2I,Extent2I,Box2I,Threshold,FootprintSet,data
 
 
 
@@ -321,7 +321,7 @@ max-returns=6
 max-branches=25
 
 # Maximum number of statements in function / method body
-max-statements=70
+max-statements=100
 
 # Maximum number of parents for a class (see R0901).
 max-parents=9
@@ -333,7 +333,7 @@ max-attributes=10
 min-public-methods=2
 
 # Maximum number of public methods for a class (see R0904).
-max-public-methods=35
+max-public-methods=50
 
 # Maximum number of boolean expressions in a if statement
 max-bool-expr=5

python/lsst/eo_utils/base/analysis.py

@@ -7,6 +7,7 @@
 
 import glob
 
+import numpy as np
 
 import lsst.pex.config as pexConfig
 
@@ -225,7 +226,6 @@ def __call__(self, butler, data, **kwargs):
         """
         raise NotImplementedError()
 
-
     def get_filename_from_format(self, formatter, suffix, **kwargs):
         """Use a `FilenameFormat` object to construct a filename for a
         specific set of input parameters.
@@ -256,34 +256,6 @@ def get_filename_from_format(self, formatter, suffix, **kwargs):
         return formatter(**format_vals)
 
 
-    @staticmethod
-    def get_superbias_amp_image(butler, superbias_frame, amp):
-        """Get the image for one amp for the superbias
-
-        Parameters
-        ----------
-        butler : `Butler` or `None`
-            Data Butler (or none)
-        superbias_frame : `MaskedCCD` or `None`
-            superbias image for the whole CCD
-        amp : `int`
-            Amplifier index
-
-        Returns
-        -------
-        superbias_im : `ImageF`
-            The image for the requested amplifier
-        """
-        if superbias_frame is not None:
-            if butler is not None:
-                superbias_im = get_raw_image(superbias_frame, amp+1)
-            else:
-                superbias_im = get_raw_image(superbias_frame, amp)
-        else:
-            superbias_im = None
-        return superbias_im
-
-
     def get_superbias_file(self, suffix, **kwargs):
         """Get the name of the superbias file for a particular run, raft, ccd...
 
@@ -499,6 +471,7 @@ def __init__(self, **kwargs):
             Used to override default configuration
         """
         BaseAnalysisTask.__init__(self, **kwargs)
+        self._handler_config = None
 
     def get_suffix(self, **kwargs):
         """Get the suffix to add to table and plot filenames
@@ -551,6 +524,36 @@ def plotfile_name(self, **kwargs):
                                              self.get_suffix(),
                                              **kwargs)
 
+
+    def get_superbias_amp_image(self, butler, superbias_frame, amp):
+        """Get the image for one amp for the superbias                                                                                                           
+
+        Parameters                                                                                                                                               
+        ----------                                                                                                                                                
+        butler : `Butler` or `None`                                                                                                                               
+            Data Butler (or none)                                                                                                                                 
+        superbias_frame : `MaskedCCD` or `None`                                                                                                                   
+            superbias image for the whole CCD                                                                                                                     
+        amp : `int`                                                                                                                                               
+            Amplifier index
+
+        Returns
+        -------
+        superbias_im : `ImageF`
+            The image for the requested amplifier
+        """                                                                                                                                                      
+        offset = 0
+        if self._handler_config is not None:
+            if self._handler_config.data_source == 'butler':
+                offset = 1
+
+        if superbias_frame is not None:                                                                                                                          
+            superbias_im = get_raw_image(superbias_frame, amp+offset)
+        else:
+            superbias_im = None                                                                                                                                   
+        return superbias_im                                                                                                                                      
+
+
     def get_superbias_frame(self, mask_files, **kwargs):
         """Get the superbias frame for a particular run, raft, ccd...
 
@@ -667,10 +670,47 @@ def __call__(self, butler, data, **kwargs):
             Used to override default configuration
          """
         self.safe_update(**kwargs)
+        self._handler_config = kwargs.get('handler_config', None)
         dtables = self.make_datatables(butler, data)
         if self.config.plot is not None:
             self.make_plots(dtables)
 
+
+    def get_ccd(self, butler, data_id, mask_files, **kwargs):
+        """CCD image from a data_id
+
+        If we are using `Butler` then this will take a
+        data_id `dict` ojbect and return an `ExposureF` object
+
+        If we are not using `Butler` (i.e., if bulter is `None`)
+        then this will take a filename and return a `MaskedCCD` object
+
+        Parameters
+        ----------
+        butler : `Butler` or `None`
+            Data Butler
+        data_id : `dict` or `str`
+            Data identier
+        mask_files : `list`
+            List of data_ids for the files to construct the pixel mask
+
+        Keywords
+        --------
+        bias_frame : `ExposureF` or `MaskedCCD` or `None`
+            Object with the bias data
+        masked_ccd : `bool`
+            Use bulter only to get filename, return as MaskedCCD object
+
+        Returns
+        -------
+        ccd : `ExposureF` or `MaskedCCD`
+            CCD data object
+        """
+        if self._handler_config is not None:
+            use_masked_ccd = self._handler_config.data_source == 'butler_file'
+            kwargs.setdefault('masked_ccd', use_masked_ccd)
+        return get_ccd_from_id(butler, data_id, mask_files, **kwargs)
+
     @abc.abstractmethod
     def extract(self, butler, data, **kwargs):
         """This needs to be implemented by the sub-class

python/lsst/eo_utils/base/batch_utils.py

@@ -107,7 +107,8 @@ def dispatch_job(jobname, logfile, **kwargs):
             sub_com += " %s " % batch_args
     elif disptach.find('slurm') == 0:
         batchfile = write_slurm_batchfile(jobname, logfile, **kwargs)
-        sub_com = "sbatch %s -o %s" % (batchfile, os.path.join('sbatch', logfile.replace('.fits', '.out')))
+        logfile_job = os.path.join('sbatch', logfile.replace('.fits', '.out'))
+        sub_com = "sbatch %s -o %s -e %s" % (batchfile, logfile_job, logfile_job)
     else:
         sub_com = ""
 

python/lsst/eo_utils/base/data_access.py

@@ -66,7 +66,7 @@ def get_trait(self, traitname, **kwargs):
         data_source = kwargs.get('data_source', DEFAULT_DATA_SOURCE)
         if teststand in ['bot_etu']:
             teststand = 'bot'
-        if data_source in ['butler', 'butler_files']:
+        if data_source in ['butler', 'butler_file']:
             data_source = 'butler'
 
         key = '%s_%s_%s' % (teststand, data_source, traitname)

python/lsst/eo_utils/base/defaults.py

@@ -15,21 +15,21 @@
     BUTLER_BOT_REPO = '/gpfs/slac/lsst/fs3/g/data/datasets/bot'
     ARCHIVE_DIR = '/gpfs/slac/lsst/fs*/g/data/jobHarness/jh_archive*'
     DEFAULT_DATA_SOURCE = os.environ.get('EO_DATA_SOURCE', 'glob')
-    DEFAULT_BATCH_ARGS = '-W 1200 -R bullet'
+    DEFAULT_BATCH_ARGS = '-W 1200 -R bubble'
     BATCH_SYSTEM = 'lsf'
 elif SITE == 'ncsa':
     BUTLER_TS8_REPO = '/datasets/ts8/repo'
     BUTLER_BOT_REPO = '/project/production/tmpdataloc/BOT/gen2repo'
     ARCHIVE_DIR = None
     DEFAULT_BATCH_ARGS = ""
-    DEFAULT_DATA_SOURCE = os.environ.get('EO_DATA_SOURCE', 'butler')
+    DEFAULT_DATA_SOURCE = os.environ.get('EO_DATA_SOURCE', 'butler_file')
     BATCH_SYSTEM = 'slurm'
 else:
     raise ValueError("Unknown site %s" % SITE)
 
 
 # TEST
-DEFAULT_TESTSTAND = os.environ.get('EO_TESTSTSAND', 'bot')
+DEFAULT_TESTSTAND = os.environ.get('EO_TESTSTAND', 'bot')
 
 if os.environ.get('EO_PRINT_OPTS', False):
     sys.stdout.write("SITE=%s\n" % SITE)
@@ -45,12 +45,18 @@
 # The slots
 ALL_SLOTS = ['S00', 'S01', 'S02', 'S10', 'S11', 'S12', 'S20', 'S21', 'S22']
 
+
 # The rafts
 ALL_RAFTS_BOT_ETU = ["R10", "R22"]
 NINE_RAFTS = ["R01", "R02", "R10",
               "R11", "R12", "R20",
               "R21", "R22", "R30"]
 
+
+RAFT_NAMES_DICT = dict(bot_etu=ALL_RAFTS_BOT_ETU,
+                       bot_9=NINE_RAFTS,
+                       bot=NINE_RAFTS)
+
 # Various types of tests
 TS8_MASK_TEST_TYPES = ['bright_pixel_mask',
                        'dark_pixel_mask',

python/lsst/eo_utils/base/file_utils.py

@@ -18,7 +18,7 @@
     print("Warning, no datacat-utilities")
 
 
-from .defaults import NINE_RAFTS, ALL_SLOTS, ARCHIVE_DIR
+from .defaults import ALL_SLOTS, ARCHIVE_DIR, RAFT_NAMES_DICT
 
 
 # These are the standard input filenames
@@ -367,7 +367,7 @@ def get_bot_files_glob(**kwargs):
     kwcopy = kwargs.copy()
     test_name = kwcopy.pop('testName').lower()
     nfiles = kwcopy.get('nfiles', None)
-    rafts = get_raft_names_dc(kwcopy['run'])
+    rafts = get_raft_names_dc(kwcopy['run'], kwcopy.get('teststand', 'bot'))
 
     for raft in rafts:
         raftdict = {}
@@ -458,6 +458,7 @@ def get_files_for_run(run_id, **kwargs):
     outkey = kwargs.get('outkey', imagetype)
     matchstr = kwargs.get('matchstr', None)
     nfiles = kwargs.get('nfiles', None)
+    teststand = kwargs.get('teststand', 'bot')
 
     outdict = {}
 
@@ -485,11 +486,13 @@ def get_files_for_run(run_id, **kwargs):
                 r_dict = get_ts8_files_glob(run=run_id,
                                             testName=test_type,
                                             imgtype=imgtype,
-                                            raft=hinfo[1])
+                                            raft=hinfo[1],
+                                            teststand=teststand)
             else:
                 r_dict = get_bot_files_glob(run=run_id,
                                             testName=test_type,
-                                            imgtype=imgtype)
+                                            imgtype=imgtype,
+                                            teststand=teststand)
         for key, val in r_dict.items():
             if hinfo[0] == 'LCA-11021':
                 # Raft level data
@@ -644,7 +647,7 @@ def read_runlist(filepath):
     return outlist
 
 
-def get_raft_names_dc(run):
+def get_raft_names_dc(run, teststand='bot'):
     """Get the list of rafts used for a particular run
 
     Parameters
@@ -668,7 +671,7 @@ def get_raft_names_dc(run):
     if htype == 'LCA-11021':
         return [hid]
     if htype == 'LCA-10134':
-        return NINE_RAFTS
+        return RAFT_NAMES_DICT[teststand]
     raise ValueError("Unrecognized hardware type %s" % htype)