diff --git a/bin/eo_plot_ccd.py b/bin/eo_plot_ccd.py index 939e3a5..e62b2c1 100755 --- a/bin/eo_plot_ccd.py +++ b/bin/eo_plot_ccd.py @@ -48,13 +48,13 @@ def main(): superbias_frame=superbias_frame, vmin=args.vmin, vmax=args.vmax) else: - figs.plot_sensor("img", None, ccd, + figs.plot_sensor("img", ccd, vmin=args.vmin, vmax=args.vmax, bias=args.bias, superbias_frame=superbias_frame, subtract_mean=args.subtract_mean) if args.stats_hist: - figs.histogram_array("hist", None, ccd, + figs.histogram_array("hist", ccd, bias=args.bias, superbias_frame=superbias_frame, xlabel="Counts", ylabel="Pixels / bin", vmin=args.vmin, vmax=args.vmax, diff --git a/python/lsst/eo_utils/base/analysis.py b/python/lsst/eo_utils/base/analysis.py index 1740f20..b6dcafd 100644 --- a/python/lsst/eo_utils/base/analysis.py +++ b/python/lsst/eo_utils/base/analysis.py @@ -246,9 +246,9 @@ def get_superbias_amp_image(butler, superbias_frame, amp): """ if superbias_frame is not None: if butler is not None: - superbias_im = get_raw_image(None, superbias_frame, amp+1) + superbias_im = get_raw_image(superbias_frame, amp+1) else: - superbias_im = get_raw_image(None, superbias_frame, amp) + superbias_im = get_raw_image(superbias_frame, amp) else: superbias_im = None return superbias_im diff --git a/python/lsst/eo_utils/base/batch_utils.py b/python/lsst/eo_utils/base/batch_utils.py index 0e0f4eb..7ad4eb8 100644 --- a/python/lsst/eo_utils/base/batch_utils.py +++ b/python/lsst/eo_utils/base/batch_utils.py @@ -13,9 +13,61 @@ import os import sys -#import pipes -#from lsst.ctrl.pool import Batch, exportEnv, UMASK +from lsst.eo_utils.base.defaults import BATCH_SYSTEM +from lsst.eo_utils.base.file_utils import makedir_safe + + + +def write_slurm_batchfile(jobname, logfile, **kwargs): + """Dispatch a single job + + Parameters + ---------- + jobname : `str` + The command to run the job + logfile : `str` + The path to the logfile + + Keywords + -------- + run : `str` + The run number + batch_args : `str` + Arguments to pass to batch command + optstring : `str` + Additional arguments to pass to the command + + Returns + ------- + batchfile : `str` + The name of the slurm submission script file + """ + run = kwargs.get('run', None) + batch_args = kwargs.get('batch_args', None) + optstring = kwargs.get('optstring', None) + batchfile = os.path.join('sbatch', logfile.replace('.log', '.sl')) + + makedir_safe(batchfile) + fout = open(batchfile, 'w') + fout.write("#!/bin/bash -l\n") + tokens = batch_args.split() + for key, val in zip(tokens[0::2], tokens[1::2]): + fout.write("#SBATCH %s %s\n" % (key, val)) + fout.write("\n") + sub_com = "srun --output %s" % logfile + + if run is None: + sub_com += " %s" % jobname + else: + sub_com += " %s --run %s" % (jobname, run) + sub_com += " %s" % optstring + + fout.write("%s\n" % sub_com) + fout.close() + return batchfile + + def dispatch_job(jobname, logfile, **kwargs): """Dispatch a single job @@ -32,7 +84,7 @@ def dispatch_job(jobname, logfile, **kwargs): run : `str` The run number batch : `str` - Where to send the jobs + Send jobs to batch farm batch_args : `str` Arguments to pass to batch command optstring : `str` @@ -41,25 +93,32 @@ def dispatch_job(jobname, logfile, **kwargs): Print command but do not run it """ run = kwargs.get('run', None) - batch = kwargs.get('batch', 'native') batch_args = kwargs.get('batch_args', None) optstring = kwargs.get('optstring', None) dry_run = kwargs.get('dry_run', False) - if batch.find('bsub') >= 0: + disptach = 'native' + if kwargs.get('batch', False): + disptach = BATCH_SYSTEM + + if disptach.find('lsf') == 0: sub_com = "bsub -o %s" % logfile if batch_args is not None: sub_com += " %s " % batch_args + elif disptach.find('slurm') == 0: + batchfile = write_slurm_batchfile(jobname, logfile, **kwargs) + sub_com = "sbatch %s" % batchfile else: sub_com = "" - if run is None: - sub_com += " %s" % jobname - else: - sub_com += " %s --run %s" % (jobname, run) + if disptach.find('slurm') < 0: + if run is None: + sub_com += " %s" % jobname + else: + sub_com += " %s --run %s" % (jobname, run) - if optstring is not None: - sub_com += " %s" % optstring + if optstring is not None: + sub_com += " %s" % optstring if dry_run: sys.stdout.write("%s\n" % sub_com) diff --git a/python/lsst/eo_utils/base/butler_utils.py b/python/lsst/eo_utils/base/butler_utils.py index cf5697f..a6a2f2c 100644 --- a/python/lsst/eo_utils/base/butler_utils.py +++ b/python/lsst/eo_utils/base/butler_utils.py @@ -32,6 +32,24 @@ def get_butler_by_repo(repo, **kwargs): return butler +def get_filename_from_id(butler, data_id): + """Return the hardware type and hardware id for a given run + + Parameters + ---------- + butler : `Bulter` + The data Butler + data_id : `dict` + The data id in question + + Returns + ------- + filename : `str` + The filename for the assocated CCD data + """ + return butler.get("raw_filename", data_id)[0][0:-3] + + def get_hardware_info(butler, run_num): """Return the hardware type and hardware id for a given run diff --git a/python/lsst/eo_utils/base/defaults.py b/python/lsst/eo_utils/base/defaults.py index 95ee010..5a3f346 100644 --- a/python/lsst/eo_utils/base/defaults.py +++ b/python/lsst/eo_utils/base/defaults.py @@ -1,20 +1,26 @@ """This module manage the default settings for eo_utils module""" - -# Location of various butler Repos -BUTLER_TS8_REPO = '/gpfs/slac/lsst/fs3/g/data/datasets/ts8' -BUTLER_BOT_REPO = '/gpfs/slac/lsst/fs3/g/data/datasets/bot' -BUTLER_TS8_NCSA_REPO = '/datasets/ts8/repo' -BUTLER_BOT_NCSA_REPO = '/datasets/bot/repo' - -# Location of slac archive -ARCHIVE_SLAC = '/gpfs/slac/lsst/fs*/g/data/jobHarness/jh_archive*' +import os + +# SITE DEPENDENT STUFF +SITE = os.environ.get('EO_UTILS_SITE', 'slac') + +if SITE == 'slac': + BUTLER_TS8_REPO = '/gpfs/slac/lsst/fs3/g/data/datasets/ts8' + BUTLER_BOT_REPO = '/gpfs/slac/lsst/fs3/g/data/datasets/bot' + ARCHIVE_DIR = '/gpfs/slac/lsst/fs*/g/data/jobHarness/jh_archive*' + DEFAULT_BATCH_ARGS = '-W 1200 -R bullet' + BATCH_SYSTEM = 'lsf' +elif SITE == 'ncsa': + BUTLER_TS8_REPO = '/datasets/ts8/repo' + BUTLER_BOT_REPO = '/datasets/bot/repo' + ARCHIVE_DIR = None + DEFAULT_BATCH_ARGS = "" + BATCH_SYSTEM = 'slurm' # Map the Butler repos to simple names BUTLER_REPO_DICT = dict(TS8=BUTLER_TS8_REPO, - BOT=BUTLER_BOT_REPO, - TS8_NCSA=BUTLER_TS8_NCSA_REPO, - BOT_NCSA=BUTLER_BOT_NCSA_REPO) + BOT=BUTLER_BOT_REPO) # The slots ALL_SLOTS = ['S00', 'S01', 'S02', 'S10', 'S11', 'S12', 'S20', 'S21', 'S22'] @@ -56,12 +62,6 @@ FE55="cyan") -# Template to make superbias files -SBIAS_TEMPLATE = 'analysis/superbias/templates/sbias_template.fits' -SFLAT_TEMPLATE = 'analysis/superflat/templates/sflat_template.fits' -SDARK_TEMPLATE = 'analysis/superdark/templates/sdark_template.fits' - - # Some default values DEFAULT_OUTDIR = 'analysis/ts8' DEFAULT_STAT_TYPE = 'median' @@ -70,4 +70,3 @@ DEFAULT_SUPERBIAS_TYPE = None DEFAULT_LOGFILE = 'temp.log' DEFAULT_NBINS = 100 -DEFAULT_BATCH_ARGS = "-W 1200 -R bullet" diff --git a/python/lsst/eo_utils/base/file_utils.py b/python/lsst/eo_utils/base/file_utils.py index 47d905c..4395e7c 100644 --- a/python/lsst/eo_utils/base/file_utils.py +++ b/python/lsst/eo_utils/base/file_utils.py @@ -18,7 +18,7 @@ print("Warning, no datacat-utilities") -from .defaults import ALL_RAFTS, ALL_SLOTS, ARCHIVE_SLAC +from .defaults import ALL_RAFTS, ALL_SLOTS, ARCHIVE_DIR # These are the standard input filenames @@ -289,10 +289,10 @@ def add_format(self, key, fmt, **kwargs): bias=None, suffix='') TS8_FORMATTER = FILENAME_FORMATS.add_format('ts8_images', TS8_GLOB_STRING, - archive=ARCHIVE_SLAC) + archive=ARCHIVE_DIR) BOT_FORMATTER = FILENAME_FORMATS.add_format('bot_images', BOT_GLOB_STRING, - archive=ARCHIVE_SLAC) + archive=ARCHIVE_DIR) TS8_EORESULTSIN_FORMATTER = FILENAME_FORMATS.add_format('ts8_eoresults_in', SLOT_FORMAT_STRING, diff --git a/python/lsst/eo_utils/base/image_utils.py b/python/lsst/eo_utils/base/image_utils.py index 0d4856d..fb8103d 100644 --- a/python/lsst/eo_utils/base/image_utils.py +++ b/python/lsst/eo_utils/base/image_utils.py @@ -35,13 +35,11 @@ AFWIMAGE_MASK = afwImage.Mask -def get_dims_from_ccd(butler, ccd): +def get_dims_from_ccd(ccd): """Get the CCD amp dimensions for a particular dataId or file Parameters ---------- - butler : `Butler` or `None` - Data Butler ccd : `ExposureF` or `MaskedCCD` CCD data object @@ -50,7 +48,7 @@ def get_dims_from_ccd(butler, ccd): odict : `dict` Dictionary with the dimensions """ - if butler is None: + if isinstance(ccd, MaskedCCD): geom = ccd.amp_geom o_dict = dict(nrow_i=geom.imaging.getHeight(), nrow_s=geom.serial_overscan.getHeight(), @@ -71,13 +69,11 @@ def get_dims_from_ccd(butler, ccd): return o_dict -def get_readout_freqs_from_ccd(butler, ccd): +def get_readout_freqs_from_ccd(ccd): """Get the frequencies corresponding to the FFTs Parameters ---------- - butler : `Butler` or `None` - Data Butler ccd : `ExposureF` or `MaskedCCD` CCD data object @@ -87,7 +83,7 @@ def get_readout_freqs_from_ccd(butler, ccd): Dictionary with the frequencies """ - if butler is None: + if isinstance(ccd, MaskedCCD): geom = ccd.amp_geom nrow_i = geom.imaging.getHeight() nrow_s = geom.serial_overscan.getHeight() @@ -194,13 +190,11 @@ def flip_data_in_place(filepath): hdus.writeto(filepath, overwrite=True) -def get_geom_regions(butler, ccd, amp): +def get_geom_regions(ccd, amp): """Get the ccd amp bounding boxes for a particular dataId or file Parameters ---------- - butler : `Butler` or `None` - Data Butler ccd : `ExposureF` or `MaskedCCD` CCD data object amp : `int` @@ -211,7 +205,7 @@ def get_geom_regions(butler, ccd, amp): odict : `dict` Dictionary with the bounding boxes """ - if butler is None: + if isinstance(ccd, MaskedCCD): geom = ccd.amp_geom o_dict = dict(imaging=geom.imaging, serial_overscan=geom.serial_overscan, @@ -233,13 +227,11 @@ def get_geom_regions(butler, ccd, amp): return o_dict -def get_dimension_arrays_from_ccd(butler, ccd): +def get_dimension_arrays_from_ccd(ccd): """Get the linear arrays with the indices for each direction and readout region Parameters ---------- - butler : `Butler` or `None` - Data Butler ccd : `ExposureF` or `MaskedCCD` CCD data object @@ -249,7 +241,7 @@ def get_dimension_arrays_from_ccd(butler, ccd): Dictionary with the arrays """ - if butler is None: + if isinstance(ccd, MaskedCCD): geom = ccd.amp_geom nrow_i = geom.imaging.getHeight() nrow_s = geom.serial_overscan.getHeight() @@ -275,13 +267,11 @@ def get_dimension_arrays_from_ccd(butler, ccd): return o_dict -def get_raw_image(butler, ccd, amp): +def get_raw_image(ccd, amp): """Get the raw image for a particular amp Parameters ---------- - butler : `Butler` or `None` - Data Butler ccd : `ExposureF` or `MaskedCCD` CCD data object amp : `int` @@ -292,7 +282,7 @@ def get_raw_image(butler, ccd, amp): odict : `ImageF` The image """ - if butler is None: + if isinstance(ccd, MaskedCCD): img = ccd[amp].getImage() else: geom = ccd.getDetector() @@ -300,7 +290,7 @@ def get_raw_image(butler, ccd, amp): return img -def get_ccd_from_id(butler, data_id, mask_files, bias_frame=None): +def get_ccd_from_id(butler, data_id, mask_files, **kwargs): """Get a CCD image from a data_id If we are using `Butler` then this will take a @@ -317,31 +307,40 @@ def get_ccd_from_id(butler, data_id, mask_files, bias_frame=None): Data identier mask_files : `list` List of data_ids for the files to construct the pixel mask + + Keywords + -------- bias_frame : `ExposureF` or `MaskedCCD` or `None` Object with the bias data + masked_ccd : `bool` + Use bulter only to get filename, return as MaskedCCD object Returns ------- ccd : `ExposureF` or `MaskedCCD` CCD data object """ + bias_frame = kwargs.get('bias_frame', None) if butler is None: exposure = MaskedCCD(str(data_id), mask_files=mask_files, bias_frame=bias_frame) + elif kwargs.get('masked_ccd', False): + filepath = butler.get('raw_filename', data_id) + exposure = MaskedCCD(str(filepath), + mask_files=mask_files, + bias_frame=bias_frame) else: exposure = butler.get('raw', data_id) apply_masks(butler, exposure, mask_files) return exposure -def get_amp_list(butler, ccd): +def get_amp_list(ccd): """Get the Geometry for a particular dataId or file Parameters ---------- - butler : `Butler` or `None` - Data Butler ccd : `ExposureF` or `MaskedCCD` CCD data object @@ -350,7 +349,7 @@ def get_amp_list(butler, ccd): amplist : `list` List of amplifier indices """ - if butler is None: + if isinstance(ccd, MaskedCCD): amplist = [amp for amp in ccd] else: amplist = [amp for amp in range(16)] @@ -388,14 +387,12 @@ def get_image_frames_2d(img, regions, regionlist=None): return o_dict -def get_data_as_read(butler, ccd, amp, regionlist=None): +def get_data_as_read(ccd, amp, regionlist=None): """Split out the arrays for the serial_overscan, parallel_overscan, and imaging regions and return the data in the readout order. Parameters ---------- - butler : `Butler` or `None` - Data Butler (or none) ccd : `ImageF` or `MaskedImageF` CCD image object amp : `int` @@ -408,8 +405,8 @@ def get_data_as_read(butler, ccd, amp, regionlist=None): o_dict : `dict` Dictionary mapping name to data array for each region """ - raw_image = get_raw_image(butler, ccd, amp) - regions = get_geom_regions(butler, ccd, amp) + raw_image = get_raw_image(ccd, amp) + regions = get_geom_regions(ccd, amp) frames = get_image_frames_2d(raw_image, regions, regionlist) return frames @@ -486,13 +483,11 @@ def unbias_amp(img, serial_oscan, bias_type=None, superbias_im=None, region=None return image -def raw_amp_image(butler, ccd, amp): +def raw_amp_image(ccd, amp): """Get the image for a particular amp Parameters ---------- - butler : `Butler` or `None - Data Butler ccd : `MaskedCCD` or `ImageF` CCD data object amp : `int` @@ -503,22 +498,20 @@ def raw_amp_image(butler, ccd, amp): The image """ if ccd is not None: - if butler is not None: - image = get_raw_image(None, ccd, amp+1) + if not isinstance(ccd, MaskedCCD): + image = get_raw_image(ccd, amp+1) else: - image = get_raw_image(None, ccd, amp) + image = get_raw_image(ccd, amp) else: image = None return image -def unbiased_ccd_image_dict(butler, ccd, **kwargs): +def unbiased_ccd_image_dict(ccd, **kwargs): """Get the images keys by amp for a ccd Parameters ---------- - butler : `Butler` or `None - Data Butler ccd : `MaskedCCD` or `ImageF` CCD data object @@ -538,27 +531,25 @@ def unbiased_ccd_image_dict(butler, ccd, **kwargs): bias_type = kwcopy.pop('bias', None) superbias_frame = kwcopy.pop('superbias_frame', None) - amps = get_amp_list(butler, ccd) + amps = get_amp_list(ccd) o_dict = {} for amp in amps: - regions = get_geom_regions(butler, ccd, amp) + regions = get_geom_regions(ccd, amp) serial_oscan = regions['serial_overscan'] - img = get_raw_image(butler, ccd, amp) - superbias_im = raw_amp_image(butler, superbias_frame, amp) + img = get_raw_image(ccd, amp) + superbias_im = raw_amp_image(superbias_frame, amp) image = unbias_amp(img, serial_oscan, bias_type=bias_type, superbias_im=superbias_im) o_dict[amp] = image return o_dict -def extract_ccd_array_dict(butler, data_id, **kwargs): +def extract_ccd_array_dict(data_id, **kwargs): """Get the Geometry for a particular dataId or file Parameters ---------- - butler : `Butler` or `None - Data Butler data_id : `dict` or `str` Data identifier or filename @@ -579,18 +570,18 @@ def extract_ccd_array_dict(butler, data_id, **kwargs): kwcopy = kwargs.copy() mask_files = kwcopy.pop('mask_files', []) ccd = get_ccd_from_id(None, data_id, mask_files) - unbiased_images = unbiased_ccd_image_dict(butler, ccd, **kwargs) + unbiased_images = unbiased_ccd_image_dict(ccd, **kwargs) o_dict = {} for amp, image in unbiased_images.items(): - regions = get_geom_regions(butler, ccd, amp) + regions = get_geom_regions(ccd, amp) frames = get_image_frames_2d(image, regions) o_dict[amp] = frames[kwcopy.pop('region', 'imaging')] return o_dict -def extract_raft_unbiased_images(butler, data_id_dict, **kwargs): +def extract_raft_unbiased_images(data_id_dict, **kwargs): """Get the unbiased raft-level images Parameters @@ -634,17 +625,15 @@ def extract_raft_unbiased_images(butler, data_id_dict, **kwargs): kwcopy['superbias_frame'] = superbias_frame ccd = get_ccd_from_id(None, data_id, mask_files) ccd_dict[slot] = ccd - o_dict[slot] = unbiased_ccd_image_dict(butler, ccd, **kwcopy) + o_dict[slot] = unbiased_ccd_image_dict(ccd, **kwcopy) return o_dict, ccd_dict -def extract_raft_array_dict(butler, data_id_dict, **kwargs): +def extract_raft_array_dict(data_id_dict, **kwargs): """Get raft level data Parameters ---------- - butler : `Butler` or `None - Data Butler data_id_dict : `dict` Dictionary, keyed by slot, of data identifiers or filenames @@ -677,14 +666,14 @@ def extract_raft_array_dict(butler, data_id_dict, **kwargs): else: superbias_frame = superbias_dict[slot] - o_dict[slot] = extract_ccd_array_dict(butler, data_id, + o_dict[slot] = extract_ccd_array_dict(data_id, mask_files=mask_files, superbias_frame=superbias_frame, **kwcopy) return o_dict -def get_exposure_time(butler, ccd): +def get_exposure_time(ccd): """Return the exposure time Parameters @@ -699,11 +688,11 @@ def get_exposure_time(butler, ccd): exptime : `float` The exposure time in seconds """ - if butler is None: + if isinstance(ccd, MaskedCCD): return ccd.md.md.get('EXPTIME') - raise NotImplementedError("Can't get exposure time for butlerlized data") + return ccd.getInfo().getVisitInfo().getExposureTime() -def get_mondiode_val(butler, ccd): +def get_mondiode_val(ccd): """Return the monitoring diode value Parameters @@ -718,12 +707,12 @@ def get_mondiode_val(butler, ccd): val : `float` The value """ - if butler is None: + if isinstance(ccd, MaskedCCD): return ccd.md.get('MONDIODE') - raise NotImplementedError("Can't get mondiode value for butlerlized data") + return ccd.getMetadata()['MONDIODE'] -def get_mono_wl(butler, ccd): +def get_mono_wl(ccd): """Return the monochromatic wavelength Parameters @@ -738,9 +727,9 @@ def get_mono_wl(butler, ccd): val : `float` The value """ - if butler is None: + if isinstance(ccd, MaskedCCD): return ccd.md.get('MONOWL') - raise NotImplementedError("Can't get monowl for butlerlized data") + return ccd.getMetadata()['MONOWL'] def stack_images(butler, in_files, statistic=afwMath.MEDIAN, **kwargs): @@ -785,14 +774,14 @@ def stack_images(butler, in_files, statistic=afwMath.MEDIAN, **kwargs): log.info(" %i" % ifile) ccd = get_ccd_from_id(butler, in_file, mask_files=[]) - exp_time += get_exposure_time(butler, ccd) - amps = get_amp_list(butler, ccd) + exp_time += get_exposure_time(ccd) + amps = get_amp_list(ccd) for amp in amps: - regions = get_geom_regions(butler, ccd, amp) + regions = get_geom_regions(ccd, amp) serial_oscan = regions['serial_overscan'] - img = get_raw_image(butler, ccd, amp) - superbias_im = raw_amp_image(butler, superbias_frame, amp) + img = get_raw_image(ccd, amp) + superbias_im = raw_amp_image(superbias_frame, amp) if ifile == 0: amp_stack_dict[amp] = [unbias_amp(img, serial_oscan, @@ -925,13 +914,11 @@ def outlier_stats(data_array, mean_val, max_offset): return o_dict -def extract_raft_imaging_data(butler, image_dict, ccd_dict, **kwargs): +def extract_raft_imaging_data(image_dict, ccd_dict, **kwargs): """Get data from the imaging regions Parameters ---------- - butler : `Butler` or `None - Data Butler image_dict : `dict` Dictionary, keyed by slot, amp of images ccd_dict : `dict` @@ -948,7 +935,7 @@ def extract_raft_imaging_data(butler, image_dict, ccd_dict, **kwargs): ccd = ccd_dict[slot] o_dict[slot] = {} for amp, image in slot_data.items(): - regions = get_geom_regions(butler, ccd, amp) + regions = get_geom_regions(ccd, amp) frames = get_image_frames_2d(image, regions) o_dict[slot][amp] = frames[kwcopy.pop('region', 'imaging')] diff --git a/python/lsst/eo_utils/base/iter_utils.py b/python/lsst/eo_utils/base/iter_utils.py index 57522f8..afc1f26 100644 --- a/python/lsst/eo_utils/base/iter_utils.py +++ b/python/lsst/eo_utils/base/iter_utils.py @@ -154,6 +154,9 @@ def call_analysis_task(self, **kwargs): Data `Butler` that is being used to access data """ kwargs.setdefault('butler', self._butler) + if kwargs.get('dry_run', False): + self._task.info("Skipping %s" % (kwargs)) + return None return self._task(**kwargs) def get_dispatch_args(self, **kwargs): @@ -200,7 +203,7 @@ def run_with_args(self, **kwargs): Used to update both the handler and `Task` configurations """ kwremain = self.safe_update(**kwargs) - if self.config.batch is None: + if self.config.batch in ['None', 'none', None]: self.call_analysis_task(**kwremain) else: jobname = os.path.basename(sys.argv[0]) @@ -350,15 +353,17 @@ def dispatch_single_run(self, run, **kwargs): kwargs Used to update `Task` configuration """ - if self.config.batch is None: + + taskname = self._task.getName().replace('Task', '') + if self.config.batch in ['None', 'none', None]: self.call_analysis_task(run, **kwargs) elif 'slot' in self.config.batch: - jobname = "eo_task.py %s" % self._task.getName().replace('Task', '') + jobname = "eo_task.py %s" % taskname slots = kwargs.pop('slots') if slots is None: slots = ALL_SLOTS for slot in slots: - logfile_slot = self.config.logfile.replace('.log', '%s_%s.log' % (run, slot)) + logfile_slot = self.config.logfile.replace('.log', '%s_%s_%s.log' % (taskname, run, slot)) kwargs['slots'] = slot kw_remain = self.get_dispatch_args(run, **kwargs) dispatch_job(jobname, logfile_slot, **kw_remain) @@ -366,7 +371,7 @@ def dispatch_single_run(self, run, **kwargs): jobname = "eo_task.py %s" % self._task.getName().replace('Task', '') kw_remain = self.get_dispatch_args(run, **kwargs) dispatch_job(jobname, - self.config.logfile.replace('.log', '_%s.log' % (run)), + self.config.logfile.replace('.log', '_%s_%s.log' % (taskname, run)), **kw_remain) @@ -388,6 +393,7 @@ def run_with_args(self, **kwargs): kw_remain = self.safe_update(**kwargs) kw_remain['slots'] = kwargs.get('slots', None) kw_remain['rafts'] = kwargs.get('rafts', None) + kw_remain['dry_run'] = kwargs.get('dry_run', False) if self.config.dataset is not None and self.config.runs is not None: raise ValueError("Either runs or dataset should be set, not both") @@ -430,6 +436,9 @@ def iterate_over_slots(analysis_task, butler, data_files, **kwargs): for slot in slot_list: slot_data = data_files[slot] kwargs['slot'] = slot + if kwargs.get('dry_run', False): + analysis_task.log.info("Skipping {run}:{raft}:{slot}".format(**kwargs)) + continue analysis_task(butler, slot_data, **kwargs) @@ -830,6 +839,9 @@ def call_analysis_task(self, **kwargs): kwargs Passed to get_data() and to the analysis function """ + if kwargs.get('dry_run', False): + self._task.info("Skipping %s" % (kwargs)) + return data_files = self.get_data(None, **kwargs) self._task(None, data_files, **kwargs) @@ -875,7 +887,7 @@ def run_with_args(self, **kwargs): """ kw_remain = self.safe_update(**kwargs) - if self.config.batch is None: + if self.config.batch in ['None', 'none', None]: self.call_analysis_task(**kw_remain) else: jobname = os.path.basename(sys.argv[0]) diff --git a/python/lsst/eo_utils/base/plot_utils.py b/python/lsst/eo_utils/base/plot_utils.py index 4a9a2c4..84be4d7 100644 --- a/python/lsst/eo_utils/base/plot_utils.py +++ b/python/lsst/eo_utils/base/plot_utils.py @@ -9,7 +9,7 @@ from lsst.eotest.raft import RaftMosaic -from lsst.eotest.sensor import parse_geom_kwd +from lsst.eotest.sensor import MaskedCCD, parse_geom_kwd import lsst.afw.image as afwImage @@ -860,15 +860,13 @@ def plot_amp_arrays(self, key, array_dict, **kwargs): return o_dict - def plot_sensor(self, key, butler, ccd, **kwargs): + def plot_sensor(self, key, ccd, **kwargs): """Plot the data from all 16 amps on a sensor in a single figure Parameters ---------- key : `str` Key for the figure. - butler : `Butler` - The data butler ccd : `MaskedCCD` or `AFWImage` Object with the image data @@ -899,10 +897,10 @@ def plot_sensor(self, key, butler, ccd, **kwargs): fig, axs = plt.subplots(2, 8, figsize=(15, 10)) axs = axs.ravel() - unbiased_images = unbiased_ccd_image_dict(butler, ccd, + unbiased_images = unbiased_ccd_image_dict(ccd, superbias_frame=superbias_frame, bias_type=bias_type) - amps = get_amp_list(butler, ccd) + amps = get_amp_list(ccd) for idx, amp in enumerate(amps): image = unbiased_images[amp] darray = image.array @@ -918,15 +916,13 @@ def plot_sensor(self, key, butler, ccd, **kwargs): return o_dict - def histogram_array(self, key, butler, ccd, **kwargs): + def histogram_array(self, key, ccd, **kwargs): """Plot the data from all 16 amps on a sensor in a single figure Parameters ---------- key : `str` Key for the figure. - butler : `Butler` - The data butler ccd : `MaskedCCD` or `AFWImage` Object with the image data @@ -954,15 +950,15 @@ def histogram_array(self, key, butler, ccd, **kwargs): o_dict = self.setup_amp_plots_grid(key, **kwsetup) axs = o_dict['axs'] - unbiased_images = unbiased_ccd_image_dict(butler, ccd, **kwcopy) + unbiased_images = unbiased_ccd_image_dict(ccd, **kwcopy) for amp, image in unbiased_images.items(): - regions = get_geom_regions(butler, ccd, amp) + regions = get_geom_regions(ccd, amp) frames = get_image_frames_2d(image, regions) darray = frames[kwcopy.pop('region', 'imaging')] if kwcopy.pop('subtract_mean', False): darray -= darray.mean() - if butler is None: + if isinstance(ccd, MaskedCCD): idx = amp - 1 else: idx = amp @@ -1135,7 +1131,7 @@ def plot_ccd_mosaic(self, key, ccd, **kwargs): for ypos in range(ny_segments): for xpos in range(nx_segments): amp = ypos*nx_segments + xpos + 1 - regions = get_geom_regions(None, ccd, amp) + regions = get_geom_regions(ccd, amp) serial_oscan = regions['serial_overscan'] imaging = regions['imaging'] @@ -1147,8 +1143,8 @@ def plot_ccd_mosaic(self, key, ccd, **kwargs): # # Extract bias-subtracted image for this segment - overscan corrected here # - superbias_im = raw_amp_image(None, superbias_frame, amp) - img = get_raw_image(None, ccd, amp) + superbias_im = raw_amp_image(superbias_frame, amp) + img = get_raw_image(ccd, amp) segment_image = unbias_amp(img, serial_oscan, bias_type=bias_type, superbias_im=superbias_im) subarr = segment_image[imaging].array diff --git a/python/lsst/eo_utils/bias/bias_fft.py b/python/lsst/eo_utils/bias/bias_fft.py index feedafe..26198e9 100644 --- a/python/lsst/eo_utils/bias/bias_fft.py +++ b/python/lsst/eo_utils/bias/bias_fft.py @@ -79,13 +79,13 @@ def extract(self, butler, data, **kwargs): ccd = get_ccd_from_id(butler, bias_file, mask_files) if ifile == 0: - freqs_dict = get_readout_freqs_from_ccd(butler, ccd) + freqs_dict = get_readout_freqs_from_ccd(ccd) for key in REGION_KEYS: freqs = freqs_dict['freqs_%s' % key] nfreqs = len(freqs) fft_data[key] = dict(freqs=freqs[0:int(nfreqs/2)]) - BiasFFTTask.get_ccd_data(self, butler, ccd, fft_data, + BiasFFTTask.get_ccd_data(self, ccd, fft_data, ifile=ifile, nfiles_used=len(bias_files), slot=slot, superbias_frame=superbias_frame) @@ -123,15 +123,13 @@ def plot(self, dtables, figs, **kwargs): ymin=0., ymax=3.) @staticmethod - def get_ccd_data(for_whom, butler, ccd, data, **kwargs): + def get_ccd_data(for_whom, ccd, data, **kwargs): """Get the fft of the overscan values and update the data dictionary Parameters ---------- for_whom : `Task` Task this is being run for - butler : `Butler` - The data butler ccd : `MaskedCCD` The ccd we are getting data from data : `dict` @@ -159,12 +157,12 @@ def get_ccd_data(for_whom, butler, ccd, data, **kwargs): nfiles_used = kwargs.get('nfiles_used', 1) superbias_frame = kwargs.get('superbias_frame', None) - amps = get_amp_list(butler, ccd) + amps = get_amp_list(ccd) for i, amp in enumerate(amps): - regions = get_geom_regions(butler, ccd, amp) + regions = get_geom_regions(ccd, amp) serial_oscan = regions['serial_overscan'] - img = get_raw_image(butler, ccd, amp) - superbias_im = raw_amp_image(butler, superbias_frame, amp) + img = get_raw_image(ccd, amp) + superbias_im = raw_amp_image(superbias_frame, amp) image = unbias_amp(img, serial_oscan, bias_type=for_whom.get_config_param('bias', None), superbias_im=superbias_im) @@ -229,13 +227,13 @@ def extract(self, butler, data, **kwargs): self.log_info_slot_msg(self.config, "") - freqs_dict = get_readout_freqs_from_ccd(None, superbias) + freqs_dict = get_readout_freqs_from_ccd(superbias) for key in REGION_KEYS: freqs = freqs_dict['freqs_%s' % key] nfreqs = len(freqs) fft_data[key] = dict(freqs=freqs[0:int(nfreqs/2)]) - BiasFFTTask.get_ccd_data(self, None, superbias, fft_data, + BiasFFTTask.get_ccd_data(self, superbias, fft_data, slot=slot, superbias_frame=None) diff --git a/python/lsst/eo_utils/bias/bias_struct.py b/python/lsst/eo_utils/bias/bias_struct.py index 6f3a95f..ec3a814 100644 --- a/python/lsst/eo_utils/bias/bias_struct.py +++ b/python/lsst/eo_utils/bias/bias_struct.py @@ -74,11 +74,11 @@ def extract(self, butler, data, **kwargs): ccd = get_ccd_from_id(butler, bias_file, mask_files) if ifile == 0: - dim_array_dict = get_dimension_arrays_from_ccd(butler, ccd) + dim_array_dict = get_dimension_arrays_from_ccd(ccd) for key, val in dim_array_dict.items(): biasstruct_data[key] = {key:val} - self.get_ccd_data(butler, ccd, biasstruct_data, + self.get_ccd_data(ccd, biasstruct_data, slot=slot, ifile=ifile, nfiles_used=len(bias_files), superbias_frame=superbias_frame) @@ -111,13 +111,11 @@ def plot(self, dtables, figs, **kwargs): x_name="%s_%s" % (dkey, rkey), y_name="biasst") - def get_ccd_data(self, butler, ccd, data, **kwargs): + def get_ccd_data(self, ccd, data, **kwargs): """Get the bias values and update the data dictionary Parameters ---------- - butler : `Butler` - The data butler ccd : `MaskedCCD` The ccd we are getting data from data : `dict` @@ -143,13 +141,13 @@ def get_ccd_data(self, butler, ccd, data, **kwargs): slot = kwargs.get('slot') superbias_frame = kwargs.get('superbias_frame', None) - amps = get_amp_list(butler, ccd) + amps = get_amp_list(ccd) for i, amp in enumerate(amps): - regions = get_geom_regions(butler, ccd, amp) + regions = get_geom_regions(ccd, amp) serial_oscan = regions['serial_overscan'] - img = get_raw_image(butler, ccd, amp) + img = get_raw_image(ccd, amp) if superbias_frame is not None: - superbias_im = get_raw_image(butler, superbias_frame, amp) + superbias_im = get_raw_image(superbias_frame, amp) else: superbias_im = None image = unbias_amp(img, serial_oscan, @@ -225,11 +223,11 @@ def extract(self, butler, data, **kwargs): biasstruct_data = {} - dim_array_dict = get_dimension_arrays_from_ccd(None, superbias) + dim_array_dict = get_dimension_arrays_from_ccd(superbias) for key, val in dim_array_dict.items(): biasstruct_data[key] = {key:val} - self.get_ccd_data(None, superbias, biasstruct_data, + self.get_ccd_data(superbias, biasstruct_data, slot=slot, bias_type=None, std=self.config.std, superbias_frame=None) diff --git a/python/lsst/eo_utils/bias/bias_v_row.py b/python/lsst/eo_utils/bias/bias_v_row.py index 4a28040..6ebdf66 100644 --- a/python/lsst/eo_utils/bias/bias_v_row.py +++ b/python/lsst/eo_utils/bias/bias_v_row.py @@ -63,10 +63,10 @@ def extract(self, butler, data, **kwargs): ccd = get_ccd_from_id(butler, bias_file, []) if ifile == 0: - dims = get_dims_from_ccd(butler, ccd) + dims = get_dims_from_ccd(ccd) xrow_s = np.linspace(0, dims['nrow_s']-1, dims['nrow_s']) - self.get_ccd_data(butler, ccd, biasval_data, + self.get_ccd_data(ccd, biasval_data, ifile=ifile, nfiles=len(bias_files)) #Need to truncate the row array to match the data @@ -103,13 +103,11 @@ def plot(self, dtables, figs, **kwargs): x_name='row_s', y_name='biasval') - def get_ccd_data(self, butler, ccd, data, **kwargs): + def get_ccd_data(self, ccd, data, **kwargs): """Get the bias values and update the data dictionary Parameters ---------- - butler : `Butler` - The data butler ccd : `MaskedCCD` The ccd we are getting data from data : `dict` @@ -129,11 +127,11 @@ def get_ccd_data(self, butler, ccd, data, **kwargs): ifile = kwargs['ifile'] nfiles = kwargs['nfiles'] - amps = get_amp_list(butler, ccd) + amps = get_amp_list(ccd) for i, amp in enumerate(amps): - regions = get_geom_regions(butler, ccd, amp) + regions = get_geom_regions(ccd, amp) serial_oscan = regions['serial_overscan'] - img = get_raw_image(butler, ccd, amp) + img = get_raw_image(ccd, amp) bimg = imutil.bias_image(img, serial_oscan, bias_method=bias_type) bimg_row_mean = bimg[serial_oscan].getArray().mean(1) key_str = "biasval_%s_a%02i" % (slot, i) diff --git a/python/lsst/eo_utils/bias/correl_wrt_oscan.py b/python/lsst/eo_utils/bias/correl_wrt_oscan.py index 5cfd963..4e11d8f 100644 --- a/python/lsst/eo_utils/bias/correl_wrt_oscan.py +++ b/python/lsst/eo_utils/bias/correl_wrt_oscan.py @@ -79,16 +79,16 @@ def extract(self, butler, data, **kwargs): ccd = get_ccd_from_id(butler, bias_file, mask_files) if ifile == 0: - dims = get_dims_from_ccd(butler, ccd) + dims = get_dims_from_ccd(ccd) nrow_i = dims['nrow_i'] ncol_i = dims['ncol_i'] - amps = get_amp_list(butler, ccd) + amps = get_amp_list(ccd) for i, amp in enumerate(amps): - regions = get_geom_regions(butler, ccd, amp) - image = get_raw_image(butler, ccd, amp) + regions = get_geom_regions(ccd, amp) + image = get_raw_image(ccd, amp) ref_frames[i] = get_image_frames_2d(image, regions) continue - self.get_ccd_data(butler, ccd, ref_frames, + self.get_ccd_data(ccd, ref_frames, ifile=ifile, s_correl=s_correl, p_correl=p_correl, nrow_i=nrow_i, ncol_i=ncol_i) @@ -136,13 +136,11 @@ def plot(self, dtables, figs, **kwargs): figs.get_obj('oscorr-col', 'axs').flat[i].hist(p_correl, bins=100, range=(-1., 1.)) - def get_ccd_data(self, butler, ccd, ref_frames, **kwargs): + def get_ccd_data(self, ccd, ref_frames, **kwargs): """Get the bias values and update the data dictionary Parameters ---------- - butler : `Butler` - The data butler ccd : `MaskedCCD` The ccd we are getting data from data : `dict` @@ -163,11 +161,11 @@ def get_ccd_data(self, butler, ccd, ref_frames, **kwargs): nrow_i = kwargs['nrow_i'] ncol_i = kwargs['ncol_i'] - amps = get_amp_list(butler, ccd) + amps = get_amp_list(ccd) for i, amp in enumerate(amps): - regions = get_geom_regions(butler, ccd, amp) - image = get_raw_image(butler, ccd, amp) + regions = get_geom_regions(ccd, amp) + image = get_raw_image(ccd, amp) frames = get_image_frames_2d(image, regions) del_i_array = frames['imaging'] - ref_frames[i]['imaging'] diff --git a/python/lsst/eo_utils/bias/data_utils.py b/python/lsst/eo_utils/bias/data_utils.py index ef5b536..cef0639 100644 --- a/python/lsst/eo_utils/bias/data_utils.py +++ b/python/lsst/eo_utils/bias/data_utils.py @@ -9,7 +9,7 @@ get_image_frames_2d, array_struct, unbias_amp -def stack_by_amps(stack_arrays, butler, ccd, **kwargs): +def stack_by_amps(stack_arrays, ccd, **kwargs): """Stack arrays for all the amps to look for coherent noise Parameters @@ -34,13 +34,13 @@ def stack_by_amps(stack_arrays, butler, ccd, **kwargs): ifile = kwargs['ifile'] superbias_frame = kwargs.get('superbias_frame', None) - amps = get_amp_list(butler, ccd) + amps = get_amp_list(ccd) for i, amp in enumerate(amps): - regions = get_geom_regions(butler, ccd, amp) + regions = get_geom_regions(ccd, amp) serial_oscan = regions['serial_overscan'] - img = get_raw_image(butler, ccd, amp) - superbias_im = raw_amp_image(butler, superbias_frame, amp) + img = get_raw_image(ccd, amp) + superbias_im = raw_amp_image(superbias_frame, amp) image = unbias_amp(img, serial_oscan, bias_type=bias_type, superbias_im=superbias_im) frames = get_image_frames_2d(image, regions) diff --git a/python/lsst/eo_utils/bias/oscan_amp_stack.py b/python/lsst/eo_utils/bias/oscan_amp_stack.py index 25791ac..a4822de 100644 --- a/python/lsst/eo_utils/bias/oscan_amp_stack.py +++ b/python/lsst/eo_utils/bias/oscan_amp_stack.py @@ -80,11 +80,11 @@ def extract(self, butler, data, **kwargs): ccd = get_ccd_from_id(butler, bias_file, mask_files) if ifile == 0: - dim_array_dict = get_dimension_arrays_from_ccd(butler, ccd) + dim_array_dict = get_dimension_arrays_from_ccd(ccd) for key, val in dim_array_dict.items(): stack_arrays[key] = np.zeros((nfiles, 16, len(val))) - stack_by_amps(stack_arrays, butler, ccd, + stack_by_amps(stack_arrays, ccd, ifile=ifile, bias_type=self.config.bias, superbias_frame=superbias_frame) diff --git a/python/lsst/eo_utils/bias/oscan_correl.py b/python/lsst/eo_utils/bias/oscan_correl.py index bf18634..7de470a 100644 --- a/python/lsst/eo_utils/bias/oscan_correl.py +++ b/python/lsst/eo_utils/bias/oscan_correl.py @@ -138,16 +138,16 @@ def get_ccd_data(self, butler, ccd, **kwargs): overscans : `list` The overscan data """ - amps = get_amp_list(butler, ccd) + amps = get_amp_list(ccd) superbias_frame = kwargs.get('superbias_frame', None) overscans = [] for amp in amps: superbias_im = self.get_superbias_amp_image(butler, superbias_frame, amp) - regions = get_geom_regions(butler, ccd, amp) + regions = get_geom_regions(ccd, amp) serial_oscan = regions['serial_overscan'] - img = get_raw_image(butler, ccd, amp) + img = get_raw_image(ccd, amp) image = unbias_amp(img, serial_oscan, bias_type=None, superbias_im=superbias_im) oscan_copy = copy.deepcopy(serial_oscan) oscan_copy.grow(-self.boundry) diff --git a/python/lsst/eo_utils/bias/superbias.py b/python/lsst/eo_utils/bias/superbias.py index 9c003b1..25930eb 100644 --- a/python/lsst/eo_utils/bias/superbias.py +++ b/python/lsst/eo_utils/bias/superbias.py @@ -8,6 +8,8 @@ from lsst.eo_utils.base.file_utils import makedir_safe +from lsst.eo_utils.base.butler_utils import get_filename_from_id + from lsst.eo_utils.base.defaults import DEFAULT_STAT_TYPE from lsst.eo_utils.base.config_utils import EOUtilOptions @@ -124,7 +126,12 @@ def make_superbias(self, butler, slot_data, **kwargs): if not self.config.skip: out_data = self.extract(butler, slot_data) - imutil.writeFits(out_data, output_file, slot_data['BIAS'][0], self.config.bitpix) + if butler is None: + template_file = slot_data['BIAS'][0] + else: + template_file = get_filename_from_id(butler, slot_data['BIAS'][0]) + + imutil.writeFits(out_data, output_file, template_file, self.config.bitpix) if butler is not None: flip_data_in_place(output_file) @@ -156,7 +163,7 @@ def plot(self, dtables, figs, **kwargs): hist_range = (self.config.vmin, self.config.vmax) if self.config.plot: - figs.plot_sensor("img", None, self._superbias_frame) + figs.plot_sensor("img", self._superbias_frame) default_array_kw = {} if self.config.stats_hist: @@ -261,7 +268,7 @@ def extract(self, butler, data, **kwargs): self._mask_file_dict[slot] = self.get_mask_files(slot=slot) self._sbias_file_dict[slot] = data[slot] - self._sbias_arrays = extract_raft_array_dict(None, self._sbias_file_dict, + self._sbias_arrays = extract_raft_array_dict(self._sbias_file_dict, mask_dict=self._mask_file_dict) out_data = outlier_raft_dict(self._sbias_arrays, 0., 10.) diff --git a/python/lsst/eo_utils/bias/superbias_stats.py b/python/lsst/eo_utils/bias/superbias_stats.py index b9be9e1..ed83b1c 100644 --- a/python/lsst/eo_utils/bias/superbias_stats.py +++ b/python/lsst/eo_utils/bias/superbias_stats.py @@ -78,7 +78,7 @@ def extract(self, butler, data, **kwargs): mask_files = self.get_mask_files(slot=slot) superbias_frame = get_ccd_from_id(None, superbias_file, mask_files) - self.get_superbias_stats(None, superbias_frame, stats_data, islot) + self.get_superbias_stats(superbias_frame, stats_data, islot) self.log_progress("Done!") @@ -108,7 +108,7 @@ def plot(self, dtables, figs, **kwargs): @staticmethod - def get_superbias_stats(butler, superbias, stats_data, islot): + def get_superbias_stats(superbias, stats_data, islot): """Get the serial overscan data Parameters @@ -125,7 +125,7 @@ def get_superbias_stats(butler, superbias, stats_data, islot): islot : `int` The slot index """ - amps = get_amp_list(butler, superbias) + amps = get_amp_list(superbias) if 'mean' not in stats_data: stats_data['mean'] = np.ndarray((9, 16)) @@ -135,7 +135,7 @@ def get_superbias_stats(butler, superbias, stats_data, islot): stats_data['max'] = np.ndarray((9, 16)) for i, amp in enumerate(amps): - img = get_raw_image(butler, superbias, amp) + img = get_raw_image(superbias, amp) stats_data['mean'][islot, i] = img.array.mean() stats_data['median'][islot, i] = np.median(img.array) stats_data['std'][islot, i] = img.array.std() diff --git a/python/lsst/eo_utils/dark/dark_current.py b/python/lsst/eo_utils/dark/dark_current.py index d4a6d33..091e7eb 100644 --- a/python/lsst/eo_utils/dark/dark_current.py +++ b/python/lsst/eo_utils/dark/dark_current.py @@ -86,13 +86,13 @@ def extract(self, butler, data, **kwargs): mask_files = self.get_mask_files(slot=slot) superdark_file = data[slot].replace('.fits.fits', '.fits') superdark_frame = get_ccd_from_id(None, superdark_file, mask_files) - exptime = get_exposure_time(None, superdark_frame) + exptime = get_exposure_time(superdark_frame) - amps = get_amp_list(None, superdark_frame) + amps = get_amp_list(superdark_frame) for iamp, amp in enumerate(amps): - regions = get_geom_regions(None, superdark_frame, amp) + regions = get_geom_regions(superdark_frame, amp) imaging = regions['imaging'] - superdark_im = get_raw_image(None, superdark_frame, amp) + superdark_im = get_raw_image(superdark_frame, amp) image_data = superdark_im[imaging].array median = np.median(image_data) stdev = np.std(image_data) diff --git a/python/lsst/eo_utils/dark/superdark.py b/python/lsst/eo_utils/dark/superdark.py index d5757c0..cda561f 100644 --- a/python/lsst/eo_utils/dark/superdark.py +++ b/python/lsst/eo_utils/dark/superdark.py @@ -8,6 +8,8 @@ from lsst.eo_utils.base.file_utils import makedir_safe +from lsst.eo_utils.base.butler_utils import get_filename_from_id + from lsst.eo_utils.base.defaults import DEFAULT_STAT_TYPE from lsst.eo_utils.base.config_utils import EOUtilOptions @@ -133,7 +135,12 @@ def make_superdark(self, butler, slot_data, **kwargs): if not self.config.skip: sdark = self.extract(butler, slot_data) - imutil.writeFits(sdark, output_file + '.fits', slot_data['DARK'][0], self.config.bitpix) + if butler is None: + template_file = slot_data['DARK'][0] + else: + template_file = get_filename_from_id(butler, slot_data['DARK'][0]) + + imutil.writeFits(sdark, output_file + '.fits', template_file, self.config.bitpix) if butler is not None: flip_data_in_place(output_file + '.fits') @@ -159,7 +166,7 @@ def plot(self, dtables, figs, **kwargs): raise ValueError("dtables should not be set in SuperdarkTask.plot") if self.config.plot: - figs.plot_sensor("img", None, self._superdark_frame) + figs.plot_sensor("img", self._superdark_frame) default_array_kw = {} if self.config.stats_hist: @@ -271,7 +278,7 @@ def extract(self, butler, data, **kwargs): self._mask_file_dict[slot] = mask_files self._sdark_file_dict[slot] = data[slot].replace('.fits.fits', '.fits') - self._sdark_arrays = extract_raft_array_dict(None, self._sdark_file_dict, + self._sdark_arrays = extract_raft_array_dict(self._sdark_file_dict, mask_dict=self._mask_file_dict) out_data = outlier_raft_dict(self._sdark_arrays, 0., 25.) diff --git a/python/lsst/eo_utils/flat/dust_linearity_analysis.py b/python/lsst/eo_utils/flat/dust_linearity_analysis.py index 45c38b3..f4be04b 100644 --- a/python/lsst/eo_utils/flat/dust_linearity_analysis.py +++ b/python/lsst/eo_utils/flat/dust_linearity_analysis.py @@ -86,7 +86,6 @@ def extract(self, butler, data, **kwargs): self.log_info_slot_msg(self.config, "%i files" % len(flat1_files)) sflat_table_file = self.get_filename_from_format(RAFT_SFLAT_TABLE_FORMATTER, "sflat.fits") - print(sflat_table_file) sflat_tables = TableDict(sflat_table_file) # dictionary of dictionaries of lists of bounding @@ -127,11 +126,11 @@ def extract(self, butler, data, **kwargs): ccd = get_ccd_from_id(butler, flat1_file, mask_files) # To be appended while looping over bounding boxes - exptime = get_exposure_time(butler, ccd) - mondiode_val = get_mondiode_val(butler, ccd) + exptime = get_exposure_time(ccd) + mondiode_val = get_mondiode_val(ccd) islot = slot_idx_dict[slot] - unbiased_images = unbiased_ccd_image_dict(butler, ccd, + unbiased_images = unbiased_ccd_image_dict(ccd, bias=self.config.bias, superbias_frame=superbias_frame) diff --git a/python/lsst/eo_utils/flat/flat_bf.py b/python/lsst/eo_utils/flat/flat_bf.py index 275c8a9..8dac3cb 100644 --- a/python/lsst/eo_utils/flat/flat_bf.py +++ b/python/lsst/eo_utils/flat/flat_bf.py @@ -91,15 +91,15 @@ def extract(self, butler, data, **kwargs): flat_1 = get_ccd_from_id(butler, id_1, []) flat_2 = get_ccd_from_id(butler, id_2, []) - amps = get_amp_list(butler, flat_1) + amps = get_amp_list(flat_1) for i, amp in enumerate(amps): - regions = get_geom_regions(butler, flat_1, amp) + regions = get_geom_regions(flat_1, amp) serial_oscan = regions['serial_overscan'] imaging = regions['imaging'] #imaging.grow(-20) - im_1 = get_raw_image(butler, flat_1, amp) - im_2 = get_raw_image(butler, flat_2, amp) + im_1 = get_raw_image(flat_1, amp) + im_2 = get_raw_image(flat_2, amp) superbias_im = self.get_superbias_amp_image(butler, superbias_frame, amp) diff --git a/python/lsst/eo_utils/flat/flat_pair.py b/python/lsst/eo_utils/flat/flat_pair.py index 8979e30..6c40c52 100644 --- a/python/lsst/eo_utils/flat/flat_pair.py +++ b/python/lsst/eo_utils/flat/flat_pair.py @@ -111,18 +111,18 @@ def extract(self, butler, data, **kwargs): flat_1 = get_ccd_from_id(butler, id_1, []) flat_2 = get_ccd_from_id(butler, id_2, []) - amps = get_amp_list(butler, flat_1) + amps = get_amp_list(flat_1) - flux_1 = get_exposure_time(butler, flat_1) - flux_2 = get_exposure_time(butler, flat_2) + flux_1 = get_exposure_time(flat_1) + flux_2 = get_exposure_time(flat_2) if flux_1 != flux_2: raise RuntimeError("Exposure times do not match for:\n%s\n%s\n" % (id_1, id_1)) data_dict['EXPTIME'].append(flux_1) - mondiode_1 = get_mondiode_val(butler, flat_1) - mondiode_2 = get_mondiode_val(butler, flat_2) + mondiode_1 = get_mondiode_val(flat_1) + mondiode_2 = get_mondiode_val(flat_2) if mondiode_1 is not None: flux_1 *= mondiode_1 @@ -138,9 +138,9 @@ def extract(self, butler, data, **kwargs): data_dict['FLUX'].append(flux) - ccd_1_ims = unbiased_ccd_image_dict(butler, flat_1, bias=self.config.bias, + ccd_1_ims = unbiased_ccd_image_dict(flat_1, bias=self.config.bias, superbias_frame=superbias_frame) - ccd_2_ims = unbiased_ccd_image_dict(butler, flat_2, bias=self.config.bias, + ccd_2_ims = unbiased_ccd_image_dict(flat_2, bias=self.config.bias, superbias_frame=superbias_frame) for i, amp in enumerate(amps): diff --git a/python/lsst/eo_utils/ppump/ppump_trap.py b/python/lsst/eo_utils/ppump/ppump_trap.py index f55429d..f0ead99 100644 --- a/python/lsst/eo_utils/ppump/ppump_trap.py +++ b/python/lsst/eo_utils/ppump/ppump_trap.py @@ -76,7 +76,7 @@ def extract(self, butler, data, **kwargs): # Analysis goes here, you should fill data_dict with data extracted # by the analysis # - ccd = get_ccd_from_id(butler, ppump_file, mask_files) + ccd = get_ccd_from_id(butler, ppump_file, mask_files, masked_ccd=True) my_traps = Traps(ccd, gains, cycles=self.config.cycles, C2_thresh=self.config.C2_thresh, diff --git a/python/lsst/eo_utils/qe/dust_color.py b/python/lsst/eo_utils/qe/dust_color.py index ad0e734..e2e5627 100644 --- a/python/lsst/eo_utils/qe/dust_color.py +++ b/python/lsst/eo_utils/qe/dust_color.py @@ -93,20 +93,20 @@ def extract(self, butler, data, **kwargs): ccd = get_ccd_from_id(butler, qe_file, mask_files) - data_dict['WL'].append(get_mono_wl(butler, ccd)) - data_dict['EXPTIME'].append(get_exposure_time(butler, ccd)) - data_dict['MONDIODE'].append(get_mondiode_val(butler, ccd)) + data_dict['WL'].append(get_mono_wl(ccd)) + data_dict['EXPTIME'].append(get_exposure_time(ccd)) + data_dict['MONDIODE'].append(get_mondiode_val(ccd)) - amps = get_amp_list(butler, ccd) + amps = get_amp_list(ccd) for i, amp in enumerate(amps): bbox_list = slot_bbox_dict[i] - regions = get_geom_regions(butler, ccd, amp) + regions = get_geom_regions(ccd, amp) serial_oscan = regions['serial_overscan'] imaging = regions['imaging'] superbias_im = self.get_superbias_amp_image(butler, superbias_frame, amp) - img = get_raw_image(butler, ccd, amp) + img = get_raw_image(ccd, amp) image = unbias_amp(img, serial_oscan, bias_type=self.config.bias, superbias_im=superbias_im, region=imaging) diff --git a/python/lsst/eo_utils/qe/qe_median.py b/python/lsst/eo_utils/qe/qe_median.py index 723b6e1..fc97884 100644 --- a/python/lsst/eo_utils/qe/qe_median.py +++ b/python/lsst/eo_utils/qe/qe_median.py @@ -83,11 +83,11 @@ def extract(self, butler, data, **kwargs): ccd = get_ccd_from_id(butler, qe_file, mask_files) - data_dict['WL'].append(get_mono_wl(butler, ccd)) - data_dict['EXPTIME'].append(get_exposure_time(butler, ccd)) - data_dict['MONDIODE'].append(get_mondiode_val(butler, ccd)) + data_dict['WL'].append(get_mono_wl(ccd)) + data_dict['EXPTIME'].append(get_exposure_time(ccd)) + data_dict['MONDIODE'].append(get_mondiode_val(ccd)) - unbiased_images = unbiased_ccd_image_dict(butler, ccd, + unbiased_images = unbiased_ccd_image_dict(ccd, bias=self.config.bias, superbias_frame=superbias_frame) diff --git a/python/lsst/eo_utils/sflat/sflat_ratio.py b/python/lsst/eo_utils/sflat/sflat_ratio.py index 7f20713..8186043 100644 --- a/python/lsst/eo_utils/sflat/sflat_ratio.py +++ b/python/lsst/eo_utils/sflat/sflat_ratio.py @@ -106,16 +106,16 @@ def extract(self, butler, data, **kwargs): # by the analysis # - amps = get_amp_list(None, ratio_frame) + amps = get_amp_list(ratio_frame) for i, amp in enumerate(amps): - dims = get_dims_from_ccd(None, ratio_frame) - regions = get_geom_regions(None, ratio_frame, amp) + dims = get_dims_from_ccd(ratio_frame) + regions = get_geom_regions(ratio_frame, amp) imaging = regions['imaging'] - l_im = get_raw_image(None, l_frame, amp) - h_im = get_raw_image(None, h_frame, amp) - ratio_im = get_raw_image(None, ratio_frame, amp) + l_im = get_raw_image(l_frame, amp) + h_im = get_raw_image(h_frame, amp) + ratio_im = get_raw_image(ratio_frame, amp) if superbias_frame is not None: - superbias_im = get_raw_image(None, superbias_frame, amp) + superbias_im = get_raw_image(superbias_frame, amp) else: superbias_im = None diff --git a/python/lsst/eo_utils/sflat/superflat.py b/python/lsst/eo_utils/sflat/superflat.py index 6910447..e3d0a75 100644 --- a/python/lsst/eo_utils/sflat/superflat.py +++ b/python/lsst/eo_utils/sflat/superflat.py @@ -12,6 +12,8 @@ from lsst.eo_utils.base.file_utils import makedir_safe +from lsst.eo_utils.base.butler_utils import get_filename_from_id + from lsst.eo_utils.base.defaults import DEFAULT_STAT_TYPE from lsst.eo_utils.base.config_utils import EOUtilOptions @@ -155,12 +157,17 @@ def make_superflats(self, butler, data, **kwargs): if not self.config.skip: sflats = self.extract(butler, data) + if butler is None: + template_file = data['SFLAT'][0] + else: + template_file = get_filename_from_id(butler, data['SFLAT'][0]) + imutil.writeFits(sflats[0], output_file + '_l.fits', - data['SFLAT'][0], self.config.bitpix) + template_file, self.config.bitpix) imutil.writeFits(sflats[1], output_file + '_h.fits', - data['SFLAT'][0], self.config.bitpix) + template_file, self.config.bitpix) imutil.writeFits(sflats[2], output_file + '_ratio.fits', - data['SFLAT'][0], self.config.bitpix) + template_file, self.config.bitpix) if butler is not None: flip_data_in_place(output_file + '_l.fits') flip_data_in_place(output_file + '_h.fits') @@ -190,9 +197,9 @@ def plot(self, dtables, figs, **kwargs): raise ValueError("dtables should not be set in SuperflatTask.plot") if self.config.plot: - figs.plot_sensor("img_l", None, self._superflat_frame_l) - figs.plot_sensor("img_h", None, self._superflat_frame_h) - figs.plot_sensor("ratio", None, self._superflat_frame_r) + figs.plot_sensor("img_l", self._superflat_frame_l) + figs.plot_sensor("img_h", self._superflat_frame_h) + figs.plot_sensor("ratio", self._superflat_frame_r) default_array_kw = {} if self.config.stats_hist: @@ -355,15 +362,13 @@ def extract(self, butler, data, **kwargs): self._sflat_file_dict_h[slot] = basename.replace('.fits.fits', '_h.fits') self._sflat_file_dict_r[slot] = basename.replace('.fits.fits', '_ratio.fits') - self._sflat_images_h, ccd_dict = extract_raft_unbiased_images(None, - self._sflat_file_dict_h, + self._sflat_images_h, ccd_dict = extract_raft_unbiased_images(self._sflat_file_dict_h, mask_dict=self._mask_file_dict) - self._sflat_array_l = extract_raft_array_dict(None, self._sflat_file_dict_l, + self._sflat_array_l = extract_raft_array_dict(self._sflat_file_dict_l, mask_dict=self._mask_file_dict) - self._sflat_array_h = extract_raft_imaging_data(None, - self._sflat_images_h, + self._sflat_array_h = extract_raft_imaging_data(self._sflat_images_h, ccd_dict) - self._sflat_array_r = extract_raft_array_dict(None, self._sflat_file_dict_r, + self._sflat_array_r = extract_raft_array_dict(self._sflat_file_dict_r, mask_dict=self._mask_file_dict) out_data_l = outlier_raft_dict(self._sflat_array_l, 1000., 300.) out_data_h = outlier_raft_dict(self._sflat_array_h, 50000., 15000.) diff --git a/templates/raft_analysis.py b/templates/raft_analysis.py index affe443..171e230 100644 --- a/templates/raft_analysis.py +++ b/templates/raft_analysis.py @@ -1,4 +1,4 @@ -"""Class to analyze the correlations between the overscans for all amplifiers on a raft""" +"""Class to analyze data for all ccds on a raft""" from lsst.eo_utils.base.defaults import ALL_SLOTS diff --git a/templates/summary_analysis.py b/templates/summary_analysis.py index f41383d..b9d697a 100644 --- a/templates/summary_analysis.py +++ b/templates/summary_analysis.py @@ -1,4 +1,4 @@ -"""Class to analyze the FFT of the bias frames""" +"""Class to summarize data from multiple runs""" from lsst.eo_utils.base.config_utils import EOUtilOptions diff --git a/templates/table_analysis.py b/templates/table_analysis.py index e617ab9..5064cd3 100644 --- a/templates/table_analysis.py +++ b/templates/table_analysis.py @@ -1,4 +1,4 @@ -"""Class to analyze the FFT of the bias frames""" +"""Class to analyze tabular data for a given CCD""" import sys