From 59d499ab35aa1fcb91a6e2cbce1718e976f7c7e5 Mon Sep 17 00:00:00 2001 From: Eric Charles Date: Mon, 28 Oct 2019 17:04:13 -0700 Subject: [PATCH] Eac dev (#20) * Added data_source to called to get data * account for different indexing in superbias image v. bulterized ccd image * Fixed notebooks for ncsa * Updated NCSA ts8 butler repo location * delinting * Skip glob based tests when not at slac * Don't run file linking tests when not at SLAC * Skip tests that require non-bulter data when not at SLAC * Changes for NCSA unit tests * whitespace cleanup * Fixes for masking and delinting * Fixes for BOT running with some rebs off * fixes for BOT running at NCSA * Allow for sending jobs to batch by raft * fix batch job output files * various changes * changes to dispatch jobs over each CCD individually * return the full VARIANCE when making FlatPair statistics * Added null point correction in nonlinearity task * changes for NCSA slurm * Fix typo in raft list * Fix up batch parallelization * Added analysis to .gitignore * Change cut between low and hig flats to be 20s * Fixes for superbias_stats * Added NAMPS header keyword to file generated by flat_pair.py * Fixes for nine raft testing and to handle bias subtraction of butlerized data * Fixes for nine raft testing and to handle bias subtraction of butlerized data * fix tests/test_02_bias.py * Fix passing teststand to get_butler function * fix manufacturer lookup * Fix a couple of unit tests * Allow stacking to skip corrupted files * Change to using LSST_NUM to identify CCD type * fix log warning * Changes to pass along data_source to analysis classes * Fixes for using data_source==butler_file * update .pylintrc to include more generated functions * Delinting * update .pylintrc to allow for longer functions * Delinting * delinting * Slurm changes * Fixes for running with data_source=butler_file --- .pylintrc | 6 +- python/lsst/eo_utils/base/analysis.py | 98 +++++++++++++------ python/lsst/eo_utils/base/batch_utils.py | 3 +- python/lsst/eo_utils/base/data_access.py | 2 +- python/lsst/eo_utils/base/defaults.py | 12 ++- python/lsst/eo_utils/base/file_utils.py | 15 +-- python/lsst/eo_utils/base/image_utils.py | 48 ++++++--- python/lsst/eo_utils/base/iter_utils.py | 33 +++++-- python/lsst/eo_utils/bias/bias_fft.py | 8 +- python/lsst/eo_utils/bias/bias_struct.py | 4 +- python/lsst/eo_utils/bias/bias_v_row.py | 4 +- python/lsst/eo_utils/bias/correl_wrt_oscan.py | 4 +- python/lsst/eo_utils/bias/oscan_amp_stack.py | 4 +- python/lsst/eo_utils/bias/oscan_correl.py | 4 +- python/lsst/eo_utils/bias/superbias.py | 6 +- python/lsst/eo_utils/bias/superbias_stats.py | 7 +- python/lsst/eo_utils/dark/analysis.py | 4 +- python/lsst/eo_utils/dark/dark_current.py | 6 +- python/lsst/eo_utils/dark/superdark.py | 17 +++- .../eo_utils/flat/dust_linearity_analysis.py | 5 +- python/lsst/eo_utils/flat/flat_bf.py | 6 +- python/lsst/eo_utils/flat/flat_oscan.py | 4 +- python/lsst/eo_utils/flat/flat_pair.py | 6 +- python/lsst/eo_utils/flat/nonlinearity.py | 64 ++++++++---- python/lsst/eo_utils/flat/ptc.py | 6 +- python/lsst/eo_utils/ppump/ppump_trap.py | 4 +- python/lsst/eo_utils/qe/dust_color.py | 6 +- python/lsst/eo_utils/qe/qe_median.py | 6 +- python/lsst/eo_utils/sflat/analysis.py | 4 +- python/lsst/eo_utils/sflat/file_utils.py | 3 +- python/lsst/eo_utils/sflat/sflat_ratio.py | 22 ++--- python/lsst/eo_utils/sflat/superflat.py | 10 +- tests/test_00_base.py | 10 +- tests/test_02_bias.py | 4 +- tests/test_06_flat.py | 4 + tests/test_08_qe.py | 6 +- tests/utils.py | 6 +- 37 files changed, 288 insertions(+), 173 deletions(-) diff --git a/.pylintrc b/.pylintrc index 9f5de4f..2dfa84e 100644 --- a/.pylintrc +++ b/.pylintrc @@ -259,7 +259,7 @@ ignored-classes= # List of members which are set dynamically and missed by pylint inference # system, and so shouldn't trigger E1101 when accessed. Python regular # expressions are accepted. -generated-members=StatisticsControl,makeStatistics,MEAN,MEDIAN,STDEVCLIP,Point2I,Extent2I,Box2I,Threshold,FootprintSet,data +generated-members=StatisticsControl,makeStatistics,MEAN,MEDIAN,STDEVCLIP,VARIANCE,Point2I,Extent2I,Box2I,Threshold,FootprintSet,data @@ -321,7 +321,7 @@ max-returns=6 max-branches=25 # Maximum number of statements in function / method body -max-statements=70 +max-statements=100 # Maximum number of parents for a class (see R0901). max-parents=9 @@ -333,7 +333,7 @@ max-attributes=10 min-public-methods=2 # Maximum number of public methods for a class (see R0904). -max-public-methods=35 +max-public-methods=50 # Maximum number of boolean expressions in a if statement max-bool-expr=5 diff --git a/python/lsst/eo_utils/base/analysis.py b/python/lsst/eo_utils/base/analysis.py index ff3ab30..5977562 100644 --- a/python/lsst/eo_utils/base/analysis.py +++ b/python/lsst/eo_utils/base/analysis.py @@ -7,6 +7,7 @@ import glob +import numpy as np import lsst.pex.config as pexConfig @@ -225,7 +226,6 @@ def __call__(self, butler, data, **kwargs): """ raise NotImplementedError() - def get_filename_from_format(self, formatter, suffix, **kwargs): """Use a `FilenameFormat` object to construct a filename for a specific set of input parameters. @@ -256,34 +256,6 @@ def get_filename_from_format(self, formatter, suffix, **kwargs): return formatter(**format_vals) - @staticmethod - def get_superbias_amp_image(butler, superbias_frame, amp): - """Get the image for one amp for the superbias - - Parameters - ---------- - butler : `Butler` or `None` - Data Butler (or none) - superbias_frame : `MaskedCCD` or `None` - superbias image for the whole CCD - amp : `int` - Amplifier index - - Returns - ------- - superbias_im : `ImageF` - The image for the requested amplifier - """ - if superbias_frame is not None: - if butler is not None: - superbias_im = get_raw_image(superbias_frame, amp+1) - else: - superbias_im = get_raw_image(superbias_frame, amp) - else: - superbias_im = None - return superbias_im - - def get_superbias_file(self, suffix, **kwargs): """Get the name of the superbias file for a particular run, raft, ccd... @@ -499,6 +471,7 @@ def __init__(self, **kwargs): Used to override default configuration """ BaseAnalysisTask.__init__(self, **kwargs) + self._handler_config = None def get_suffix(self, **kwargs): """Get the suffix to add to table and plot filenames @@ -551,6 +524,36 @@ def plotfile_name(self, **kwargs): self.get_suffix(), **kwargs) + + def get_superbias_amp_image(self, butler, superbias_frame, amp): + """Get the image for one amp for the superbias + + Parameters + ---------- + butler : `Butler` or `None` + Data Butler (or none) + superbias_frame : `MaskedCCD` or `None` + superbias image for the whole CCD + amp : `int` + Amplifier index + + Returns + ------- + superbias_im : `ImageF` + The image for the requested amplifier + """ + offset = 0 + if self._handler_config is not None: + if self._handler_config.data_source == 'butler': + offset = 1 + + if superbias_frame is not None: + superbias_im = get_raw_image(superbias_frame, amp+offset) + else: + superbias_im = None + return superbias_im + + def get_superbias_frame(self, mask_files, **kwargs): """Get the superbias frame for a particular run, raft, ccd... @@ -667,10 +670,47 @@ def __call__(self, butler, data, **kwargs): Used to override default configuration """ self.safe_update(**kwargs) + self._handler_config = kwargs.get('handler_config', None) dtables = self.make_datatables(butler, data) if self.config.plot is not None: self.make_plots(dtables) + + def get_ccd(self, butler, data_id, mask_files, **kwargs): + """CCD image from a data_id + + If we are using `Butler` then this will take a + data_id `dict` ojbect and return an `ExposureF` object + + If we are not using `Butler` (i.e., if bulter is `None`) + then this will take a filename and return a `MaskedCCD` object + + Parameters + ---------- + butler : `Butler` or `None` + Data Butler + data_id : `dict` or `str` + Data identier + mask_files : `list` + List of data_ids for the files to construct the pixel mask + + Keywords + -------- + bias_frame : `ExposureF` or `MaskedCCD` or `None` + Object with the bias data + masked_ccd : `bool` + Use bulter only to get filename, return as MaskedCCD object + + Returns + ------- + ccd : `ExposureF` or `MaskedCCD` + CCD data object + """ + if self._handler_config is not None: + use_masked_ccd = self._handler_config.data_source == 'butler_file' + kwargs.setdefault('masked_ccd', use_masked_ccd) + return get_ccd_from_id(butler, data_id, mask_files, **kwargs) + @abc.abstractmethod def extract(self, butler, data, **kwargs): """This needs to be implemented by the sub-class diff --git a/python/lsst/eo_utils/base/batch_utils.py b/python/lsst/eo_utils/base/batch_utils.py index 01d5d5f..1d0aeaa 100644 --- a/python/lsst/eo_utils/base/batch_utils.py +++ b/python/lsst/eo_utils/base/batch_utils.py @@ -107,7 +107,8 @@ def dispatch_job(jobname, logfile, **kwargs): sub_com += " %s " % batch_args elif disptach.find('slurm') == 0: batchfile = write_slurm_batchfile(jobname, logfile, **kwargs) - sub_com = "sbatch %s -o %s" % (batchfile, os.path.join('sbatch', logfile.replace('.fits', '.out'))) + logfile_job = os.path.join('sbatch', logfile.replace('.fits', '.out')) + sub_com = "sbatch %s -o %s -e %s" % (batchfile, logfile_job, logfile_job) else: sub_com = "" diff --git a/python/lsst/eo_utils/base/data_access.py b/python/lsst/eo_utils/base/data_access.py index cabc322..5964610 100644 --- a/python/lsst/eo_utils/base/data_access.py +++ b/python/lsst/eo_utils/base/data_access.py @@ -66,7 +66,7 @@ def get_trait(self, traitname, **kwargs): data_source = kwargs.get('data_source', DEFAULT_DATA_SOURCE) if teststand in ['bot_etu']: teststand = 'bot' - if data_source in ['butler', 'butler_files']: + if data_source in ['butler', 'butler_file']: data_source = 'butler' key = '%s_%s_%s' % (teststand, data_source, traitname) diff --git a/python/lsst/eo_utils/base/defaults.py b/python/lsst/eo_utils/base/defaults.py index 3b5e761..216edf2 100644 --- a/python/lsst/eo_utils/base/defaults.py +++ b/python/lsst/eo_utils/base/defaults.py @@ -15,21 +15,21 @@ BUTLER_BOT_REPO = '/gpfs/slac/lsst/fs3/g/data/datasets/bot' ARCHIVE_DIR = '/gpfs/slac/lsst/fs*/g/data/jobHarness/jh_archive*' DEFAULT_DATA_SOURCE = os.environ.get('EO_DATA_SOURCE', 'glob') - DEFAULT_BATCH_ARGS = '-W 1200 -R bullet' + DEFAULT_BATCH_ARGS = '-W 1200 -R bubble' BATCH_SYSTEM = 'lsf' elif SITE == 'ncsa': BUTLER_TS8_REPO = '/datasets/ts8/repo' BUTLER_BOT_REPO = '/project/production/tmpdataloc/BOT/gen2repo' ARCHIVE_DIR = None DEFAULT_BATCH_ARGS = "" - DEFAULT_DATA_SOURCE = os.environ.get('EO_DATA_SOURCE', 'butler') + DEFAULT_DATA_SOURCE = os.environ.get('EO_DATA_SOURCE', 'butler_file') BATCH_SYSTEM = 'slurm' else: raise ValueError("Unknown site %s" % SITE) # TEST -DEFAULT_TESTSTAND = os.environ.get('EO_TESTSTSAND', 'bot') +DEFAULT_TESTSTAND = os.environ.get('EO_TESTSTAND', 'bot') if os.environ.get('EO_PRINT_OPTS', False): sys.stdout.write("SITE=%s\n" % SITE) @@ -45,12 +45,18 @@ # The slots ALL_SLOTS = ['S00', 'S01', 'S02', 'S10', 'S11', 'S12', 'S20', 'S21', 'S22'] + # The rafts ALL_RAFTS_BOT_ETU = ["R10", "R22"] NINE_RAFTS = ["R01", "R02", "R10", "R11", "R12", "R20", "R21", "R22", "R30"] + +RAFT_NAMES_DICT = dict(bot_etu=ALL_RAFTS_BOT_ETU, + bot_9=NINE_RAFTS, + bot=NINE_RAFTS) + # Various types of tests TS8_MASK_TEST_TYPES = ['bright_pixel_mask', 'dark_pixel_mask', diff --git a/python/lsst/eo_utils/base/file_utils.py b/python/lsst/eo_utils/base/file_utils.py index 5b853bf..cacda7d 100644 --- a/python/lsst/eo_utils/base/file_utils.py +++ b/python/lsst/eo_utils/base/file_utils.py @@ -18,7 +18,7 @@ print("Warning, no datacat-utilities") -from .defaults import NINE_RAFTS, ALL_SLOTS, ARCHIVE_DIR +from .defaults import ALL_SLOTS, ARCHIVE_DIR, RAFT_NAMES_DICT # These are the standard input filenames @@ -367,7 +367,7 @@ def get_bot_files_glob(**kwargs): kwcopy = kwargs.copy() test_name = kwcopy.pop('testName').lower() nfiles = kwcopy.get('nfiles', None) - rafts = get_raft_names_dc(kwcopy['run']) + rafts = get_raft_names_dc(kwcopy['run'], kwcopy.get('teststand', 'bot')) for raft in rafts: raftdict = {} @@ -458,6 +458,7 @@ def get_files_for_run(run_id, **kwargs): outkey = kwargs.get('outkey', imagetype) matchstr = kwargs.get('matchstr', None) nfiles = kwargs.get('nfiles', None) + teststand = kwargs.get('teststand', 'bot') outdict = {} @@ -485,11 +486,13 @@ def get_files_for_run(run_id, **kwargs): r_dict = get_ts8_files_glob(run=run_id, testName=test_type, imgtype=imgtype, - raft=hinfo[1]) + raft=hinfo[1], + teststand=teststand) else: r_dict = get_bot_files_glob(run=run_id, testName=test_type, - imgtype=imgtype) + imgtype=imgtype, + teststand=teststand) for key, val in r_dict.items(): if hinfo[0] == 'LCA-11021': # Raft level data @@ -644,7 +647,7 @@ def read_runlist(filepath): return outlist -def get_raft_names_dc(run): +def get_raft_names_dc(run, teststand='bot'): """Get the list of rafts used for a particular run Parameters @@ -668,7 +671,7 @@ def get_raft_names_dc(run): if htype == 'LCA-11021': return [hid] if htype == 'LCA-10134': - return NINE_RAFTS + return RAFT_NAMES_DICT[teststand] raise ValueError("Unrecognized hardware type %s" % htype) diff --git a/python/lsst/eo_utils/base/image_utils.py b/python/lsst/eo_utils/base/image_utils.py index 2e44a66..92b8bd4 100644 --- a/python/lsst/eo_utils/base/image_utils.py +++ b/python/lsst/eo_utils/base/image_utils.py @@ -15,8 +15,7 @@ import lsst.afw.detection as afwDetect -#import lsst.afw.geom as afwGeom -from lsst.geom import Point2I, Extent2I, Box2I +import lsst.geom as afwGeom from lsst.pex.exceptions.wrappers import LengthError @@ -169,10 +168,14 @@ def get_geom_steps_from_amp(ccd, amp): step_y : `int` Step to take in y to go from readout to physical order """ - try: - manu = ccd.getInfo().getMetadata().getString('CCD_MANU') - except Exception: + if isinstance(ccd, MaskedCCD): + try: + manu = ccd.md.get('CCD_MANU') + except KeyError: + manu = ccd.md.get('LSST_NUM')[0:3] + else: manu = ccd.getDetector().getSerial()[0:3] + if manu == 'ITL': flip_y = -1 elif manu == 'E2V': @@ -206,7 +209,6 @@ def flip_data_in_place(filepath): hdus[amp].data = hdus[amp].data[::step_x, ::step_y] hdus.writeto(filepath, overwrite=True) - def get_geom_regions(ccd, amp): """Get the ccd amp bounding boxes for a particular dataId or file @@ -584,6 +586,8 @@ def unbiased_ccd_image_dict(ccd, **kwargs): amps = get_amp_list(ccd) + is_masked_ccd = isinstance(ccd, MaskedCCD) + o_dict = {} offset = get_amp_offset(ccd, superbias_frame) @@ -598,6 +602,12 @@ def unbiased_ccd_image_dict(ccd, **kwargs): img = get_raw_image(ccd, amp) superbias_im = raw_amp_image(superbias_frame, amp + offset) + if not is_masked_ccd and superbias_frame is not None: + # Flip the superbias image for the subtraction + (step_x, step_y) = get_geom_steps_from_amp(superbias_frame, amp + offset) + superbias_im.mask.array = superbias_im.mask.array[::step_x, ::step_y] + superbias_im.image.array = superbias_im.image.array[::step_x, ::step_y] + image = unbias_amp(img, serial_oscan, bias_type=bias_type, superbias_im=superbias_im, region=trim_region) if nlc is not None: @@ -631,10 +641,15 @@ def extract_ccd_array_dict(data_id, **kwargs): """ kwcopy = kwargs.copy() mask_files = kwcopy.pop('mask_files', []) - ccd = get_ccd_from_id(None, data_id, mask_files) + o_dict = {} + + try: + ccd = get_ccd_from_id(None, data_id, mask_files) + except Exception: + return o_dict + unbiased_images = unbiased_ccd_image_dict(ccd, **kwargs) - o_dict = {} for amp, image in unbiased_images.items(): regions = get_geom_regions(ccd, amp) frames = get_image_frames_2d(image, regions) @@ -872,13 +887,20 @@ def stack_images(butler, in_files, statistic=afwMath.MEDIAN, **kwargs): out_dict = {} exp_time = 0.0 + used_files = 0 for ifile, in_file in enumerate(in_files): if ifile % 10 == 0: if log is not None: log.info(" %i" % ifile) - ccd = get_ccd_from_id(butler, in_file, mask_files=[]) + try: + ccd = get_ccd_from_id(butler, in_file, mask_files=[]) + except Exception: + if log is not None: + log.warn(" Failed to read %s, skipping" % (str(in_file))) + + used_files += 1 exp_time += get_exposure_time(ccd) amps = get_amp_list(ccd) @@ -899,7 +921,7 @@ def stack_images(butler, in_files, statistic=afwMath.MEDIAN, **kwargs): superbias_im=superbias_im)) if in_files: - exp_time /= len(in_files) + exp_time /= float(used_files) out_dict['METADATA'] = dict(EXPTIME=exp_time) for key, val in amp_stack_dict.items(): @@ -1150,9 +1172,9 @@ def fill_footprint_dict(image, fp_dict, amp, slot, **kwargs): fp_dict['x_peak'].append(peak_x) fp_dict['y_peak'].append(peak_y) - peak = Point2I(peak_x, peak_y) - extent = Extent2I(1, 1) - bbox_expand = Box2I(peak, extent) + peak = afwGeom.Point2I(peak_x, peak_y) + extent = afwGeom.Extent2I(1, 1) + bbox_expand = afwGeom.Box2I(peak, extent) fp_dict['ratio_full'].append(np.mean(cutout)/median) diff --git a/python/lsst/eo_utils/base/iter_utils.py b/python/lsst/eo_utils/base/iter_utils.py index 6e306bb..4bd650b 100644 --- a/python/lsst/eo_utils/base/iter_utils.py +++ b/python/lsst/eo_utils/base/iter_utils.py @@ -5,7 +5,7 @@ import lsst.pex.config as pexConfig -from .defaults import ALL_SLOTS, NINE_RAFTS +from .defaults import ALL_SLOTS, RAFT_NAMES_DICT from .config_utils import EOUtilOptions, Configurable,\ setup_parser, add_pex_arguments,\ @@ -59,21 +59,28 @@ def __init__(self, task, **kwargs): self._task = task self._butler = None - def get_butler(self): + def get_butler(self, **kwargs): """Return a data Butler This uses the config.data_surce and config.teststand parameters to pick the correct `Butler` or `None` if not using butler + Keywords + -------- + teststand : `str` + The teststand we arg getting a butler for + Returns ------- butler : `Butler` The requested data Butler """ + teststand = kwargs.get('teststand', self._task.config.teststand) + if self.config.data_source not in ['butler', 'butler_file']: self._butler = None else: - self._butler = get_butler_by_repo(self._task.config.teststand) + self._butler = get_butler_by_repo(teststand) return self._butler @@ -155,6 +162,7 @@ def call_analysis_task(self, **kwargs): Data `Butler` that is being used to access data """ kwargs.setdefault('butler', self._butler) + kwargs.setdefault('handler_config', self.config) if kwargs.get('dry_run', False): self._task.info("Skipping %s" % (kwargs)) return None @@ -353,7 +361,7 @@ def dispatch_single_run(self, run, **kwargs): """ taskname = self._task.getName().replace('Task', '') - htype, hid = self.get_hardware(self._butler, run) + htype, _ = self.get_hardware(self._butler, run) if self.config.batch in ['None', 'none', None]: self.call_analysis_task(run, **kwargs) @@ -406,7 +414,7 @@ def run_with_args(self, **kwargs): else: raise ValueError("Either runs or dataset must be set") - self.get_butler() + self.get_butler(**kw_remain) for run in runs: if self.config.nofail: @@ -464,11 +472,12 @@ def dispatch_by_raft_slot(handler, taskname, run, rafts, slots, **kwargs): if slots is None: slots = ALL_SLOTS if rafts is None: - rafts = NINE_RAFTS + rafts = RAFT_NAMES_DICT[kwcopy.get('teststand', 'bot')] for raft in rafts: kwcopy['rafts'] = raft for slot in slots: - logfile_slot = handler.config.logfile.replace('.log', '%s_%s_%s_%s.log' % (taskname, run, raft, slot)) + logfile_slot = handler.config.logfile.replace('.log', + '%s_%s_%s_%s.log' % (taskname, run, raft, slot)) kwcopy['slots'] = slot kw_remain = handler.get_dispatch_args(run, **kwcopy) dispatch_job(jobname, logfile_slot, **kw_remain) @@ -490,7 +499,7 @@ def dispatch_by_raft(handler, taskname, run, rafts, **kwargs): jobname = "eo_task.py %s" % taskname kwcopy = kwargs.copy() if rafts is None: - rafts = NINE_RAFTS + rafts = RAFT_NAMES_DICT[kwcopy.get('teststand', 'bot')] for raft in rafts: logfile_raft = handler.config.logfile.replace('.log', '%s_%s_%s.log' % (taskname, run, raft)) kwcopy['rafts'] = raft @@ -660,6 +669,7 @@ def call_analysis_task(self, run, **kwargs): data_files = self.get_data(self._butler, run, **kwdata) kwargs['run'] = run + kwargs.setdefault('handler_config', self.config) if htype == "LCA-10134": iterate_over_rafts_slots(self._task, self._butler, data_files, **kwargs) @@ -743,6 +753,7 @@ def call_analysis_task(self, run, **kwargs): data_files = self.get_data(self._butler, run, **kwdata) kwargs['run'] = run + kwargs.setdefault('handler_config', self.config) if htype == "LCA-10134": iterate_over_rafts(self._task, self._butler, data_files, **kwargs) @@ -932,6 +943,8 @@ def call_analysis_task(self, run, **kwargs): """ data_files = self.get_data(self._butler, run, **kwargs) kwargs['run'] = run + kwargs.setdefault('handler_config', self.config) + if self._task is not None: self._task(self._butler, data_files, **kwargs) @@ -982,6 +995,7 @@ def call_analysis_task(self, **kwargs): ------ ValueError : If the hardware type (raft or focal plane) is not recognized """ + kwargs.setdefault('handler_config', self.config) if kwargs.get('dry_run', False): self._task.info("Skipping %s" % (kwargs)) return @@ -1061,6 +1075,7 @@ def call_analysis_task(self, **kwargs): kwargs Passed to get_data() and to the analysis function """ + kwargs.setdefault('handler_config', self.config) if kwargs.get('dry_run', False): self._task.info("Skipping %s" % (kwargs)) return @@ -1082,7 +1097,7 @@ def get_dispatch_args(self, **kwargs): """ kwcopy = kwargs.copy() kwcopy.pop('task', None) - optstring=make_argstring(self._task.config, **kwcopy) + optstring = make_argstring(self._task.config, **kwcopy) ret_dict = dict(optstring=optstring, batch_args=self.config.batch_args, batch=self.config.batch, diff --git a/python/lsst/eo_utils/bias/bias_fft.py b/python/lsst/eo_utils/bias/bias_fft.py index 4d0bd90..c718b59 100644 --- a/python/lsst/eo_utils/bias/bias_fft.py +++ b/python/lsst/eo_utils/bias/bias_fft.py @@ -13,7 +13,7 @@ from lsst.eo_utils.base.butler_utils import make_file_dict from lsst.eo_utils.base.image_utils import REGION_KEYS, REGION_NAMES,\ - raw_amp_image, get_readout_freqs_from_ccd, get_ccd_from_id, get_raw_image,\ + raw_amp_image, get_readout_freqs_from_ccd, get_raw_image,\ get_geom_regions, get_amp_list, get_image_frames_2d, array_struct, unbias_amp,\ get_amp_offset @@ -82,7 +82,7 @@ def extract(self, butler, data, **kwargs): if ifile % 10 == 0: self.log_progress(" %i" % ifile) - ccd = get_ccd_from_id(butler, bias_file, mask_files) + ccd = self.get_ccd(butler, bias_file, mask_files) if ifile == 0: freqs_dict = get_readout_freqs_from_ccd(ccd) print(freqs_dict.keys()) @@ -210,7 +210,7 @@ def get_ccd_data(for_whom, ccd, data, **kwargs): if key_str not in data[key_col]: data[key_col][key_str] = np.zeros((int(nvals/2), nfiles_used)) data[key_col][key_str][:, ifile] = np.sqrt(fft_mean_by_col[0:int(nvals/2)]) - + class SuperbiasFFTConfig(SuperbiasSlotTableAnalysisConfig): @@ -254,7 +254,7 @@ def extract(self, butler, data, **kwargs): mask_files = self.get_mask_files() superbias_file = data[0] - superbias = get_ccd_from_id(None, superbias_file, mask_files) + superbias = self.get_ccd(None, superbias_file, mask_files) fft_data = {} diff --git a/python/lsst/eo_utils/bias/bias_struct.py b/python/lsst/eo_utils/bias/bias_struct.py index 51fdb85..2d5e8fc 100644 --- a/python/lsst/eo_utils/bias/bias_struct.py +++ b/python/lsst/eo_utils/bias/bias_struct.py @@ -9,7 +9,7 @@ from lsst.eo_utils.base.butler_utils import make_file_dict from lsst.eo_utils.base.image_utils import REGION_KEYS, REGION_NAMES, REGION_LABELS,\ - get_dimension_arrays_from_ccd, get_ccd_from_id, get_raw_image, get_amp_offset,\ + get_dimension_arrays_from_ccd, get_raw_image, get_amp_offset,\ get_geom_regions, get_amp_list, get_image_frames_2d, array_struct, unbias_amp from lsst.eo_utils.base.iter_utils import AnalysisBySlot @@ -72,7 +72,7 @@ def extract(self, butler, data, **kwargs): if ifile % 10 == 0: self.log_progress(" %i" % ifile) - ccd = get_ccd_from_id(butler, bias_file, mask_files) + ccd = self.get_ccd(butler, bias_file, mask_files) if ifile == 0: dim_array_dict = get_dimension_arrays_from_ccd(ccd) for key, val in dim_array_dict.items(): diff --git a/python/lsst/eo_utils/bias/bias_v_row.py b/python/lsst/eo_utils/bias/bias_v_row.py index 257be39..1c8dfac 100644 --- a/python/lsst/eo_utils/bias/bias_v_row.py +++ b/python/lsst/eo_utils/bias/bias_v_row.py @@ -11,7 +11,7 @@ from lsst.eo_utils.base.butler_utils import make_file_dict from lsst.eo_utils.base.image_utils import get_dims_from_ccd,\ - get_ccd_from_id, get_raw_image, get_geom_regions, get_amp_list + get_raw_image, get_geom_regions, get_amp_list from lsst.eo_utils.base.iter_utils import AnalysisBySlot @@ -63,7 +63,7 @@ def extract(self, butler, data, **kwargs): if ifile % 10 == 0: self.log_progress(" %i" % ifile) - ccd = get_ccd_from_id(butler, bias_file, mask_files) + ccd = self.get_ccd(butler, bias_file, mask_files) if ifile == 0: dims = get_dims_from_ccd(ccd) xrow_s = np.linspace(0, dims['nrow_s']-1, dims['nrow_s']) diff --git a/python/lsst/eo_utils/bias/correl_wrt_oscan.py b/python/lsst/eo_utils/bias/correl_wrt_oscan.py index d11db68..77126e9 100644 --- a/python/lsst/eo_utils/bias/correl_wrt_oscan.py +++ b/python/lsst/eo_utils/bias/correl_wrt_oscan.py @@ -10,7 +10,7 @@ from lsst.eo_utils.base.butler_utils import make_file_dict -from lsst.eo_utils.base.image_utils import get_dims_from_ccd, get_ccd_from_id,\ +from lsst.eo_utils.base.image_utils import get_dims_from_ccd,\ get_raw_image, get_geom_regions, get_amp_list, get_image_frames_2d from lsst.eo_utils.base.iter_utils import AnalysisBySlot @@ -77,7 +77,7 @@ def extract(self, butler, data, **kwargs): if ifile % 10 == 0: self.log_progress(" %i" % ifile) - ccd = get_ccd_from_id(butler, bias_file, mask_files) + ccd = self.get_ccd(butler, bias_file, mask_files) if ifile == 0: dims = get_dims_from_ccd(ccd) nrow_i = dims['nrow_i'] diff --git a/python/lsst/eo_utils/bias/oscan_amp_stack.py b/python/lsst/eo_utils/bias/oscan_amp_stack.py index e8ebf86..0e5bc3f 100644 --- a/python/lsst/eo_utils/bias/oscan_amp_stack.py +++ b/python/lsst/eo_utils/bias/oscan_amp_stack.py @@ -12,7 +12,7 @@ from lsst.eo_utils.base.butler_utils import make_file_dict from lsst.eo_utils.base.image_utils import REGION_KEYS,\ - get_ccd_from_id, get_dimension_arrays_from_ccd + get_dimension_arrays_from_ccd from lsst.eo_utils.base.iter_utils import AnalysisBySlot @@ -77,7 +77,7 @@ def extract(self, butler, data, **kwargs): if ifile % 10 == 0: self.log_progress(" %i" % ifile) - ccd = get_ccd_from_id(butler, bias_file, mask_files) + ccd = self.get_ccd(butler, bias_file, mask_files) if ifile == 0: dim_array_dict = get_dimension_arrays_from_ccd(ccd) diff --git a/python/lsst/eo_utils/bias/oscan_correl.py b/python/lsst/eo_utils/bias/oscan_correl.py index 2048e23..3c75870 100644 --- a/python/lsst/eo_utils/bias/oscan_correl.py +++ b/python/lsst/eo_utils/bias/oscan_correl.py @@ -14,7 +14,7 @@ from lsst.eo_utils.base.butler_utils import make_file_dict -from lsst.eo_utils.base.image_utils import get_ccd_from_id, get_raw_image,\ +from lsst.eo_utils.base.image_utils import get_raw_image,\ get_geom_regions, get_amp_list, unbias_amp from lsst.eo_utils.base.iter_utils import AnalysisByRaft @@ -77,7 +77,7 @@ def extract(self, butler, data, **kwargs): mask_files = self.get_mask_files(slot=slot) superbias_frame = self.get_superbias_frame(mask_files, slot=slot) - ccd = get_ccd_from_id(butler, bias_files[0], []) + ccd = self.get_ccd(butler, bias_files[0], []) overscans += self.get_ccd_data(butler, ccd, superbias_frame=superbias_frame) namps = len(overscans) diff --git a/python/lsst/eo_utils/bias/superbias.py b/python/lsst/eo_utils/bias/superbias.py index 36f6e98..c9cd7cc 100644 --- a/python/lsst/eo_utils/bias/superbias.py +++ b/python/lsst/eo_utils/bias/superbias.py @@ -17,8 +17,8 @@ from lsst.eo_utils.base.data_utils import TableDict -from lsst.eo_utils.base.image_utils import get_ccd_from_id,\ - flip_data_in_place, stack_images, extract_raft_array_dict,\ +from lsst.eo_utils.base.image_utils import flip_data_in_place,\ + stack_images, extract_raft_array_dict,\ outlier_raft_dict from lsst.eo_utils.base.iter_utils import AnalysisBySlot @@ -152,7 +152,7 @@ def make_superbias(self, butler, slot_data, **kwargs): flip_data_in_place(output_file) - self._superbias_frame = get_ccd_from_id(None, output_file, mask_files) + self._superbias_frame = self.get_ccd(None, output_file, mask_files) def plot(self, dtables, figs, **kwargs): diff --git a/python/lsst/eo_utils/bias/superbias_stats.py b/python/lsst/eo_utils/bias/superbias_stats.py index 950e274..61a5efc 100644 --- a/python/lsst/eo_utils/bias/superbias_stats.py +++ b/python/lsst/eo_utils/bias/superbias_stats.py @@ -12,8 +12,7 @@ from lsst.eo_utils.base.butler_utils import make_file_dict -from lsst.eo_utils.base.image_utils import get_ccd_from_id,\ - get_raw_image, get_amp_list +from lsst.eo_utils.base.image_utils import get_raw_image, get_amp_list from lsst.eo_utils.base.factory import EO_TASK_FACTORY @@ -80,7 +79,7 @@ def extract(self, butler, data, **kwargs): mask_files = self.get_mask_files(slot=slot) try: - superbias_frame = get_ccd_from_id(None, superbias_file, mask_files) + superbias_frame = self.get_ccd(None, superbias_file, mask_files) except Exception: self.log.warn("Skipping %s:%s:%s" % (self.config.run, self.config.raft, slot)) superbias_frame = None @@ -143,7 +142,7 @@ def get_superbias_stats(superbias, stats_data, islot): amps = get_amp_list(superbias) for i, amp in enumerate(amps): - + img = get_raw_image(superbias, amp).image stats_data['mean'][islot, i] = img.array.mean() stats_data['median'][islot, i] = np.median(img.array) diff --git a/python/lsst/eo_utils/dark/analysis.py b/python/lsst/eo_utils/dark/analysis.py index ae5af57..bb34dc3 100644 --- a/python/lsst/eo_utils/dark/analysis.py +++ b/python/lsst/eo_utils/dark/analysis.py @@ -6,8 +6,6 @@ from lsst.eo_utils.base.iter_utils import AnalysisBySlot -from lsst.eo_utils.base.image_utils import get_ccd_from_id - from lsst.eo_utils.base.analysis import AnalysisConfig, AnalysisTask from lsst.eo_utils.dark.file_utils import SLOT_DARK_TABLE_FORMATTER, SLOT_DARK_PLOT_FORMATTER,\ @@ -77,4 +75,4 @@ def get_superdark_frame(self, mask_files, **kwargs): """ self.safe_update(**kwargs) superdark_file = self.get_superdark_file('') - return get_ccd_from_id(None, superdark_file, mask_files) + return self.get_ccd(None, superdark_file, mask_files) diff --git a/python/lsst/eo_utils/dark/dark_current.py b/python/lsst/eo_utils/dark/dark_current.py index 9878e43..57d5260 100644 --- a/python/lsst/eo_utils/dark/dark_current.py +++ b/python/lsst/eo_utils/dark/dark_current.py @@ -10,8 +10,8 @@ from lsst.eo_utils.base.stat_utils import gauss_fit -from lsst.eo_utils.base.image_utils import get_ccd_from_id,\ - get_exposure_time, get_amp_list, get_raw_image, get_geom_regions +from lsst.eo_utils.base.image_utils import get_exposure_time,\ + get_amp_list, get_raw_image, get_geom_regions from lsst.eo_utils.base.factory import EO_TASK_FACTORY @@ -85,7 +85,7 @@ def extract(self, butler, data, **kwargs): mask_files = self.get_mask_files(slot=slot) superdark_file = data[slot] - superdark_frame = get_ccd_from_id(None, superdark_file, mask_files) + superdark_frame = self.get_ccd(None, superdark_file, mask_files) exptime = get_exposure_time(superdark_frame) amps = get_amp_list(superdark_frame) diff --git a/python/lsst/eo_utils/dark/superdark.py b/python/lsst/eo_utils/dark/superdark.py index 9df9cab..fd6041b 100644 --- a/python/lsst/eo_utils/dark/superdark.py +++ b/python/lsst/eo_utils/dark/superdark.py @@ -1,5 +1,7 @@ """Class to construct superdark frames""" +import os + import lsst.afw.math as afwMath import lsst.eotest.image_utils as imutil @@ -16,8 +18,8 @@ from lsst.eo_utils.base.data_utils import TableDict -from lsst.eo_utils.base.image_utils import get_ccd_from_id,\ - flip_data_in_place, stack_images, extract_raft_array_dict,\ +from lsst.eo_utils.base.image_utils import flip_data_in_place,\ + stack_images, extract_raft_array_dict,\ outlier_raft_dict from lsst.eo_utils.bias.analysis import AnalysisTask @@ -132,6 +134,10 @@ def make_superdark(self, butler, slot_data, **kwargs): mask_files = self.get_mask_files() output_file = self.tablefile_name() + '.fits' + + if not slot_data['DARK']: + return + makedir_safe(output_file) if not self.config.skip: @@ -145,7 +151,7 @@ def make_superdark(self, butler, slot_data, **kwargs): if butler is not None: flip_data_in_place(output_file) - self._superdark_frame = get_ccd_from_id(None, output_file, mask_files) + self._superdark_frame = self.get_ccd(None, output_file, mask_files) def plot(self, dtables, figs, **kwargs): @@ -275,6 +281,9 @@ def extract(self, butler, data, **kwargs): slots = ALL_SLOTS for slot in slots: + if not os.path.exists(data[slot]): + self.log.warn("skipping missing data for %s:%s" % (self.config.raft, slot)) + continue mask_files = self.get_mask_files(slot=slot) self._mask_file_dict[slot] = mask_files self._sdark_file_dict[slot] = data[slot] @@ -302,7 +311,7 @@ def plot(self, dtables, figs, **kwargs): """ self.safe_update(**kwargs) - figs.make_raft_outlier_plots(dtables['outliers']) + #figs.make_raft_outlier_plots(dtables['outliers']) if self.config.skip: return diff --git a/python/lsst/eo_utils/flat/dust_linearity_analysis.py b/python/lsst/eo_utils/flat/dust_linearity_analysis.py index fde1a8d..c01a648 100644 --- a/python/lsst/eo_utils/flat/dust_linearity_analysis.py +++ b/python/lsst/eo_utils/flat/dust_linearity_analysis.py @@ -16,8 +16,7 @@ from lsst.eo_utils.base.iter_utils import AnalysisBySlot -from lsst.eo_utils.base.image_utils import get_ccd_from_id,\ - get_exposure_time, get_mondiode_val,\ +from lsst.eo_utils.base.image_utils import get_exposure_time, get_mondiode_val,\ unbiased_ccd_image_dict from lsst.eo_utils.base.factory import EO_TASK_FACTORY @@ -123,7 +122,7 @@ def extract(self, butler, data, **kwargs): if ifile % 10 == 0: self.log_progress(" %i" % ifile) - ccd = get_ccd_from_id(butler, flat1_file, mask_files) + ccd = self.get_ccd(butler, flat1_file, mask_files) # To be appended while looping over bounding boxes exptime = get_exposure_time(ccd) diff --git a/python/lsst/eo_utils/flat/flat_bf.py b/python/lsst/eo_utils/flat/flat_bf.py index 4e1df3f..a38af57 100644 --- a/python/lsst/eo_utils/flat/flat_bf.py +++ b/python/lsst/eo_utils/flat/flat_bf.py @@ -12,7 +12,7 @@ from lsst.eo_utils.base.iter_utils import AnalysisBySlot -from lsst.eo_utils.base.image_utils import get_ccd_from_id, get_amp_list,\ +from lsst.eo_utils.base.image_utils import get_amp_list,\ get_geom_regions, unbias_amp, get_raw_image from lsst.eo_utils.base.factory import EO_TASK_FACTORY @@ -88,8 +88,8 @@ def extract(self, butler, data, **kwargs): if ifile % 10 == 0: self.log_progress(" %i" % ifile) - flat_1 = get_ccd_from_id(butler, id_1, []) - flat_2 = get_ccd_from_id(butler, id_2, []) + flat_1 = self.get_ccd(butler, id_1, []) + flat_2 = self.get_ccd(butler, id_2, []) amps = get_amp_list(flat_1) diff --git a/python/lsst/eo_utils/flat/flat_oscan.py b/python/lsst/eo_utils/flat/flat_oscan.py index 813992d..149da18 100644 --- a/python/lsst/eo_utils/flat/flat_oscan.py +++ b/python/lsst/eo_utils/flat/flat_oscan.py @@ -12,8 +12,6 @@ from lsst.eo_utils.base.iter_utils import AnalysisBySlot -from lsst.eo_utils.base.image_utils import get_ccd_from_id - from lsst.eo_utils.base.factory import EO_TASK_FACTORY from .analysis import FlatAnalysisConfig, FlatAnalysisTask @@ -83,7 +81,7 @@ def extract(self, butler, data, **kwargs): if ifile % 10 == 0: self.log_progress(" %i" % ifile) - flat = get_ccd_from_id(butler, flat_id, mask_files, bias_frame=superbias_frame) + flat = self.get_ccd(butler, flat_id, mask_files, bias_frame=superbias_frame) fitter.process_image(flat, gains) self.xmax = fitter.xmax_val diff --git a/python/lsst/eo_utils/flat/flat_pair.py b/python/lsst/eo_utils/flat/flat_pair.py index 133b062..7af0907 100644 --- a/python/lsst/eo_utils/flat/flat_pair.py +++ b/python/lsst/eo_utils/flat/flat_pair.py @@ -20,7 +20,7 @@ from lsst.eo_utils.base.butler_utils import make_file_dict -from lsst.eo_utils.base.image_utils import get_ccd_from_id, get_amp_list,\ +from lsst.eo_utils.base.image_utils import get_amp_list,\ get_exposure_time, get_mondiode_val, get_mono_slit_b, unbiased_ccd_image_dict @@ -169,8 +169,8 @@ def extract(self, butler, data, **kwargs): if ifile % 10 == 0: self.log_progress(" %i" % ifile) - flat_1 = get_ccd_from_id(butler, id_1, mask_files) - flat_2 = get_ccd_from_id(butler, id_2, mask_files) + flat_1 = self.get_ccd(butler, id_1, mask_files) + flat_2 = self.get_ccd(butler, id_2, mask_files) amps = get_amp_list(flat_1) diff --git a/python/lsst/eo_utils/flat/nonlinearity.py b/python/lsst/eo_utils/flat/nonlinearity.py index 053093b..08ebbc3 100644 --- a/python/lsst/eo_utils/flat/nonlinearity.py +++ b/python/lsst/eo_utils/flat/nonlinearity.py @@ -48,7 +48,8 @@ class NonlinearityTask(FlatSlotTableAnalysisTask): model_func_choice = 1 do_profiles = True null_point = 0. - + num_profile_points = 40 + @staticmethod def _correct_null_point(profile_x, profile_y, null_point): """Force the spline to go through zero at a particular x-xvalue @@ -61,7 +62,7 @@ def _correct_null_point(profile_x, profile_y, null_point): The y-bin values null_point : `float` The x-value where the spline should go through zero - + Returns ------- y_vals_offset @@ -69,10 +70,13 @@ def _correct_null_point(profile_x, profile_y, null_point): """ try: uni_spline = UnivariateSpline(profile_x, profile_y) - offset = uni_spline(null_point) - except Exception: + offset = uni_spline(null_point) + print("offset", offset) + except Exception as msg: + print("Failed to extract null point") + print(msg) offset = 0. - return profile_y - offset + return ((1 + profile_y) / (1 + offset)) - 1. def extract(self, butler, data, **kwargs): @@ -173,20 +177,31 @@ def extract(self, butler, data, **kwargs): results, _, frac_resid, frac_resid_err =\ perform_linear_chisq_fit(amp_vals, flux_vals, mask, self.model_func_choice) + # Correct the error for the conversion to ne + frac_resid_err *= results[0][0] results_inv, _, frac_resid_inv, frac_resid_err_inv =\ perform_linear_chisq_fit(flux_vals, amp_vals, mask, self.model_func_choice) if self.do_profiles: - profile_xbins = np.linspace(0., amp_vals[mask].max(), 11) - profile_xbins_inv = np.linspace(0., flux_vals[mask].max(), 11) - - profile_x, profile_y, profile_yerr =\ - make_profile_hist(profile_xbins, amp_vals, frac_resid, - yerrs=frac_resid_err, stderr=True) - profile_x_inv, profile_y_inv, profile_yerr_inv =\ - make_profile_hist(profile_xbins_inv, flux_vals, frac_resid_inv, - yerrs=frac_resid_err_inv, stderr=True) + if self.num_profile_points is not None: + profile_xbins = np.linspace(0., amp_vals[mask].max(), self.num_profile_points) + profile_xbins_inv = np.linspace(0., flux_vals[mask].max(), self.num_profile_points) + + profile_x, profile_y, profile_yerr =\ + make_profile_hist(profile_xbins, amp_vals, frac_resid, + yerrs=frac_resid_err, stderr=True) + profile_x_inv, profile_y_inv, profile_yerr_inv =\ + make_profile_hist(profile_xbins_inv, flux_vals, frac_resid_inv, + yerrs=frac_resid_err_inv, stderr=True) + else: + idx_sort = np.argsort(amp_vals) + idx_sort_inv = np.argsort(flux_vals) + profile_x, profile_y, profile_yerr =\ + amp_vals[idx_sort], frac_resid[idx_sort], frac_resid_err[idx_sort] + profile_x_inv, profile_y_inv, profile_yerr_inv =\ + flux_vals[idx_sort_inv], frac_resid_inv[idx_sort_inv],\ + frac_resid_err_inv[idx_sort_inv] if self.null_point is not None: profile_y_corr = self._correct_null_point(profile_x, profile_y, self.null_point) @@ -235,7 +250,7 @@ def extract(self, butler, data, **kwargs): return outtables - def _plot_nonlinearity(self, flux, slit_widths, dtables, figs, inverse=False): + def _plot_nonlinearity(self, flux, slit_widths, dtables, figs, **kwargs): """Plot the non-linearity data Parameters @@ -248,9 +263,14 @@ def _plot_nonlinearity(self, flux, slit_widths, dtables, figs, inverse=False): The table dictionary figs : `FigureDict` The resulting figures + + Keywords + -------- inverse : bool Flip the x and y axes """ + inverse = kwargs.get('inverse', False) + if inverse: suffix = "_inv" xlabel_short = 'Flux [a.u.]' @@ -288,27 +308,31 @@ def _plot_nonlinearity(self, flux, slit_widths, dtables, figs, inverse=False): xscale='lin', yscale='log') fig_nonlin_log = figs.setup_figure("non_lin_log%s" % suffix, - xlabel=xlabel_full, ylabel=ylabel_resid_full) + xlabel=xlabel_full, ylabel=ylabel_resid_full, + figsize=(7, 5)) axes_nonlin_log = fig_nonlin_log['axes'] #axes_nonlin_log.set_xlim(1., 3000.) axes_nonlin_log.set_ylim(self.plot_resid_ymin, self.plot_resid_ymax) axes_nonlin_log.set_xscale('log') fig_nonlin = figs.setup_figure("non_lin%s" % suffix, - xlabel=xlabel_full, ylabel=ylabel_resid_full) + xlabel=xlabel_full, ylabel=ylabel_resid_full, + figsize=(7, 5)) axes_nonlin = fig_nonlin['axes'] #axes_nonlin.set_xlim(0., 3000.) axes_nonlin.set_ylim(self.plot_resid_ymin, self.plot_resid_ymax) fig_nonlin_log_stack = figs.setup_figure("non_lin_log_stack%s" % suffix, - xlabel=xlabel_full, ylabel=ylabel_resid_full) + xlabel=xlabel_full, ylabel=ylabel_resid_full, + figsize=(7, 5)) axes_nonlin_log_stack = fig_nonlin_log_stack['axes'] #axes_nonlin_log_stack.set_xlim(1., 3000.) axes_nonlin_log_stack.set_ylim(self.plot_resid_ymin, self.plot_resid_ymax) axes_nonlin_log_stack.set_xscale('log') fig_nonlin_stack = figs.setup_figure("non_lin_stack%s" % suffix, - xlabel=xlabel_full, ylabel=ylabel_resid_full) + xlabel=xlabel_full, ylabel=ylabel_resid_full, + figsize=(7, 5)) axes_nonlin_stack = fig_nonlin_stack['axes'] #axes_nonlin_stack.set_xlim(0., 3000.) axes_nonlin_stack.set_ylim(self.plot_resid_ymin, self.plot_resid_ymax) @@ -421,7 +445,7 @@ def plot(self, dtables, figs, **kwargs): slit_widths = np.hstack([slit_widths, slit_widths]) self._plot_nonlinearity(flux, slit_widths, dtables, figs) - self._plot_nonlinearity(flux, slit_widths, dtables, figs, True) + self._plot_nonlinearity(flux, slit_widths, dtables, figs, inverse=True) EO_TASK_FACTORY.add_task_class('Nonlinearity', NonlinearityTask) diff --git a/python/lsst/eo_utils/flat/ptc.py b/python/lsst/eo_utils/flat/ptc.py index 5689dae..82abd0d 100644 --- a/python/lsst/eo_utils/flat/ptc.py +++ b/python/lsst/eo_utils/flat/ptc.py @@ -185,11 +185,13 @@ def plot_fits(dtables, figs): fig_nonlin_log = figs.setup_figure("non_lin_log_%s" % slot, xlabel="Flux [a.u.]", - ylabel='Frac. Resid') + ylabel='Frac. Resid', + figsize=(7, 5)) axes_nonlin_log = fig_nonlin_log['axes'] axes_nonlin_log.set_xscale('log') - fig_nonlin = figs.setup_figure("non_lin_%s" % slot, xlabel="Flux [a.u.]", ylabel='Frac. Resid') + fig_nonlin = figs.setup_figure("non_lin_%s" % slot, xlabel="Flux [a.u.]", ylabel='Frac. Resid', + figsize=(7, 5)) axes_nonlin = fig_nonlin['axes'] for amp in range(16): diff --git a/python/lsst/eo_utils/ppump/ppump_trap.py b/python/lsst/eo_utils/ppump/ppump_trap.py index fd9238b..e8354b3 100644 --- a/python/lsst/eo_utils/ppump/ppump_trap.py +++ b/python/lsst/eo_utils/ppump/ppump_trap.py @@ -12,8 +12,6 @@ from lsst.eo_utils.base.iter_utils import AnalysisBySlot -from lsst.eo_utils.base.image_utils import get_ccd_from_id - from lsst.eo_utils.base.factory import EO_TASK_FACTORY from lsst.eo_utils.ppump.analysis import PpumpAnalysisConfig, PpumpAnalysisTask @@ -77,7 +75,7 @@ def extract(self, butler, data, **kwargs): # Analysis goes here, you should fill data_dict with data extracted # by the analysis # - ccd = get_ccd_from_id(butler, ppump_file, mask_files, masked_ccd=True) + ccd = self.get_ccd(butler, ppump_file, mask_files, masked_ccd=True) my_traps = Traps(ccd, gains, cycles=self.config.cycles, C2_thresh=self.config.C2_thresh, diff --git a/python/lsst/eo_utils/qe/dust_color.py b/python/lsst/eo_utils/qe/dust_color.py index 62effc6..ba3d757 100644 --- a/python/lsst/eo_utils/qe/dust_color.py +++ b/python/lsst/eo_utils/qe/dust_color.py @@ -12,7 +12,7 @@ from lsst.eo_utils.base.iter_utils import AnalysisBySlot -from lsst.eo_utils.base.image_utils import get_ccd_from_id, get_amp_list,\ +from lsst.eo_utils.base.image_utils import get_amp_list,\ get_geom_regions, get_raw_image, unbias_amp from lsst.eo_utils.base.factory import EO_TASK_FACTORY @@ -99,7 +99,7 @@ def extract(self, butler, data, **kwargs): # For this slot, loop over qe files, loop over amplifiers, loop over dust spots in that amplifier, # record the flux and median for each dust spot # Then loop over dust spots CCDs, amplifiers, and QE files to accumulate the output dictionary - ccd = get_ccd_from_id(butler, qe_files[0], mask_files) + ccd = self.get_ccd(butler, qe_files[0], mask_files) amps = get_amp_list(ccd) spot = 0 for ifile, qe_file in enumerate(qe_files): @@ -112,7 +112,7 @@ def extract(self, butler, data, **kwargs): self.log.warn("Skipping slot:amp %s:%i with %i defects" % (slot, amp, len(bbox_list))) continue - ccd = get_ccd_from_id(butler, qe_file, mask_files) + ccd = self.get_ccd(butler, qe_file, mask_files) regions = get_geom_regions(ccd, amp) serial_oscan = regions['serial_overscan'] imaging = regions['imaging'] diff --git a/python/lsst/eo_utils/qe/qe_median.py b/python/lsst/eo_utils/qe/qe_median.py index 730c109..f768ee0 100644 --- a/python/lsst/eo_utils/qe/qe_median.py +++ b/python/lsst/eo_utils/qe/qe_median.py @@ -12,8 +12,8 @@ from lsst.eo_utils.base.iter_utils import AnalysisBySlot -from lsst.eo_utils.base.image_utils import get_ccd_from_id,\ - get_exposure_time, get_mondiode_val, get_mono_wl,\ +from lsst.eo_utils.base.image_utils import get_exposure_time,\ + get_mondiode_val, get_mono_wl,\ unbiased_ccd_image_dict from lsst.eo_utils.base.factory import EO_TASK_FACTORY @@ -81,7 +81,7 @@ def extract(self, butler, data, **kwargs): if ifile % 10 == 0: self.log_progress(" %i" % ifile) - ccd = get_ccd_from_id(butler, qe_file, mask_files) + ccd = self.get_ccd(butler, qe_file, mask_files) data_dict['WL'].append(get_mono_wl(ccd)) data_dict['EXPTIME'].append(get_exposure_time(ccd)) diff --git a/python/lsst/eo_utils/sflat/analysis.py b/python/lsst/eo_utils/sflat/analysis.py index 27361fa..6247093 100644 --- a/python/lsst/eo_utils/sflat/analysis.py +++ b/python/lsst/eo_utils/sflat/analysis.py @@ -6,8 +6,6 @@ from lsst.eo_utils.base.iter_utils import AnalysisBySlot -from lsst.eo_utils.base.image_utils import get_ccd_from_id - from lsst.eo_utils.base.analysis import AnalysisConfig, AnalysisTask from lsst.eo_utils.sflat.file_utils import SLOT_SFLAT_TABLE_FORMATTER,\ @@ -82,6 +80,6 @@ def get_superflat_frames(self, mask_files, types=None, **kwargs): types = ['l', 'h', 'ratio'] superflat_file = self.get_superflat_file('').replace('.fits', '') - o_dict = {key:get_ccd_from_id(None, superflat_file + '_%s.fits' % key, mask_files) + o_dict = {key:self.get_ccd(None, superflat_file + '_%s.fits' % key, mask_files) for key in types} return o_dict diff --git a/python/lsst/eo_utils/sflat/file_utils.py b/python/lsst/eo_utils/sflat/file_utils.py index 2016ea4..4634fe3 100644 --- a/python/lsst/eo_utils/sflat/file_utils.py +++ b/python/lsst/eo_utils/sflat/file_utils.py @@ -5,7 +5,8 @@ """This module contains functions to find files of a particular type in the SLAC directory tree""" from lsst.eo_utils.base.file_utils import FILENAME_FORMATS,\ - SLOT_FORMAT_STRING, RAFT_FORMAT_STRING, SUMMARY_FORMAT_STRING + SLOT_FORMAT_STRING, RAFT_FORMAT_STRING, SUMMARY_FORMAT_STRING,\ + get_hardware_type_and_id, get_files_for_run SUPERFLAT_FORMAT_STRING =\ '{outdir}/{teststand}/superflat/{raft}/{raft}-{run}-{slot}_superflat_b-{bias}_s-{superbias}{suffix}_l' diff --git a/python/lsst/eo_utils/sflat/sflat_ratio.py b/python/lsst/eo_utils/sflat/sflat_ratio.py index 9db8203..9edbf35 100644 --- a/python/lsst/eo_utils/sflat/sflat_ratio.py +++ b/python/lsst/eo_utils/sflat/sflat_ratio.py @@ -8,8 +8,8 @@ from lsst.eo_utils.base.butler_utils import make_file_dict -from lsst.eo_utils.base.image_utils import get_ccd_from_id,\ - get_dims_from_ccd, get_raw_image, get_geom_regions, get_amp_list +from lsst.eo_utils.base.image_utils import get_dims_from_ccd,\ + get_raw_image, get_geom_regions, get_amp_list from lsst.eo_utils.base.factory import EO_TASK_FACTORY @@ -83,15 +83,15 @@ def extract(self, butler, data, **kwargs): superbias_frame = self.get_superbias_frame(mask_files) superflat_file = data[0] - l_frame = get_ccd_from_id(None, - superflat_file.replace('_l.fits', '_l.fits'), - mask_files) - h_frame = get_ccd_from_id(None, - superflat_file.replace('_l.fits', '_h.fits'), - mask_files) - ratio_frame = get_ccd_from_id(None, - superflat_file.replace('_l.fits', '_ratio.fits'), - mask_files) + l_frame = self.get_ccd(None, + superflat_file.replace('_l.fits', '_l.fits'), + mask_files) + h_frame = self.get_ccd(None, + superflat_file.replace('_l.fits', '_h.fits'), + mask_files) + ratio_frame = self.get_ccd(None, + superflat_file.replace('_l.fits', '_ratio.fits'), + mask_files) # This is a dictionary of dictionaries to store all the # data you extract from the sflat_files diff --git a/python/lsst/eo_utils/sflat/superflat.py b/python/lsst/eo_utils/sflat/superflat.py index 0cfa61b..d6e6210 100644 --- a/python/lsst/eo_utils/sflat/superflat.py +++ b/python/lsst/eo_utils/sflat/superflat.py @@ -20,8 +20,8 @@ from lsst.eo_utils.base.data_utils import TableDict -from lsst.eo_utils.base.image_utils import get_ccd_from_id,\ - flip_data_in_place, sort_sflats, stack_images, extract_raft_array_dict,\ +from lsst.eo_utils.base.image_utils import flip_data_in_place,\ + sort_sflats, stack_images, extract_raft_array_dict,\ outlier_raft_dict, fill_footprint_dict, extract_raft_imaging_data,\ extract_raft_unbiased_images @@ -177,9 +177,9 @@ def make_superflats(self, butler, data, **kwargs): flip_data_in_place(output_file + '_h.fits') flip_data_in_place(output_file + '_ratio.fits') - self._superflat_frame_l = get_ccd_from_id(None, output_file + '_l.fits', mask_files) - self._superflat_frame_h = get_ccd_from_id(None, output_file + '_h.fits', mask_files) - self._superflat_frame_r = get_ccd_from_id(None, output_file + '_ratio.fits', mask_files) + self._superflat_frame_l = self.get_ccd(None, output_file + '_l.fits', mask_files) + self._superflat_frame_h = self.get_ccd(None, output_file + '_h.fits', mask_files) + self._superflat_frame_r = self.get_ccd(None, output_file + '_ratio.fits', mask_files) def plot(self, dtables, figs, **kwargs): diff --git a/tests/test_00_base.py b/tests/test_00_base.py index b63fc0c..3ce2033 100644 --- a/tests/test_00_base.py +++ b/tests/test_00_base.py @@ -56,11 +56,13 @@ def test_file_utils_get_bot(): bias_files = get_files_for_run('6545D', imagetype='BIAS', testtypes=['DARK'], - outkey='BIAS') + outkey='BIAS', + teststand='bot_etu') dark_files = get_files_for_run('6545D', imagetype='DARK', testtypes=['DARK'], - outkey='DARK') + outkey='DARK', + teststand='bot_etu') files = merge_file_dicts(bias_files, dark_files) assert len(files) == 2 assert 'R10' in files @@ -74,11 +76,11 @@ def test_file_utils_get_bot(): def test_file_utils_get_names_dc(): """Test the file_utils module""" - rafts_6106 = get_raft_names_dc('6106D') + rafts_6106 = get_raft_names_dc('6106D', 'bot_etu') assert len(rafts_6106) == 1 assert rafts_6106[0] == 'RTM-004' - rafts_6545 = get_raft_names_dc('6545D') + rafts_6545 = get_raft_names_dc('6545D', 'bot_etu') assert len(rafts_6545) == 2 assert rafts_6545[0] == 'R10' assert rafts_6545[1] == 'R22' diff --git a/tests/test_02_bias.py b/tests/test_02_bias.py index acc859a..82958fb 100644 --- a/tests/test_02_bias.py +++ b/tests/test_02_bias.py @@ -20,7 +20,7 @@ def test_bias_file_utils(): bias_files_6545 = bias.BiasAnalysisTask.get_data(None, '6545D', **DATA_OPTIONS_BOT_GLOB) - assert_data_dict(bias_files_6545, 'R10', 'BIAS', (2, 9, 1, 159)) + assert_data_dict(bias_files_6545, 'R10', 'BIAS', (2, 9, 1, 169)) def test_bias_butler_utils(): @@ -34,7 +34,7 @@ def test_bias_butler_utils(): bias_files_6545 = bias.BiasAnalysisTask.get_data(bot_butler, '6545D', **DATA_OPTIONS_BOT_BUTLER) - assert_data_dict(bias_files_6545, 'R10', 'BIAS', (2, 9, 1, 159)) + assert_data_dict(bias_files_6545, 'R10', 'BIAS', (2, 9, 1, 169)) def test_superbias(): diff --git a/tests/test_06_flat.py b/tests/test_06_flat.py index fff9b87..8b610de 100644 --- a/tests/test_06_flat.py +++ b/tests/test_06_flat.py @@ -44,12 +44,14 @@ def test_flat_oscan(): if RUN_TASKS: task.run(slots=['S00'], **RUN_OPTIONS) +@requires_site('slac') def test_flat_pair(): """Test the FlatPairTask""" task = flat.FlatPairTask() if RUN_TASKS: task.run(slots=['S00'], **RUN_OPTIONS) +@requires_site('slac') def test_flat_linearity(): """Test the FlatLinearityTask""" task = flat.FlatLinearityTask() @@ -62,12 +64,14 @@ def test_flat_bf(): if RUN_TASKS: task.run(slots=['S00'], **RUN_OPTIONS) +@requires_site('slac') def test_flat_ptc(): """Test the PTCTask""" task = flat.PTCTask() if RUN_TASKS: task.run(slots=['S00'], **RUN_OPTIONS_NOPLOT) +@requires_site('slac') def test_flat_ptc_sum(): """Test the PTCSummaryTask""" task = flat.PTCSummaryTask() diff --git a/tests/test_08_qe.py b/tests/test_08_qe.py index 5260319..175a246 100644 --- a/tests/test_08_qe.py +++ b/tests/test_08_qe.py @@ -35,11 +35,7 @@ def test_qe_butler_utils(): qe_files_6545 = qe.QeAnalysisTask.get_data(bot_butler, '6545D', **DATA_OPTIONS_BOT_BUTLER) - #FIXME - if SITE in ['slac']: - assert_data_dict(qe_files_6545, 'R10', 'LAMBDA', (2, 9, 1, 12)) - else: - assert_data_dict(qe_files_6545, 'R10', 'LAMBDA', (2, 9, 1, 6)) + assert_data_dict(qe_files_6545, 'R10', 'LAMBDA', (2, 9, 1, 12)) def test_qe_median(): diff --git a/tests/utils.py b/tests/utils.py index 6a23767..9a15b19 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -19,9 +19,9 @@ if os.environ.get('EO_TEST_NO_RUN_TASKS', False): RUN_TASKS = False -RUN_OPTIONS = dict(runs=['6106D'], outdir='test_out', plot='png') -RUN_OPTIONS_NOPLOT = dict(runs=['6106D'], outdir='test_out') -SUMMARY_OPTIONS = dict(dataset='tests/test', outdir='test_out', plot='png') +RUN_OPTIONS = dict(runs=['6106D'], outdir='test_out', plot='png', teststand='ts8') +RUN_OPTIONS_NOPLOT = dict(runs=['6106D'], outdir='test_out', teststand='ts8') +SUMMARY_OPTIONS = dict(dataset='tests/test', outdir='test_out', plot='png', teststand='ts8') def requires_file(filepath): """Skip test if required file is missing"""