From 4bcf33a14ac70fca08d9c550b5bde4d0e497d9f8 Mon Sep 17 00:00:00 2001 From: James Chiang Date: Tue, 1 Oct 2019 14:14:36 -0700 Subject: [PATCH 1/5] write selected LSST Stack package versions and tags to FITS primary header for the set up products --- data/default_imsim_configs | 5 ++++ python/desc/imsim/camera_readout.py | 15 ++++++++++-- python/desc/imsim/imSim.py | 27 ++++++++++++++++++-- tests/test_get_stack_products.py | 38 +++++++++++++++++++++++++++++ 4 files changed, 81 insertions(+), 4 deletions(-) create mode 100644 tests/test_get_stack_products.py diff --git a/data/default_imsim_configs b/data/default_imsim_configs index cd51bc9d..b676ae50 100644 --- a/data/default_imsim_configs +++ b/data/default_imsim_configs @@ -55,3 +55,8 @@ sort_magnorm = True # AtmosphericPSF + OptWF PSF to account for additional instrumental # effects. gaussianFWHM = 0.4 + +[stack_packages] +lsst_sims +throughputs +sims_skybrightness_data diff --git a/python/desc/imsim/camera_readout.py b/python/desc/imsim/camera_readout.py index 90d32161..c91adefd 100644 --- a/python/desc/imsim/camera_readout.py +++ b/python/desc/imsim/camera_readout.py @@ -32,7 +32,7 @@ getRotSkyPos, ObservationMetaData, altAzPaFromRaDec from lsst.sims.GalSimInterface import LsstObservatory from .camera_info import CameraInfo, getHourAngle -from .imSim import get_logger, get_config, airmass +from .imSim import get_logger, get_config, airmass, get_stack_products from .cosmic_rays import CosmicRays from .version import __version__ as imsim_version @@ -476,7 +476,6 @@ def write_fits_file(self, outfile, overwrite=True, run_number=None, output[0].header.insert(5, ('WCSAXES', wcsaxes, '')) if run_number is None: run_number = self.visit - output[0].header['IMSIMVER'] = imsim_version output[0].header['RUNNUM'] = str(run_number) output[0].header['DARKTIME'] = output[0].header['EXPTIME'] output[0].header['TIMESYS'] = 'TAI' @@ -531,6 +530,18 @@ def write_fits_file(self, outfile, overwrite=True, run_number=None, amp_name = '_C'.join((self.sensor_id, seg_id)) output.append(self.get_amplifier_hdu(amp_name, compress=compress)) output[-1].header['EXTNAME'] = 'Segment%s' % seg_id + + # Set the imSim version and LSST Stack product versions and + # tags in the primary HDU. + output[0].header['IMSIMVER'] = imsim_version + products = get_stack_products() + for iprod, (product_name, product) in enumerate(products.items()): + output[0].header[f'PKG{iprod:05d}'] = product_name + output[0].header[f'VER{iprod:05d}'] = product.version + # Use the "first" semantically meaningful tag. + tag = sorted([_ for _ in product.tags if _ != 'current'])[0] + output[0].header[f'TAG{iprod:05d}'] = tag + self.fits_atomic_write(output, outfile, overwrite=overwrite) @staticmethod diff --git a/python/desc/imsim/imSim.py b/python/desc/imsim/imSim.py index df6510a0..441d1681 100644 --- a/python/desc/imsim/imSim.py +++ b/python/desc/imsim/imSim.py @@ -13,6 +13,7 @@ import copy import psutil import galsim +import eups # python_future no longer handles configparser as of 0.16. # This is needed for PY2/3 compatibility. @@ -61,7 +62,8 @@ '_POINT_SOURCE', '_SERSIC_2D', '_RANDOM_WALK', '_FITS_IMAGE', 'parsePhoSimInstanceFile', 'add_treering_info', 'airmass', 'FWHMeff', 'FWHMgeom', 'make_psf', - 'save_psf', 'load_psf', 'TracebackDecorator', 'GsObjectList'] + 'save_psf', 'load_psf', 'TracebackDecorator', 'GsObjectList', + 'get_stack_products'] class PhosimInstanceCatalogParseError(RuntimeError): @@ -611,7 +613,7 @@ def read_config(config_file=None): config_file. """ my_config = ImSimConfiguration() - cp = configparser.ConfigParser() + cp = configparser.ConfigParser(allow_no_value=True) cp.optionxform = str if config_file is None: config_file = os.path.join(lsstUtils.getPackageDir('imsim'), @@ -904,3 +906,24 @@ def __call__(self, *args, **kwds): except Exception as eobj: traceback.print_exc() raise eobj + +def get_stack_products(product_names=None): + """ + Get the LSST Stack products corresponding to a list of product + names. + + Parameters + ---------- + product_names: list-like [None] + A list of LSST Stack package names for which to get the + corresponding set up eups.Product. If None, then return the + products listed in the config file. + + Returns + ------- + dict of eups.Products keyed by package name. + """ + config = get_config() + stack_packages = set(config['stack_packages'].keys()) + eupsenv = eups.Eups() + return {_: eupsenv.getSetupProducts(_)[0] for _ in stack_packages} diff --git a/tests/test_get_stack_products.py b/tests/test_get_stack_products.py new file mode 100644 index 00000000..c590a41e --- /dev/null +++ b/tests/test_get_stack_products.py @@ -0,0 +1,38 @@ +""" +Unit test for get_stack_products function. +""" +import unittest +import subprocess +from desc.imsim import get_stack_products + + +class GetStackProductsTestCase(unittest.TestCase): + """ + TestCase subclass for testing get_stacks_products. + """ + def setUp(self): + pass + + def tearDown(self): + pass + + def test_get_stack_products(self): + """Test the get_stack_products function.""" + targets = 'lsst_sims throughputs sims_skybrightness_data'.split() + products = get_stack_products(targets) + + for target in targets: + # Test result against eups command line result. + command = f'eups list {target} -s' + line = subprocess.check_output(command, shell=True) + tokens = line.decode('utf-8').strip().split() + version = tokens[0] + tags = set(tokens[1:]) + tags.remove('setup') + + self.assertEqual(products[target].version, version) + self.assertEqual(set(products[target].tags), tags) + + +if __name__ == '__main__': + unittest.main() From d502aa1059d221ca2398e10984d5e18d195b164c Mon Sep 17 00:00:00 2001 From: James Chiang Date: Sun, 20 Oct 2019 21:56:30 -0700 Subject: [PATCH 2/5] write eups tags for metapackages, git tags (i.e., versions) for non-metapackages --- data/default_imsim_configs | 2 +- python/desc/imsim/camera_readout.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/data/default_imsim_configs b/data/default_imsim_configs index b676ae50..71fed5ad 100644 --- a/data/default_imsim_configs +++ b/data/default_imsim_configs @@ -57,6 +57,6 @@ sort_magnorm = True gaussianFWHM = 0.4 [stack_packages] -lsst_sims +lsst_sims = metapackage throughputs sims_skybrightness_data diff --git a/python/desc/imsim/camera_readout.py b/python/desc/imsim/camera_readout.py index c91adefd..280aa41d 100644 --- a/python/desc/imsim/camera_readout.py +++ b/python/desc/imsim/camera_readout.py @@ -537,11 +537,11 @@ def write_fits_file(self, outfile, overwrite=True, run_number=None, products = get_stack_products() for iprod, (product_name, product) in enumerate(products.items()): output[0].header[f'PKG{iprod:05d}'] = product_name - output[0].header[f'VER{iprod:05d}'] = product.version - # Use the "first" semantically meaningful tag. - tag = sorted([_ for _ in product.tags if _ != 'current'])[0] - output[0].header[f'TAG{iprod:05d}'] = tag - + if product.type == 'metapackage': + tag = [_ for _ in product.tags if _ != 'current'][0] + output[0].header[f'TAG{iprod:05d}'] = tag + else: + output[0].header[f'VER{iprod:05d}'] = product.version self.fits_atomic_write(output, outfile, overwrite=overwrite) @staticmethod From 835ed1c5183e173d3c308504283ce178781b5afa Mon Sep 17 00:00:00 2001 From: James Chiang Date: Sun, 20 Oct 2019 21:57:36 -0700 Subject: [PATCH 3/5] add product type (metapackage or not) to product object --- python/desc/imsim/imSim.py | 11 +++++++++-- tests/test_get_stack_products.py | 31 +++++++++++++++++-------------- 2 files changed, 26 insertions(+), 16 deletions(-) diff --git a/python/desc/imsim/imSim.py b/python/desc/imsim/imSim.py index 441d1681..ef082fa5 100644 --- a/python/desc/imsim/imSim.py +++ b/python/desc/imsim/imSim.py @@ -924,6 +924,13 @@ def get_stack_products(product_names=None): dict of eups.Products keyed by package name. """ config = get_config() - stack_packages = set(config['stack_packages'].keys()) + if product_names is not None: + stack_packages = {_: None for _ in product_names} + else: + stack_packages = config['stack_packages'] eupsenv = eups.Eups() - return {_: eupsenv.getSetupProducts(_)[0] for _ in stack_packages} + products = dict() + for product_name, product_type in stack_packages.items(): + products[product_name] = eupsenv.getSetupProducts(product_name)[0] + products[product_name].type = product_type + return products diff --git a/tests/test_get_stack_products.py b/tests/test_get_stack_products.py index c590a41e..91810000 100644 --- a/tests/test_get_stack_products.py +++ b/tests/test_get_stack_products.py @@ -18,20 +18,23 @@ def tearDown(self): def test_get_stack_products(self): """Test the get_stack_products function.""" - targets = 'lsst_sims throughputs sims_skybrightness_data'.split() - products = get_stack_products(targets) - - for target in targets: - # Test result against eups command line result. - command = f'eups list {target} -s' - line = subprocess.check_output(command, shell=True) - tokens = line.decode('utf-8').strip().split() - version = tokens[0] - tags = set(tokens[1:]) - tags.remove('setup') - - self.assertEqual(products[target].version, version) - self.assertEqual(set(products[target].tags), tags) + target_lists = [None, 'afw sims_photUtils sims_utils'.split()] + + for targets in target_lists: + products = get_stack_products(targets) + if targets is not None: + self.assertEqual(set(products.keys()), set(targets)) + for target in products: + # Test result against eups command line result. + command = f'eups list {target} -s' + line = subprocess.check_output(command, shell=True) + tokens = line.decode('utf-8').strip().split() + version = tokens[0] + tags = set(tokens[1:]) + tags.remove('setup') + + self.assertEqual(products[target].version, version) + self.assertEqual(set(products[target].tags), tags) if __name__ == '__main__': From ec3344d5f54b85246829031ad1a0e04f9206997c Mon Sep 17 00:00:00 2001 From: James Chiang Date: Mon, 21 Oct 2019 21:16:22 +0000 Subject: [PATCH 4/5] add version keywords to eimage files and refactor version keyword handling --- python/desc/imsim/ImageSimulator.py | 9 ++++++- python/desc/imsim/camera_readout.py | 15 +++-------- python/desc/imsim/imSim.py | 39 +++++++++++++++++++++-------- tests/test_get_stack_products.py | 38 ++++++++++++++++++++++++---- 4 files changed, 73 insertions(+), 28 deletions(-) diff --git a/python/desc/imsim/ImageSimulator.py b/python/desc/imsim/ImageSimulator.py index 90a833c5..ea819266 100644 --- a/python/desc/imsim/ImageSimulator.py +++ b/python/desc/imsim/ImageSimulator.py @@ -12,13 +12,14 @@ import sqlite3 import numpy as np from astropy._erfa import ErfaWarning +import galsim from lsst.afw.cameraGeom import WAVEFRONT, GUIDER from lsst.sims.photUtils import BandpassDict from lsst.sims.GalSimInterface import make_galsim_detector from lsst.sims.GalSimInterface import make_gs_interpreter from lsst.sims.GalSimInterface import LSSTCameraWrapper from .imSim import read_config, parsePhoSimInstanceFile, add_cosmic_rays,\ - add_treering_info, get_logger, TracebackDecorator + add_treering_info, get_logger, TracebackDecorator, get_version_keywords from .bleed_trails import apply_channel_bleeding from .skyModel import make_sky_model from .process_monitor import process_monitor @@ -502,6 +503,12 @@ def write_eimage_files(self, gs_interpreter): ---------- gs_interpreter: GalSimInterpreter object """ + # Add version keywords to eimage headers + version_keywords = get_version_keywords() + for image in gs_interpreter.detectorImages.values(): + image.header = galsim.FitsHeader(header=version_keywords) + + # Write the eimage files using filenames containing the visit number. prefix = IMAGE_SIMULATOR.config['persistence']['eimage_prefix'] obsHistID = str(IMAGE_SIMULATOR.obs_md.OpsimMetaData['obshistID']) nameRoot = os.path.join(IMAGE_SIMULATOR.outdir, prefix) + obsHistID diff --git a/python/desc/imsim/camera_readout.py b/python/desc/imsim/camera_readout.py index 280aa41d..616a6676 100644 --- a/python/desc/imsim/camera_readout.py +++ b/python/desc/imsim/camera_readout.py @@ -32,9 +32,8 @@ getRotSkyPos, ObservationMetaData, altAzPaFromRaDec from lsst.sims.GalSimInterface import LsstObservatory from .camera_info import CameraInfo, getHourAngle -from .imSim import get_logger, get_config, airmass, get_stack_products +from .imSim import get_logger, get_config, airmass, get_version_keywords from .cosmic_rays import CosmicRays -from .version import __version__ as imsim_version __all__ = ['ImageSource', 'set_itl_bboxes', 'set_e2v_bboxes', 'set_phosim_bboxes', 'set_noao_keywords', 'cte_matrix'] @@ -533,15 +532,9 @@ def write_fits_file(self, outfile, overwrite=True, run_number=None, # Set the imSim version and LSST Stack product versions and # tags in the primary HDU. - output[0].header['IMSIMVER'] = imsim_version - products = get_stack_products() - for iprod, (product_name, product) in enumerate(products.items()): - output[0].header[f'PKG{iprod:05d}'] = product_name - if product.type == 'metapackage': - tag = [_ for _ in product.tags if _ != 'current'][0] - output[0].header[f'TAG{iprod:05d}'] = tag - else: - output[0].header[f'VER{iprod:05d}'] = product.version + version_keywords = get_version_keywords() + for key, value in version_keywords.items(): + output[0].header[key] = value self.fits_atomic_write(output, outfile, overwrite=overwrite) @staticmethod diff --git a/python/desc/imsim/imSim.py b/python/desc/imsim/imSim.py index ef082fa5..bb47ccbb 100644 --- a/python/desc/imsim/imSim.py +++ b/python/desc/imsim/imSim.py @@ -47,6 +47,7 @@ from .trim import InstCatTrimmer from .sed_wrapper import SedWrapper from .atmPSF import AtmosphericPSF +from .version import __version__ as imsim_version _POINT_SOURCE = 1 _SERSIC_2D = 2 @@ -63,7 +64,7 @@ 'parsePhoSimInstanceFile', 'add_treering_info', 'airmass', 'FWHMeff', 'FWHMgeom', 'make_psf', 'save_psf', 'load_psf', 'TracebackDecorator', 'GsObjectList', - 'get_stack_products'] + 'get_stack_products', 'get_version_keywords'] class PhosimInstanceCatalogParseError(RuntimeError): @@ -908,29 +909,45 @@ def __call__(self, *args, **kwds): raise eobj def get_stack_products(product_names=None): - """ - Get the LSST Stack products corresponding to a list of product + """Get the LSST Stack products corresponding to a set of product names. Parameters ---------- - product_names: list-like [None] - A list of LSST Stack package names for which to get the - corresponding set up eups.Product. If None, then return the - products listed in the config file. + product_names: dict [None] + A dict with LSST Stack package names as keys and either + 'metapackage' or None as values. The setup eups.Product + corresponding to each of these packages will be returned in a + dictionary. If None, then use the products listed in the + config file. Returns ------- dict of eups.Products keyed by package name. + """ config = get_config() - if product_names is not None: - stack_packages = {_: None for _ in product_names} - else: - stack_packages = config['stack_packages'] + stack_packages = config['stack_packages'] if product_names is None \ + else product_names eupsenv = eups.Eups() products = dict() for product_name, product_type in stack_packages.items(): products[product_name] = eupsenv.getSetupProducts(product_name)[0] products[product_name].type = product_type return products + +def get_version_keywords(): + """ + Return a dictionary of header keywords containing eups tagging and + version information. + """ + keywords = {'IMSIMVER': imsim_version} + products = get_stack_products() + for iprod, (product_name, product) in enumerate(products.items()): + keywords[f'PKG{iprod:05d}'] = product_name + if product.type == 'metapackage': + tag = [_ for _ in product.tags if _ != 'current'][0] + keywords[f'TAG{iprod:05d}'] = tag + else: + keywords[f'VER{iprod:05d}'] = product.version + return keywords diff --git a/tests/test_get_stack_products.py b/tests/test_get_stack_products.py index 91810000..e549ccc3 100644 --- a/tests/test_get_stack_products.py +++ b/tests/test_get_stack_products.py @@ -3,7 +3,8 @@ """ import unittest import subprocess -from desc.imsim import get_stack_products +from collections import defaultdict +import desc.imsim class GetStackProductsTestCase(unittest.TestCase): @@ -18,12 +19,13 @@ def tearDown(self): def test_get_stack_products(self): """Test the get_stack_products function.""" - target_lists = [None, 'afw sims_photUtils sims_utils'.split()] + target_dicts = [None, {_: None for _ in + 'afw sims_photUtils sims_utils'.split()}] - for targets in target_lists: - products = get_stack_products(targets) + for targets in target_dicts: + products = desc.imsim.get_stack_products(targets) if targets is not None: - self.assertEqual(set(products.keys()), set(targets)) + self.assertEqual(products.keys(), targets.keys()) for target in products: # Test result against eups command line result. command = f'eups list {target} -s' @@ -36,6 +38,32 @@ def test_get_stack_products(self): self.assertEqual(products[target].version, version) self.assertEqual(set(products[target].tags), tags) + def test_get_version_keywords(self): + """Test the get_version_keywords function.""" + class KeywordInfo: + """Class to hold FITS keyword name and type values for + versioning keywords.""" + pass + version_keywords = desc.imsim.get_version_keywords() + keyword_info = defaultdict(KeywordInfo) + for key, value in version_keywords.items(): + if key.startswith('PKG'): + iprod = int(key[len('PKG'):]) + keyword_info[iprod].name = value + if key.startswith('TAG'): + iprod = int(key[len('TAG'):]) + keyword_info[iprod].type = 'metapackage' + if key.startswith('VER'): + iprod = int(key[len('VER'):]) + keyword_info[iprod].type = '' + # Repackage the keyword_info dict to use package names instead + # of the iprod index so that we can compare directly to the + # 'stack_packages' config. + keyword_info = {_.name: _.type for _ in keyword_info.values()} + + config = desc.imsim.get_config() + self.assertEqual(keyword_info, config['stack_packages']) + if __name__ == '__main__': unittest.main() From 31d7c2f2724431d7baf13db663b636d4daa799a6 Mon Sep 17 00:00:00 2001 From: James Chiang Date: Mon, 21 Oct 2019 15:02:20 -0700 Subject: [PATCH 5/5] use .update instead of a for loop over .items() --- python/desc/imsim/camera_readout.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/python/desc/imsim/camera_readout.py b/python/desc/imsim/camera_readout.py index 616a6676..2bd828ef 100644 --- a/python/desc/imsim/camera_readout.py +++ b/python/desc/imsim/camera_readout.py @@ -532,9 +532,7 @@ def write_fits_file(self, outfile, overwrite=True, run_number=None, # Set the imSim version and LSST Stack product versions and # tags in the primary HDU. - version_keywords = get_version_keywords() - for key, value in version_keywords.items(): - output[0].header[key] = value + output[0].header.update(get_version_keywords()) self.fits_atomic_write(output, outfile, overwrite=overwrite) @staticmethod