From ebf36e57be609f799f57eec5dfd74d6b0149f20b Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Wed, 24 Jan 2024 10:11:12 -0800 Subject: [PATCH 01/63] adding some utility functions (from Sergey Koposov) for working with streams --- py/desitarget/streamutilities.py | 291 +++++++++++++++++++++++++++++++ 1 file changed, 291 insertions(+) create mode 100644 py/desitarget/streamutilities.py diff --git a/py/desitarget/streamutilities.py b/py/desitarget/streamutilities.py new file mode 100644 index 00000000..98ba2bd3 --- /dev/null +++ b/py/desitarget/streamutilities.py @@ -0,0 +1,291 @@ +""" +desitarget.streamutilties +========================= + +Utilities for the DESI MWS Stellar Stream programs. + +Borrows heavily from Sergey Koposov's `astrolibpy routines`_. + +.. _`astrolibpy routines`: https://github.com/segasai/astrolibpy/blob/master/my_utils +""" +import numpy as np +import astropy.coordinates as acoo +import astropy.units as auni + +# ADM Galactic reference frame. Use astropy v4.0 defaults. +GCPARAMS = acoo.galactocentric_frame_defaults.get_from_registry( + "v4.0")['parameters'] + +# ADM some standard units. +kms = auni.km / auni.s +masyr = auni.mas / auni.year + + +def cosd(x): + """Return cos(x) for an angle x in degrees. + """ + return np.cos(np.deg2rad(x)) + + +def sind(x): + """Return sin(x) for an angle x in degrees. + """ + return np.sin(np.deg2rad(x)) + + +def betw(x, x1, x2): + """Whether x lies in the range x1 <= x < x2. + + Parameters + ---------- + x : :class:`~numpy.ndarray` or `int` or `float` + Value(s) that need checked against x1, x2. + + x1 : :class:`~numpy.ndarray` or `int` or `float` + Lower range to check against (inclusive). + + x2 : :class:`~numpy.ndarray` or `int` or `float` + Upper range to check against (exclusive). + + Returns + ------- + :class:`array_like` or `boolean` + ``True`` for values of `x` that lie in the range x1 <= x < x2. + If any input is an array then the output will be a Boolean array. + + Notes + ----- + - Very permissive. Arrays can be checked against other arrays, + scalars against scalars and arrays against arrays. For example, if + all the inputs are arrays the calculation will be element-wise. If + `x1` and `x2` are floats and `x` is array-like then each element of + `x` will be checked against the range. If `x` and `x2` are floats + and `x1` is an array, all possible x1->x2 ranges will be checked. + """ + return (x >= x1) & (x < x2) + + +def torect(ra, dec): + """Convert equatorial coordinates to Cartesian coordinates. + + Parameters + ---------- + ra : :class:`~numpy.ndarray` or `float` + Right Ascension in DEGREES. + + dec : :class:`~numpy.ndarray` or `float` + Declination in DEGREES. + + Returns + ------- + :class:`tuple` + A tuple of the x, y, z converted values. If `ra`, `dec` are + passed as arrays this will be a tuple of x, y, z, arrays. + """ + x = cosd(ra) * cosd(dec) + y = sind(ra) * cosd(dec) + z = sind(dec) + + return x, y, z + + +def fromrect(x, y, z): + """Convert equatorial coordinates to Cartesian coordinates. + + Parameters + ---------- + x, y, z : :class:`~numpy.ndarray` or `float` + Cartesian coordinates. + + Returns + ------- + :class:`tuple` + A tuple of the RA, Dec converted values in DEGREES. If `x`, `y`, + `z` are passed as arrays this will be a tuple of RA, Dec arrays. + """ + ra = np.arctan2(y, x) * 57.295779513082323 + dec = 57.295779513082323 * np.arctan2(z, np.sqrt(x**2 + y**2)) + + return ra, dec + + +def rotation_matrix(rapol, decpol, ra0): + """Return the rotation matrix corresponding to the pole of rapol, + decpol and with the zero of the new latitude corresponding to ra=ra0. + The resulting matrix needs to be np.dot'ed with a vector to forward + transform that vector. + + Parameters + ---------- + rapol, decpol : :class:`float` + Pole of the new coordinate system in DEGREES. + + ra0 : :class:`float` + Zero latitude of the new coordinate system in DEGREES. + + Returns + ------- + :class:`~numpy.ndarray` + 3x3 Rotation matrix. + """ + tmppol = np.array(torect(rapol, decpol)) # pole axis + tmpvec1 = np.array(torect(ra0, 0)) # x axis + tmpvec1 = np.array(tmpvec1) + + tmpvec1[2] = (-tmppol[0] * tmpvec1[0] - tmppol[1] * tmpvec1[1]) / tmppol[2] + tmpvec1 /= np.sqrt((tmpvec1**2).sum()) + tmpvec2 = np.cross(tmppol, tmpvec1) # y axis + M = np.array([tmpvec1, tmpvec2, tmppol]) + + return M + + +def sphere_rotate(ra, dec, rapol, decpol, ra0, revert=False): + """Rotate ra, dec to a new spherical coordinate system. + + Parameters + ---------- + ra : :class:`~numpy.ndarray` or `float` + Right Ascension in DEGREES. + + dec : :class:`~numpy.ndarray` or `float` + Declination in DEGREES. + + rapol, decpol : :class:`float` + Pole of the new coordinate system in DEGREES. + + ra0 : :class:`float` + Zero latitude of the new coordinate system in DEGREES. + + revert : :class:`bool`, optional, defaults to ``False`` + Reverse the rotation. + + Returns + ------- + :class:`tuple` + A tuple of the the new RA, Dec values in DEGREES. If `ra`, `dec` + are passed as arrays this will be a tuple of RA, Dec arrays. + """ + x, y, z = torect(ra, dec) + M = rotation_matrix(rapol, decpol, ra0) + + if not revert: + Axx, Axy, Axz = M[0] + Ayx, Ayy, Ayz = M[1] + Azx, Azy, Azz = M[2] + else: + Axx, Ayx, Azx = M[0] + Axy, Ayy, Azy = M[1] + Axz, Ayz, Azz = M[2] + xnew = x * Axx + y * Axy + z * Axz + ynew = x * Ayx + y * Ayy + z * Ayz + znew = x * Azx + y * Azy + z * Azz + del x, y, z + tmp = fromrect(xnew, ynew, znew) + + return (tmp[0], tmp[1]) + + +def rotate_pm(ra, dec, pmra, pmdec, rapol, decpol, ra0, revert=False): + """ + Rotate proper motions to a new spherical coordinate system. + + Parameters + ---------- + ra, dec : :class:`~numpy.ndarray` or `float` + Right Ascension, Declination in DEGREES. + + pmra, pmdec : :class:`~numpy.ndarray` or `float` + Proper motion in Right Ascension, Declination in mas/yr. + + pmdec : :class:`~numpy.ndarray` or `float` + Proper motion in Declination in mas/yr. + + rapol, decpol : :class:`float` + Pole of the new coordinate system in DEGREES. + + ra0 : :class:`float` + Zero latitude of the new coordinate system in DEGREES. + + revert : :class:`bool`, optional, defaults to ``False`` + Reverse the rotation. + + Returns + ------- + :class:`tuple` + A tuple of the the new pmra, pmdec values in DEGREES. If `ra`, + `dec`, etc. are passed as arrays this will be a tuple of arrays. + """ + ra, dec, pmra, pmdec = [np.atleast_1d(_) for _ in [ra, dec, pmra, pmdec]] + M = rotation_matrix(rapol, decpol, ra0) + if revert: + M = M.T + # unit vectors + e_mura = np.array([-sind(ra), cosd(ra), ra * 0]) + e_mudec = np.array( + [-sind(dec) * cosd(ra), -sind(dec) * sind(ra), + cosd(dec)]) + # velocity vector in arbitrary units + V = pmra * e_mura + pmdec * e_mudec + del e_mura, e_mudec + # apply rotation to velocity + V1 = M @ V + del V + X = np.array([cosd(ra) * cosd(dec), sind(ra) * cosd(dec), sind(dec)]) + # apply rotation to position + X1 = M @ X + del X + # rotated coordinates in radians + lon = np.arctan2(X1[1, :], X1[0, :]) + lat = np.arctan2(X1[2, :], np.sqrt(X1[0, :]**2 + X1[1, :]**2)) + del X1 + # unit vectors in rotated coordinates + e_mura = np.array([-np.sin(lon), np.cos(lon), lon * 0]) + e_mudec = np.array( + [-np.sin(lat) * np.cos(lon), -np.sin(lat) * np.sin(lon), + np.cos(lat)]) + del lon, lat + + return np.sum(e_mura * V1, axis=0), np.sum(e_mudec * V1, axis=0) + + +def correct_pm(ra, dec, pmra, pmdec, dist): + """Corrects proper motions for the Sun's motion. + + Parameters + ---------- + ra, dec : :class:`~numpy.ndarray` or `float` + Right Ascension, Declination in DEGREES. + + pmra, pmdec : :class:`~numpy.ndarray` or `float` + Proper motion in Right Ascension, Declination in mas/yr. + `pmra` includes the cosine term. + + dist : :class:`float` + Distance in kpc. + + Returns + ------- + :class:`tuple` + A tuple of the the new (pmra, pmdec) values in DEGREES. If `ra`, + `dec`, etc. are passed as arrays this will be a tuple of arrays. + """ + C = acoo.ICRS(ra=ra * auni.deg, + dec=dec * auni.deg, + radial_velocity=0 * kms, + distance=dist * auni.kpc, + pm_ra_cosdec=pmra * masyr, + pm_dec=pmdec * masyr) + frame = acoo.Galactocentric(**GCPARAMS) + Cg = C.transform_to(frame) + Cg1 = acoo.Galactocentric(x=Cg.x, + y=Cg.y, + z=Cg.z, + v_x=Cg.v_x * 0, + v_y=Cg.v_y * 0, + v_z=Cg.v_z * 0, + **GCPARAMS) + C1 = Cg1.transform_to(acoo.ICRS()) + + return ((C.pm_ra_cosdec - C1.pm_ra_cosdec).to_value(masyr), + (C.pm_dec - C1.pm_dec).to_value(masyr)) From 2027c0a17b7a01a9a3b6f90f04aa3b4f282ee872 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Wed, 24 Jan 2024 14:19:32 -0800 Subject: [PATCH 02/63] continue adding stream utilities, trying to make them as generic as possible --- py/desitarget/streamutilities.py | 84 +++++++++++++++++++++++++++++++- 1 file changed, 83 insertions(+), 1 deletion(-) diff --git a/py/desitarget/streamutilities.py b/py/desitarget/streamutilities.py index 98ba2bd3..1f082604 100644 --- a/py/desitarget/streamutilities.py +++ b/py/desitarget/streamutilities.py @@ -250,7 +250,7 @@ def rotate_pm(ra, dec, pmra, pmdec, rapol, decpol, ra0, revert=False): def correct_pm(ra, dec, pmra, pmdec, dist): - """Corrects proper motions for the Sun's motion. + """Correct proper motions for the Sun's motion. Parameters ---------- @@ -289,3 +289,85 @@ def correct_pm(ra, dec, pmra, pmdec, dist): return ((C.pm_ra_cosdec - C1.pm_ra_cosdec).to_value(masyr), (C.pm_dec - C1.pm_dec).to_value(masyr)) + + +def pm12_sel_func(pm1track, pm2track, pmfi1, pmfi2, pm_err, pad=2, mult=2.5): + """Select stream members using proper motion, padded by some error. + + Parameters + ---------- + pm1track : :class:`~numpy.ndarray` or `float` + Allowed proper motions of stream targets, RA-sense. + + pm2track : :class:`~numpy.ndarray` or `float` + Allowed proper motions of stream targets, Dec-sense. + + pmfi1 : :class:`~numpy.ndarray` or `float` + Proper motion in stream coordinates of possible targets, derived + from RA. + + pmfi2 : :class:`~numpy.ndarray` or `float` + Proper motion in stream coordinates of possible targets, derived + from Dec. + + pm_err : :class:`~numpy.ndarray` or `float` + Proper motion error in stream coordinates of possible targets, + combined across `pmfi1` and `pmfi2` errors. + + pad: : :class:`float` or `int`, defaults to 2 + Extra offset with which to pad `mult`*proper_motion_error. + + mult : :class:`float` or `int`, defaults to 2.5 + Multiple of the proper motion error to use for padding. + + Returns + ------- + :class:`array_like` or `boolean` + ``True`` for stream members. + """ + + return np.sqrt((pmfi2 - pm2track)**2 + + (pmfi1 - pm1track)**2) < pad + mult * pm_err + + +def plx_sel_func(dist, D, mult, plx_sys=0.05): + """Select stream members using parallax, padded by some error. + + Parameters + ---------- + dist : :class:`~numpy.ndarray` or `float` + Distance of possible stream members. + + D : :class:`~numpy.ndarray` + Numpy structured array of Gaia information that contains at least + the columns `RA`, `ASTROMETRIC_PARAMS_SOLVED`, `PHOT_G_MEAN_MAG`, + `NU_EFF_USED_IN_ASTRONOMY`, `PSEUDOCOLOUR`, `ECL_LAT`, `PARALLAX` + `PARALLAX_ERROR`. + + mult : :class:`float` or `int` + Multiple of the parallax error to use for padding. + + plx_sys: : :class:`float` + Extra offset with which to pad `mult`*parallax_error. + + Returns + ------- + :class:`array_like` or `boolean` + ``True`` for stream members. + """ + # extra plx systematic error padding + plx_sys = 0.05 + subset = np.in1d(D['ASTROMETRIC_PARAMS_SOLVED'], [31, 95]) + plx_zpt_tmp = gaia_zpt.zpt.get_zpt(D['PHOT_G_MEAN_MAG'][subset], + D['NU_EFF_USED_IN_ASTROMETRY'][subset], + D['PSEUDOCOLOUR'][subset], + D['ECL_LAT'][subset], + D['ASTROMETRIC_PARAMS+SOLVED'][subset], + _warnings=False) + plx_zpt = np.zeros(len(D['RA'])) + plx_zpt_tmp[~np.isfinite(plx_zpt_tmp)] = 0 + plx_zpt[subset] = plx_zpt_tmp + plx = D['PARALLAX'] - plx_zpt + dplx = 1 / dist - plx + + return np.abs(dplx) < plx_sys + mult * D['PARALLAX_ERROR'] From 8cad20d72b90d7561c9f909fc5364d631e5b0730 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Wed, 24 Jan 2024 15:29:03 -0800 Subject: [PATCH 03/63] move utilities into a "streams" directory --- py/desitarget/{ => streams}/streamutilities.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename py/desitarget/{ => streams}/streamutilities.py (100%) diff --git a/py/desitarget/streamutilities.py b/py/desitarget/streams/streamutilities.py similarity index 100% rename from py/desitarget/streamutilities.py rename to py/desitarget/streams/streamutilities.py From 30c7364b38236c69caef16b886bb02fd1f556654 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Wed, 24 Jan 2024 15:30:26 -0800 Subject: [PATCH 04/63] add a more generic isochrone interpolator --- py/desitarget/streams/streamutilities.py | 42 ++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/py/desitarget/streams/streamutilities.py b/py/desitarget/streams/streamutilities.py index 1f082604..95c05cb8 100644 --- a/py/desitarget/streams/streamutilities.py +++ b/py/desitarget/streams/streamutilities.py @@ -11,6 +11,10 @@ import numpy as np import astropy.coordinates as acoo import astropy.units as auni +import yaml +import os +from pkg_resources import resource_filename +from scipy.interpolate import UnivariateSpline # ADM Galactic reference frame. Use astropy v4.0 defaults. GCPARAMS = acoo.galactocentric_frame_defaults.get_from_registry( @@ -291,6 +295,44 @@ def correct_pm(ra, dec, pmra, pmdec, dist): (C.pm_dec - C1.pm_dec).to_value(masyr)) +def get_CMD_interpolator(stream_name): + """Isochrones via interpolating over points in color-magnitude space. + + Parameters + ---------- + stream_name : :class:`str` + Name of a stream that appears in the ../data/streams.yaml file. + Possibilities include 'GD1'. + + Returns + ------- + A scipy interpolated UnivariateSpline. + + Notes + ----- + - Parameters for each stream are in the ../data/streams.yaml file. + """ + # ADM open and load the parameter yaml file. + fn = resource_filename('desitarget', os.path.join('data', 'streams.yaml')) + with open(fn) as f: + stream = yaml.safe_load(f) + + # ADM retrieve the color and magnitude offsets. + coloff = stream[stream_name]["COLOFF"] + magoff = stream[stream_name]["MAGOFF"] + + # ADM the isochrones to interpolate over. + iso_dartmouth_g = np.array(stream[stream_name]["ISO_G"]) + iso_dartmouth_r = np.array(stream[stream_name]["ISO_R"]) + + # ADM UnivariateSpline is from scipy.interpolate. + CMD_II = UnivariateSpline(iso_dartmouth_r[::-1] + magoff, + (iso_dartmouth_g - iso_dartmouth_r - coloff)[::-1], + s=0) + + return CMD_II + + def pm12_sel_func(pm1track, pm2track, pmfi1, pmfi2, pm_err, pad=2, mult=2.5): """Select stream members using proper motion, padded by some error. From 5e2a76a0f1fc3eba9fe590d028a2be43a7a034bf Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Wed, 24 Jan 2024 15:30:52 -0800 Subject: [PATCH 05/63] add yaml file to track isochrones for different streams --- py/desitarget/data/streams.yaml | 76 +++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 py/desitarget/data/streams.yaml diff --git a/py/desitarget/data/streams.yaml b/py/desitarget/data/streams.yaml new file mode 100644 index 00000000..fc0dc9a8 --- /dev/null +++ b/py/desitarget/data/streams.yaml @@ -0,0 +1,76 @@ +# ADM This yaml file stores isochrones for stellar streams. + +# ADM the gd1 stream. +GD1: + # ADM color offset. + COLOFF: -0.01 + # ADM magnitude offset. + MAGOFF: -0.1 + # ADM g-band isochrone. + ISO_G: [ + 14.1916, 13.8004, 13.4031, 12.9995, 12.6317, 12.3286, 12.0868, 11.8791, + 11.6837, 11.4824, 11.2802, 11.0934, 10.9082, 10.7581, 10.6532, 10.586, + 10.5413, 10.5083, 10.4813, 10.4547, 10.426, 10.3941, 10.3567, 10.3109, + 10.2414, 9.9928, 9.7144, 9.5754, 9.4487, 9.3196, 9.1814, 9.0267, + 8.8643, 8.706, 8.5551, 8.4145, 8.2813, 8.15, 8.0148, 7.8739, 7.7934, + 7.7147, 7.637, 7.5598, 7.4838, 7.4089, 7.3364, 7.2656, 7.1963, 7.1287, + 7.0627, 6.9984, 6.9357, 6.8747, 6.8153, 6.7572, 6.7005, 6.645, 6.5907, + 6.5374, 6.4851, 6.4338, 6.3837, 6.3347, 6.2869, 6.2399, 6.1939, 6.1487, + 6.1045, 6.0611, 5.9994, 5.9382, 5.8771, 5.8163, 5.7558, 5.6955, 5.6354, + 5.5754, 5.5156, 5.4559, 5.3965, 5.3375, 5.2791, 5.2213, 5.1642, 5.1079, + 5.0524, 4.9977, 4.9436, 4.8902, 4.8372, 4.7848, 4.7329, 4.6817, 4.6309, + 4.5808, 4.5312, 4.482, 4.4333, 4.385, 4.3699, 4.3544, 4.339, 4.3232, + 4.3073, 4.2913, 4.2752, 4.259, 4.2426, 4.2262, 4.2096, 4.1929, 4.176, + 4.159, 4.1419, 4.1246, 4.1073, 4.0899, 4.0723, 4.0547, 4.0369, 4.0192, + 4.0013, 3.9833, 3.9652, 3.9471, 3.9289, 3.9106, 3.8923, 3.8739, 3.8555, + 3.837, 3.8185, 3.7999, 3.7814, 3.7629, 3.7444, 3.726, 3.7077, 3.6895, + 3.6715, 3.6537, 3.6362, 3.6189, 3.6018, 3.5851, 3.5688, 3.5529, 3.5373, + 3.5228, 3.5145, 3.5072, 3.5006, 3.4933, 3.4841, 3.4738, 3.4632, 3.4515, + 3.4384, 3.4234, 3.4063, 3.387, 3.3656, 3.3425, 3.3181, 3.2925, 3.2661, + 3.2391, 3.2118, 3.184, 3.1526, 3.1202, 3.0872, 3.0534, 3.0192, 2.9844, + 2.9492, 2.9137, 2.8777, 2.8416, 2.805, 2.7678, 2.7301, 2.6921, 2.6533, + 2.6143, 2.5749, 2.5341, 2.4911, 2.4469, 2.4014, 2.3549, 2.3073, 2.2589, + 2.2096, 2.1599, 2.1095, 2.0585, 2.0069, 1.9546, 1.9018, 1.8486, 1.7948, + 1.7407, 1.6863, 1.6768, 1.6668, 1.6566, 1.6464, 1.6365, 1.6264, 1.6162, + 1.6064, 1.5965, 1.5863, 1.474, 1.3721, 1.2678, 1.1626, 1.063, 0.9624, + 0.8616, 0.7646, 0.6683, 0.5721, 0.4812, 0.3935, 0.3142, 0.2533, 0.2509, + 0.2242, 0.1251, 0.0243, -0.0714, -0.165, -0.2543, -0.3415, -0.4262, + -0.5093, -0.5895, -0.6659, -0.7401, -0.8117, -0.8808, -0.9467, -1.0111, + -1.0726, -1.1319, -1.1889, -1.2441, -1.2969, -1.3477, -1.3951, -1.4404, + -1.4833, -1.5249, -1.5657, -1.6049, -1.6429, -1.6793, -1.7145, -1.7484, + -1.7809, -1.8122, -1.8428, -1.8719, -1.8998, -1.9234 + ] + # ADM r-band isochrone. + ISO_R: [ + 12.5842, 12.2175, 11.8471, 11.4737, 11.1368, 10.8625, 10.6455, 10.4599, + 10.2854, 10.1058, 9.9258, 9.7602, 9.5962, 9.4637, 9.3714, 9.3122, + 9.273, 9.2441, 9.2205, 9.1974, 9.1726, 9.1447, 9.112, 9.0719, 9.0111, + 8.7983, 8.5647, 8.4486, 8.3433, 8.2368, 8.1238, 7.9988, 7.8693, 7.7443, + 7.6265, 7.5175, 7.4151, 7.3147, 7.2116, 7.1038, 7.0416, 6.9804, 6.9199, + 6.86, 6.8008, 6.7426, 6.6853, 6.629, 6.5737, 6.5194, 6.4663, 6.4143, + 6.3635, 6.3138, 6.2652, 6.2177, 6.1712, 6.1257, 6.081, 6.0372, 5.9942, + 5.952, 5.9105, 5.8698, 5.8298, 5.7905, 5.7517, 5.7137, 5.6763, 5.6395, + 5.587, 5.5345, 5.4821, 5.4298, 5.3774, 5.3252, 5.273, 5.2211, 5.1692, + 5.1172, 5.0653, 5.0136, 4.9622, 4.9112, 4.8607, 4.8107, 4.761, 4.7117, + 4.663, 4.6148, 4.5673, 4.5202, 4.4734, 4.4268, 4.3804, 4.3341, 4.2879, + 4.2418, 4.196, 4.1503, 4.1361, 4.1216, 4.1071, 4.0924, 4.0777, 4.0628, + 4.0478, 4.0327, 4.0174, 4.0018, 3.986, 3.97, 3.9538, 3.9374, 3.9208, + 3.904, 3.8871, 3.87, 3.8528, 3.8354, 3.8178, 3.8001, 3.782, 3.7638, + 3.7454, 3.7268, 3.708, 3.689, 3.6697, 3.6502, 3.6306, 3.6108, 3.5909, + 3.5707, 3.5504, 3.5299, 3.5093, 3.4885, 3.4675, 3.4464, 3.425, 3.4035, + 3.3819, 3.3601, 3.3382, 3.3163, 3.2942, 3.2721, 3.2501, 3.2281, 3.2084, + 3.1879, 3.1675, 3.1473, 3.1269, 3.1062, 3.0853, 3.0637, 3.0411, 3.0169, + 2.9913, 2.9644, 2.9363, 2.9075, 2.8779, 2.8478, 2.8174, 2.7866, 2.7555, + 2.7242, 2.6892, 2.6536, 2.6175, 2.5809, 2.5442, 2.5071, 2.4696, 2.432, + 2.3942, 2.3563, 2.318, 2.2792, 2.2401, 2.2004, 2.1602, 2.1197, 2.079, + 2.0367, 1.9924, 1.9466, 1.8995, 1.8514, 1.8021, 1.752, 1.7009, 1.6492, + 1.5967, 1.5434, 1.4894, 1.4348, 1.3796, 1.3238, 1.2675, 1.2106, 1.1533, + 1.1429, 1.1325, 1.1219, 1.1113, 1.1007, 1.0901, 1.0794, 1.0687, 1.058, + 1.0472, 0.9282, 0.8196, 0.7079, 0.5949, 0.487, 0.3786, 0.2698, 0.165, + 0.0607, -0.0441, -0.1435, -0.2399, -0.3271, -0.394, -0.3956, -0.4237, + -0.5341, -0.6472, -0.7552, -0.8614, -0.9633, -1.0632, -1.1607, -1.2567, + -1.3499, -1.4398, -1.5283, -1.6146, -1.6986, -1.7795, -1.8595, -1.9366, + -2.0117, -2.0847, -2.1561, -2.2251, -2.2923, -2.357, -2.42, -2.4803, + -2.5395, -2.5976, -2.6541, -2.7092, -2.7626, -2.8146, -2.8653, -2.9145, + -2.9622, -3.0091, -3.0542, -3.0979, -3.1371 + ] \ No newline at end of file From 9d8f9027e5e0d2a769265ed1273c2b24d6e1a6bf Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Thu, 25 Jan 2024 16:14:12 -0800 Subject: [PATCH 06/63] rename streamutilities.py now its in a streams directory --- py/desitarget/streams/{streamutilities.py => utilities.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename py/desitarget/streams/{streamutilities.py => utilities.py} (100%) diff --git a/py/desitarget/streams/streamutilities.py b/py/desitarget/streams/utilities.py similarity index 100% rename from py/desitarget/streams/streamutilities.py rename to py/desitarget/streams/utilities.py From 07ca6ad72153f368d4f87bbe624c788b3b093450 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Mon, 29 Jan 2024 23:23:26 -0800 Subject: [PATCH 07/63] Structure of code for reading and caching data. Still need to perform match to Gaia DR3 --- py/desitarget/streams/utilities.py | 169 ++++++++++++++++++++++++++++- 1 file changed, 165 insertions(+), 4 deletions(-) diff --git a/py/desitarget/streams/utilities.py b/py/desitarget/streams/utilities.py index 95c05cb8..cd35c615 100644 --- a/py/desitarget/streams/utilities.py +++ b/py/desitarget/streams/utilities.py @@ -8,13 +8,27 @@ .. _`astrolibpy routines`: https://github.com/segasai/astrolibpy/blob/master/my_utils """ +import yaml +import os +import fitsio import numpy as np +import healpy as hp import astropy.coordinates as acoo import astropy.units as auni -import yaml -import os from pkg_resources import resource_filename from scipy.interpolate import UnivariateSpline +from time import time + +from desitarget import io +from desitarget.geomask import pixarea2nside, add_hp_neighbors, sweep_files_touch_hp +from desitarget.targets import resolve + +# ADM set up the DESI default logger. +from desiutil.log import get_logger +log = get_logger() + +# ADM start the clock. +start = time() # ADM Galactic reference frame. Use astropy v4.0 defaults. GCPARAMS = acoo.galactocentric_frame_defaults.get_from_registry( @@ -312,6 +326,9 @@ def get_CMD_interpolator(stream_name): ----- - Parameters for each stream are in the ../data/streams.yaml file. """ + # ADM guard against stream being passed as lower-case. + stream_name = stream_name.upper() + # ADM open and load the parameter yaml file. fn = resource_filename('desitarget', os.path.join('data', 'streams.yaml')) with open(fn) as f: @@ -382,14 +399,14 @@ def plx_sel_func(dist, D, mult, plx_sys=0.05): D : :class:`~numpy.ndarray` Numpy structured array of Gaia information that contains at least - the columns `RA`, `ASTROMETRIC_PARAMS_SOLVED`, `PHOT_G_MEAN_MAG`, + the columns `RA`, `ASTROMETRIC_PARAMS_SOLVED`, `PHOT_G_MEAN_MAG`, `NU_EFF_USED_IN_ASTRONOMY`, `PSEUDOCOLOUR`, `ECL_LAT`, `PARALLAX` `PARALLAX_ERROR`. mult : :class:`float` or `int` Multiple of the parallax error to use for padding. - plx_sys: : :class:`float` + plx_sys : :class:`float` Extra offset with which to pad `mult`*parallax_error. Returns @@ -413,3 +430,147 @@ def plx_sel_func(dist, D, mult, plx_sys=0.05): dplx = 1 / dist - plx return np.abs(dplx) < plx_sys + mult * D['PARALLAX_ERROR'] + + +def read_data(swdir, rapol, decpol, ra_ref, mind, maxd, stream_name, + cache=True, addnors=True): + """Assemble the data needed for a particular stream program. + + Example values for GD1: + swdir = "/global/cfs/cdirs/cosmo/data/legacysurvey/dr9/south/sweep/9.0" + rapol, decpol, ra_ref = 34.5987, 29.7331, 200 + mind, maxd = 80, 100 + + Parameters + ---------- + swdir : :class:`str` + Root directory of Legacy Surveys sweep files for a given data + release for ONE of EITHER north or south, e.g. + "/global/cfs/cdirs/cosmo/data/legacysurvey/dr9/south/sweep/9.0". + + rapol, decpol : :class:`float` + Pole in the stream coordinate system in DEGREES. + + ra_ref : :class:`float` + Zero latitude in the stream coordinate system in DEGREES. + + mind, maxd : :class:`float` or `int` + Minimum and maximum angular distance from the pole of the stream + coordinate system to search for members in DEGREES. + + stream_name : :class:`str` + Name of a stream. Used to make the cached filename, e.g. "GD1". + + cache : :class:`bool` + If ``True`` read from a previously constructed and cached file + automatically, IF such a file exists. + $TARG_DIR/streamcache/streamname-drX-cache.fits is the name of + the cached file, where streamname is the lower-case version of + the passed `stream_name` and drX is the Legacy Surveys Data + Release (parsed from `swdir`). + + addnors : :class:`bool` + If ``True`` then if `swdir` contains "north" add sweep files from + the south by substituting "south" in place of "north" (and vice + versa, i.e. if `swdir` contains "south" add sweep files from the + north by substituting "north" in place of "south"). + + Returns + ------- + :class:`array_like` or `boolean` + ``True`` for stream members. + + Notes + ----- + - The $TARG_DIR environment variable must be set to read/write from + a cache. If $TARG_DIR is not set, no cache will be written. + """ + # ADM check whether $TARG_DIR exists. If it does, agree to read from + # ADM and write to the cache. + writecache = True + targdir = os.environ.get("TARG_DIR") + if targdir is None and cache: + msg = "Set $TARG_DIR environment variable to use the cache!" + log.info(msg) + cache = False + writecache = False + else: + # ADM retrieve the data release from the passed sweep directory. + dr = [i for i in swdir.split(os.sep) if "dr" in i] + # ADM fail if this doesn't look like a standard sweep directory. + if len(dr) != 1: + msg = 'swdir not parsed: should include a construction like ' + msg += '"dr9" or "dr10"' + raise ValueError(msg) + cachefile = os.path.join(os.getenv("TARG_DIR"), "streamcache", + f"{stream_name.lower()}-{dr[0]}-cache.fits") + + # ADM if we have a cache, simply read it and return the data. + if cache: + if os.path.isfile(cachefile): + objs = fitsio.read(cachefile, ext="STREAMCACHE") + msg = f"Read {len(objs)} objects from {cachefile} cache file" + log.info(msg) + return objs + else: + msg = f"{cachefile} file doesn't exist. Proceeding as if cache=False" + log.info(msg) + + # ADM read in the sweep files. + infiles = io.list_sweepfiles(swdir) + # ADM read both the north and south directories, if requested. + if addnors: + if "south" in swdir: + infiles2 = swdir.replace("south", "north") + elif "north" in swdir: + infiles2 = swdir.replace("north", "south") + else: + msg = "addnors passed but swdir does not contain north or south!" + raise ValueError(msg) + infiles += io.list_sweepfiles(infiles2) + + # ADM calculate nside for HEALPixel of approximately 1o to limit + # ADM number of sweeps files that need to be read. + nside = pixarea2nside(1) + # ADM determine RA, Dec of all HEALPixels at this nside. + allpix = np.arange(hp.nside2npix(pixarea2nside(1))) + theta, phi = hp.pix2ang(nside, allpix, nest=True) + ra, dec = np.degrees(phi), 90-np.degrees(theta) + # ADM only retain HEALPixels in the stream, based on mind and maxd. + cpix = acoo.SkyCoord(ra*auni.degree, dec*auni.degree) + cstream = acoo.SkyCoord(rapol*auni.degree, decpol*auni.degree) + sep = cpix.separation(cstream) + ii = betw(sep.value, mind, maxd) + pixlist = allpix[ii] + # ADM determine which sweep files touch the relevant HEALPixels. + filesperpixel, _, _ = sweep_files_touch_hp(nside, pixlist, infiles) + infiles = list(np.unique(np.hstack([filesperpixel[pix] for pix in pixlist]))) + # ADM loop through the sweep files and limit to objects in the stream. + allobjs = [] + for i, filename in enumerate(infiles[:11]): + objs = io.read_tractor(filename) + cobjs = acoo.SkyCoord(objs["RA"]*auni.degree, objs["DEC"]*auni.degree) + sep = cobjs.separation(cstream) + # ADM only retain objects in the stream... + ii = betw(sep.value, mind, maxd) + # ADM ...that aren't very faint (> 22.5 mag). + ii &= objs["FLUX_R"] > 1 + objs = objs[ii] + # ADM limit to northern objects in northern imaging and southern + # ADM objects in southern imaging. + objs = resolve(objs) + allobjs.append(objs) + if i % 10 == 9: + log.info(f"Ran {i+1}/{len(infiles)} files...t={time()-start:.1f}s") + # ADM assemble all of the relevant objects. + allobjs = np.concatenate(allobjs) + log.info(f"Found {len(allobjs)} total objects...t={time()-start:.1f}s") + + # ADM if cache was passed and $TARG_DIR was set write the data. + if writecache: + # ADM if the file doesn't exist we may need to make the directory. + log.info(f"Writing cache...t={time()-start:.1f}s") + os.makedirs(os.path.dirname(cachefile), exist_ok=True) + io.write_with_units(cachefile, allobjs, extname="STREAMCACHE") + + return allobjs From fbf6d292e29ea90ce3acde585d66d7db2ff37acb Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Fri, 2 Feb 2024 07:40:43 -0800 Subject: [PATCH 08/63] debugging; reorganize caching --- py/desitarget/streams/utilities.py | 32 ++++++++++++++++++------------ 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/py/desitarget/streams/utilities.py b/py/desitarget/streams/utilities.py index cd35c615..957e1433 100644 --- a/py/desitarget/streams/utilities.py +++ b/py/desitarget/streams/utilities.py @@ -433,7 +433,7 @@ def plx_sel_func(dist, D, mult, plx_sys=0.05): def read_data(swdir, rapol, decpol, ra_ref, mind, maxd, stream_name, - cache=True, addnors=True): + readcache=True, addnors=True): """Assemble the data needed for a particular stream program. Example values for GD1: @@ -461,13 +461,13 @@ def read_data(swdir, rapol, decpol, ra_ref, mind, maxd, stream_name, stream_name : :class:`str` Name of a stream. Used to make the cached filename, e.g. "GD1". - cache : :class:`bool` + readcache : :class:`bool` If ``True`` read from a previously constructed and cached file - automatically, IF such a file exists. - $TARG_DIR/streamcache/streamname-drX-cache.fits is the name of - the cached file, where streamname is the lower-case version of - the passed `stream_name` and drX is the Legacy Surveys Data - Release (parsed from `swdir`). + automatically, IF such a file exists. If ``False`` OVERWRITE the + cached file, if it exists. The cached file is named + $TARG_DIR/streamcache/streamname-drX-cache.fits, where streamname + is the lower-case version of the passed `stream_name` and drX is + the Legacy Surveys Data Release (parsed from `swdir`). addnors : :class:`bool` If ``True`` then if `swdir` contains "north" add sweep files from @@ -483,16 +483,16 @@ def read_data(swdir, rapol, decpol, ra_ref, mind, maxd, stream_name, Notes ----- - The $TARG_DIR environment variable must be set to read/write from - a cache. If $TARG_DIR is not set, no cache will be written. + a cache. If $TARG_DIR is not set, caching is completely ignored. """ # ADM check whether $TARG_DIR exists. If it does, agree to read from # ADM and write to the cache. writecache = True targdir = os.environ.get("TARG_DIR") - if targdir is None and cache: + if targdir is None: msg = "Set $TARG_DIR environment variable to use the cache!" log.info(msg) - cache = False + readcache = False writecache = False else: # ADM retrieve the data release from the passed sweep directory. @@ -505,15 +505,16 @@ def read_data(swdir, rapol, decpol, ra_ref, mind, maxd, stream_name, cachefile = os.path.join(os.getenv("TARG_DIR"), "streamcache", f"{stream_name.lower()}-{dr[0]}-cache.fits") - # ADM if we have a cache, simply read it and return the data. - if cache: + # ADM if we have a cache, read it if requested and return the data. + if readcache: if os.path.isfile(cachefile): objs = fitsio.read(cachefile, ext="STREAMCACHE") msg = f"Read {len(objs)} objects from {cachefile} cache file" log.info(msg) return objs else: - msg = f"{cachefile} file doesn't exist. Proceeding as if cache=False" + msg = f"{cachefile} cache file doesn't exist. " + msg += f"Proceeding as if readcache=False" log.info(msg) # ADM read in the sweep files. @@ -542,6 +543,11 @@ def read_data(swdir, rapol, decpol, ra_ref, mind, maxd, stream_name, sep = cpix.separation(cstream) ii = betw(sep.value, mind, maxd) pixlist = allpix[ii] + # ADM pad with neighboring pixels to ensure stream is fully covered. + print(len(pixlist)) + newpixlist = add_hp_neighbors(nside, pixlist) + print(len(newpixlist), len(set(newpixlist)-set(pixlist))) + import pdb; pdb.set_trace() # ADM determine which sweep files touch the relevant HEALPixels. filesperpixel, _, _ = sweep_files_touch_hp(nside, pixlist, infiles) infiles = list(np.unique(np.hstack([filesperpixel[pix] for pix in pixlist]))) From 520ac592cab097324f7c50f7802f805d2c39f980 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Mon, 19 Feb 2024 17:06:02 -0800 Subject: [PATCH 09/63] only retain relevant columns to make file sizes more manageable --- py/desitarget/gaiamatch.py | 2 +- py/desitarget/streams/utilities.py | 48 ++++++++++++++++++++++-------- 2 files changed, 37 insertions(+), 13 deletions(-) diff --git a/py/desitarget/gaiamatch.py b/py/desitarget/gaiamatch.py index 2f8f64e0..6c077951 100644 --- a/py/desitarget/gaiamatch.py +++ b/py/desitarget/gaiamatch.py @@ -134,7 +134,7 @@ ('DR3_PMDEC', '>f4'), ('DR3_PMDEC_IVAR', '>f4') ]) -# ADM the data model for reading ALL columns from Gaia EDR3 files. +# ADM the data model for reading ALL columns from Gaia DR3 files. indr3datamodelfull = np.array([], dtype=[ ('SOLUTION_ID', '>i8'), ('DESIGNATION', 'i8'), ('RANDOM_INDEX', '>i8'), ('REF_CAT', 'S2'), ('REF_EPOCH', '>f4'), ('RA', '>f8'), diff --git a/py/desitarget/streams/utilities.py b/py/desitarget/streams/utilities.py index 957e1433..06b2fa03 100644 --- a/py/desitarget/streams/utilities.py +++ b/py/desitarget/streams/utilities.py @@ -38,6 +38,17 @@ kms = auni.km / auni.s masyr = auni.mas / auni.year +# ADM the standard data model for working with streams. +streamcols = np.array([], dtype=[ + ('RELEASE', '>i2'), ('BRICKID', '>i4'), ('TYPE', 'S4'), + ('OBJID', '>i4'), ('RA', '>f8'), ('DEC', '>f8'), ('EBV', '>f4'), + ('FLUX_G', '>f4'), ('FLUX_R', '>f4'), ('FLUX_Z', '>f4'), + ('REF_EPOCH', '>f4'), ('PARALLAX', '>f4'), ('PARALLAX_ERROR', '>f4'), + ('PMRA', '>f4'), ('PMRA_ERROR', '>f4'), + ('PMDEC', '>f4'), ('PMDEC_ERROR', '>f4'), + ('ASTROMETRIC_PARAMS_SOLVED', '>i1'), ('NU_EFF_USED_IN_ASTROMETRY', '>f4'), + ('PSEUDOCOLOUR', '>f4'), ('PHOT_G_MEAN_MAG', '>f4'), ('ECL_LAT', '>f8') +]) def cosd(x): """Return cos(x) for an angle x in degrees. @@ -463,11 +474,11 @@ def read_data(swdir, rapol, decpol, ra_ref, mind, maxd, stream_name, readcache : :class:`bool` If ``True`` read from a previously constructed and cached file - automatically, IF such a file exists. If ``False`` OVERWRITE the - cached file, if it exists. The cached file is named - $TARG_DIR/streamcache/streamname-drX-cache.fits, where streamname - is the lower-case version of the passed `stream_name` and drX is - the Legacy Surveys Data Release (parsed from `swdir`). + automatically, IF such a file exists. If ``False`` don't read + from the cache AND OVERWRITE the cached file, if it exists. The + cached file is $TARG_DIR/streamcache/streamname-drX-cache.fits, + where streamname is the lower-case passed `stream_name` and drX + is the Legacy Surveys Data Release (parsed from `swdir`). addnors : :class:`bool` If ``True`` then if `swdir` contains "north" add sweep files from @@ -533,41 +544,54 @@ def read_data(swdir, rapol, decpol, ra_ref, mind, maxd, stream_name, # ADM calculate nside for HEALPixel of approximately 1o to limit # ADM number of sweeps files that need to be read. nside = pixarea2nside(1) + # ADM determine RA, Dec of all HEALPixels at this nside. - allpix = np.arange(hp.nside2npix(pixarea2nside(1))) + allpix = np.arange(hp.nside2npix(nside)) theta, phi = hp.pix2ang(nside, allpix, nest=True) ra, dec = np.degrees(phi), 90-np.degrees(theta) + # ADM only retain HEALPixels in the stream, based on mind and maxd. cpix = acoo.SkyCoord(ra*auni.degree, dec*auni.degree) cstream = acoo.SkyCoord(rapol*auni.degree, decpol*auni.degree) sep = cpix.separation(cstream) ii = betw(sep.value, mind, maxd) pixlist = allpix[ii] + # ADM pad with neighboring pixels to ensure stream is fully covered. - print(len(pixlist)) newpixlist = add_hp_neighbors(nside, pixlist) - print(len(newpixlist), len(set(newpixlist)-set(pixlist))) - import pdb; pdb.set_trace() + # ADM determine which sweep files touch the relevant HEALPixels. filesperpixel, _, _ = sweep_files_touch_hp(nside, pixlist, infiles) infiles = list(np.unique(np.hstack([filesperpixel[pix] for pix in pixlist]))) # ADM loop through the sweep files and limit to objects in the stream. allobjs = [] - for i, filename in enumerate(infiles[:11]): + for i, filename in enumerate(infiles): objs = io.read_tractor(filename) cobjs = acoo.SkyCoord(objs["RA"]*auni.degree, objs["DEC"]*auni.degree) sep = cobjs.separation(cstream) + # ADM only retain objects in the stream... ii = betw(sep.value, mind, maxd) + # ADM ...that aren't very faint (> 22.5 mag). ii &= objs["FLUX_R"] > 1 objs = objs[ii] + # ADM limit to northern objects in northern imaging and southern # ADM objects in southern imaging. objs = resolve(objs) - allobjs.append(objs) + + # ADM only retain critical columns from the global data model. + data = np.zeros(len(objs), dtype=streamcols.dtype) + sharedcols = set(data.dtype.names).intersection(set(objs.dtype.names)) + for col in sharedcols: + data[col] = objs[col] + + # ADM retain the data from this part of the loop. + allobjs.append(data) if i % 10 == 9: log.info(f"Ran {i+1}/{len(infiles)} files...t={time()-start:.1f}s") + # ADM assemble all of the relevant objects. allobjs = np.concatenate(allobjs) log.info(f"Found {len(allobjs)} total objects...t={time()-start:.1f}s") @@ -575,7 +599,7 @@ def read_data(swdir, rapol, decpol, ra_ref, mind, maxd, stream_name, # ADM if cache was passed and $TARG_DIR was set write the data. if writecache: # ADM if the file doesn't exist we may need to make the directory. - log.info(f"Writing cache...t={time()-start:.1f}s") + log.info(f"Writing cache to {cachefile}...t={time()-start:.1f}s") os.makedirs(os.path.dirname(cachefile), exist_ok=True) io.write_with_units(cachefile, allobjs, extname="STREAMCACHE") From a83093ad7eeb0308c23532544c360b7409363d83 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Tue, 20 Feb 2024 14:52:25 -0800 Subject: [PATCH 10/63] begin updating Gaia-matching infrastructure for DR3 --- py/desitarget/gaiamatch.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/py/desitarget/gaiamatch.py b/py/desitarget/gaiamatch.py index 6c077951..4201ca2f 100644 --- a/py/desitarget/gaiamatch.py +++ b/py/desitarget/gaiamatch.py @@ -1131,12 +1131,12 @@ def read_gaia_file(filename, header=False, addobjid=False, dr="dr2"): outcol = "{}_{}".format(prefix, col) w = np.where(outdata[outcol] != 0)[0] outdata[outcol][w] = 1./(outdata[outcol][w]**2.) - else: + elif dr == "dr2": readcolumns = list(ingaiadatamodel.dtype.names) outdata = fx[1].read(columns=readcolumns) # ADM basic check for mismatched files. if 'G3' in outdata["REF_CAT"]: - msg = "{} is a dr3 file, but the dr input is {}".format(filename, dr) + msg = "{} is an edr3 file, but dr input is {}".format(filename, dr) log.error(msg) raise ValueError(msg) outdata.dtype.names = gaiadatamodel.dtype.names @@ -1146,6 +1146,24 @@ def read_gaia_file(filename, header=False, addobjid=False, dr="dr2"): for col in ['PMRA_IVAR', 'PMDEC_IVAR', 'PARALLAX_IVAR']: w = np.where(outdata[col] != 0)[0] outdata[col][w] = 1./(outdata[col][w]**2.) + # ADM as of DR3 I decided to deprecate different column names as it's + # ADM hard to maintain. Instead try to pass Gaia DR in file headers. + else: + readcolumns = list(indr3datamodelfull.dtype.names) + outdata = fx[1].read(columns=readcolumns) + # ADM basic check for mismatched files. + if 'G2' in outdata["REF_CAT"] or 'G3' in outdata["REF_CAT"]: + msg = "{} is a dr2/edr3 file, but dr set to {}".format(filename, dr) + log.error(msg) + raise ValueError(msg) + # ADM the output data model. + outdata.dtype.names = [nm.replace("ERROR", "IVAR") if "ERROR" in nm + else nm for nm in indr3datamodelfull.dtype.names] + # ADM convert any errors to IVARs. + cols = [nom for nom in outdata.dtype.names if "IVAR" in nom] + for col in cols: + ii = outdata[outcol] != 0 + outdata[outcol][ii] = 1./(outdata[outcol][ii]**2.) # ADM if requested, add an object identifier for each file row. if addobjid: From 43939b799f64d69ca2fb3618100fb4a8b6878014 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Mon, 26 Feb 2024 11:29:27 -0800 Subject: [PATCH 11/63] Finish updating Gaia-matching infrastructure for DR3 --- py/desitarget/gaiamatch.py | 154 +++++++++++++++++++++-------- py/desitarget/streams/utilities.py | 58 ++++++++--- 2 files changed, 158 insertions(+), 54 deletions(-) diff --git a/py/desitarget/gaiamatch.py b/py/desitarget/gaiamatch.py index 4201ca2f..e531a180 100644 --- a/py/desitarget/gaiamatch.py +++ b/py/desitarget/gaiamatch.py @@ -189,6 +189,61 @@ ('EBPMINRP_GSPPHOT_UPPER', '>f4'), ('LIBNAME_GSPPHOT', 'i8'), ('DESIGNATION', 'i8'), + ('RANDOM_INDEX', '>i8'), ('REF_CAT', 'S2'), ('REF_EPOCH', '>f4'), ('RA', '>f8'), + ('RA_IVAR', '>f8'), ('DEC', '>f8'), ('DEC_IVAR', '>f8'), + ('PARALLAX', '>f4'), ('PARALLAX_IVAR', '>f4'), ('PARALLAX_OVER_ERROR', '>f4'), + ('PM', '>f4'), ('PMRA', '>f4'), ('PMRA_IVAR', '>f4'), + ('PMDEC', '>f4'), ('PMDEC_IVAR', '>f4'), ('RA_DEC_CORR', '>f4'), + ('RA_PARALLAX_CORR', '>f4'), ('RA_PMRA_CORR', '>f4'), ('RA_PMDEC_CORR', '>f4'), + ('DEC_PARALLAX_CORR', '>f4'), ('DEC_PMRA_CORR', '>f4'), ('DEC_PMDEC_CORR', '>f4'), + ('PARALLAX_PMRA_CORR', '>f4'), ('PARALLAX_PMDEC_CORR', '>f4'), ('PMRA_PMDEC_CORR', '>f4'), + ('ASTROMETRIC_N_OBS_AL', '>i2'), ('ASTROMETRIC_N_OBS_AC', '>i2'), ('ASTROMETRIC_N_GOOD_OBS_AL', '>i2'), + ('ASTROMETRIC_N_BAD_OBS_AL', '>i2'), ('ASTROMETRIC_GOF_AL', '>f4'), ('ASTROMETRIC_CHI2_AL', '>f4'), + ('ASTROMETRIC_EXCESS_NOISE', '>f4'), ('ASTROMETRIC_EXCESS_NOISE_SIG', '>f4'), ('ASTROMETRIC_PARAMS_SOLVED', '>i1'), + ('ASTROMETRIC_PRIMARY_FLAG', 'f4'), ('PSEUDOCOLOUR', '>f4'), + ('PSEUDOCOLOUR_ERROR', '>f4'), ('RA_PSEUDOCOLOUR_CORR', '>f4'), ('DEC_PSEUDOCOLOUR_CORR', '>f4'), + ('PARALLAX_PSEUDOCOLOUR_CORR', '>f4'), ('PMRA_PSEUDOCOLOUR_CORR', '>f4'), ('PMDEC_PSEUDOCOLOUR_CORR', '>f4'), + ('ASTROMETRIC_MATCHED_TRANSITS', '>i2'), ('VISIBILITY_PERIODS_USED', '>i2'), ('ASTROMETRIC_SIGMA5D_MAX', '>f4'), + ('MATCHED_TRANSITS', '>i2'), ('NEW_MATCHED_TRANSITS', '>i2'), ('MATCHED_TRANSITS_REMOVED', '>i2'), + ('IPD_GOF_HARMONIC_AMPLITUDE', '>f4'), ('IPD_GOF_HARMONIC_PHASE', '>f4'), ('IPD_FRAC_MULTI_PEAK', '>i1'), + ('IPD_FRAC_ODD_WIN', '|u1'), ('RUWE', '>f4'), ('SCAN_DIRECTION_STRENGTH_K1', '>f4'), + ('SCAN_DIRECTION_STRENGTH_K2', '>f4'), ('SCAN_DIRECTION_STRENGTH_K3', '>f4'), ('SCAN_DIRECTION_STRENGTH_K4', '>f4'), + ('SCAN_DIRECTION_MEAN_K1', '>f4'), ('SCAN_DIRECTION_MEAN_K2', '>f4'), ('SCAN_DIRECTION_MEAN_K3', '>f4'), + ('SCAN_DIRECTION_MEAN_K4', '>f4'), ('DUPLICATED_SOURCE', '?'), ('PHOT_G_N_OBS', '>i4'), + ('PHOT_G_MEAN_FLUX', '>f8'), ('PHOT_G_MEAN_FLUX_ERROR', '>f4'), ('PHOT_G_MEAN_FLUX_OVER_ERROR', '>f4'), + ('PHOT_G_MEAN_MAG', '>f4'), ('PHOT_BP_N_OBS', '>i4'), ('PHOT_BP_MEAN_FLUX', '>f8'), + ('PHOT_BP_MEAN_FLUX_ERROR', '>f4'), ('PHOT_BP_MEAN_FLUX_OVER_ERROR', '>f4'), ('PHOT_BP_MEAN_MAG', '>f4'), + ('PHOT_RP_N_OBS', '>i4'), ('PHOT_RP_MEAN_FLUX', '>f8'), ('PHOT_RP_MEAN_FLUX_ERROR', '>f4'), + ('PHOT_RP_MEAN_FLUX_OVER_ERROR', '>f4'), ('PHOT_RP_MEAN_MAG', '>f4'), ('PHOT_BP_RP_EXCESS_FACTOR', '>f4'), + ('PHOT_BP_N_CONTAMINATED_TRANSITS', '>i2'), ('PHOT_BP_N_BLENDED_TRANSITS', '>i2'), ('PHOT_RP_N_CONTAMINATED_TRANSITS', '>i2'), + ('PHOT_RP_N_BLENDED_TRANSITS', '>i2'), ('PHOT_PROC_MODE', '|u1'), ('BP_RP', '>f4'), + ('BP_G', '>f4'), ('G_RP', '>f4'), ('RADIAL_VELOCITY', '>f4'), + ('RADIAL_VELOCITY_ERROR', '>f4'), ('RV_METHOD_USED', '|u1'), ('RV_NB_TRANSITS', '>i2'), + ('RV_NB_DEBLENDED_TRANSITS', '>i2'), ('RV_VISIBILITY_PERIODS_USED', '>i2'), ('RV_EXPECTED_SIG_TO_NOISE', '>f4'), + ('RV_RENORMALISED_GOF', '>f4'), ('RV_CHISQ_PVALUE', '>f4'), ('RV_TIME_DURATION', '>f4'), + ('RV_AMPLITUDE_ROBUST', '>f4'), ('RV_TEMPLATE_TEFF', '>f4'), ('RV_TEMPLATE_LOGG', '>f4'), + ('RV_TEMPLATE_FE_H', '>f4'), ('RV_ATM_PARAM_ORIGIN', '>i2'), ('VBROAD', '>f4'), + ('VBROAD_ERROR', '>f4'), ('VBROAD_NB_TRANSITS', '>i2'), ('GRVS_MAG', '>f4'), + ('GRVS_MAG_ERROR', '>f4'), ('GRVS_MAG_NB_TRANSITS', '>i2'), ('RVS_SPEC_SIG_TO_NOISE', '>f4'), + ('PHOT_VARIABLE_FLAG', 'f8'), ('B', '>f8'), + ('ECL_LON', '>f8'), ('ECL_LAT', '>f8'), ('IN_QSO_CANDIDATES', 'i2'), ('HAS_XP_CONTINUOUS', 'f4'), ('CLASSPROB_DSC_COMBMOD_GALAXY', '>f4'), + ('CLASSPROB_DSC_COMBMOD_STAR', '>f4'), ('TEFF_GSPPHOT', '>f4'), ('TEFF_GSPPHOT_LOWER', '>f4'), + ('TEFF_GSPPHOT_UPPER', '>f4'), ('LOGG_GSPPHOT', '>f4'), ('LOGG_GSPPHOT_LOWER', '>f4'), + ('LOGG_GSPPHOT_UPPER', '>f4'), ('MH_GSPPHOT', '>f4'), ('MH_GSPPHOT_LOWER', '>f4'), + ('MH_GSPPHOT_UPPER', '>f4'), ('DISTANCE_GSPPHOT', '>f4'), ('DISTANCE_GSPPHOT_LOWER', '>f4'), + ('DISTANCE_GSPPHOT_UPPER', '>f4'), ('AZERO_GSPPHOT', '>f4'), ('AZERO_GSPPHOT_LOWER', '>f4'), + ('AZERO_GSPPHOT_UPPER', '>f4'), ('AG_GSPPHOT', '>f4'), ('AG_GSPPHOT_LOWER', '>f4'), + ('AG_GSPPHOT_UPPER', '>f4'), ('EBPMINRP_GSPPHOT', '>f4'), ('EBPMINRP_GSPPHOT_LOWER', '>f4'), + ('EBPMINRP_GSPPHOT_UPPER', '>f4'), ('LIBNAME_GSPPHOT', 'i2'), ('BRICKID', '>i4'), ('TYPE', 'S4'), ('OBJID', '>i4'), ('RA', '>f8'), ('DEC', '>f8'), ('EBV', '>f4'), ('FLUX_G', '>f4'), ('FLUX_R', '>f4'), ('FLUX_Z', '>f4'), - ('REF_EPOCH', '>f4'), ('PARALLAX', '>f4'), ('PARALLAX_ERROR', '>f4'), - ('PMRA', '>f4'), ('PMRA_ERROR', '>f4'), - ('PMDEC', '>f4'), ('PMDEC_ERROR', '>f4'), + ('REF_EPOCH', '>f4'), ('PARALLAX', '>f4'), ('PARALLAX_IVAR', '>f4'), + ('PMRA', '>f4'), ('PMRA_IVAR', '>f4'), + ('PMDEC', '>f4'), ('PMDEC_IVAR', '>f4'), ('ASTROMETRIC_PARAMS_SOLVED', '>i1'), ('NU_EFF_USED_IN_ASTROMETRY', '>f4'), ('PSEUDOCOLOUR', '>f4'), ('PHOT_G_MEAN_MAG', '>f4'), ('ECL_LAT', '>f8') ]) + def cosd(x): """Return cos(x) for an angle x in degrees. """ @@ -412,7 +414,8 @@ def plx_sel_func(dist, D, mult, plx_sys=0.05): Numpy structured array of Gaia information that contains at least the columns `RA`, `ASTROMETRIC_PARAMS_SOLVED`, `PHOT_G_MEAN_MAG`, `NU_EFF_USED_IN_ASTRONOMY`, `PSEUDOCOLOUR`, `ECL_LAT`, `PARALLAX` - `PARALLAX_ERROR`. + `PARALLAX_ERROR`. `PARALLAX_IVAR` will be used instead of + `PARALLAX_ERROR` if `PARALLAX_ERROR` is not present. mult : :class:`float` or `int` Multiple of the parallax error to use for padding. @@ -440,11 +443,19 @@ def plx_sel_func(dist, D, mult, plx_sys=0.05): plx = D['PARALLAX'] - plx_zpt dplx = 1 / dist - plx - return np.abs(dplx) < plx_sys + mult * D['PARALLAX_ERROR'] + if 'PARALLAX_ERROR' in D.dtype.names: + parallax_error = D['PARALLAX_ERROR'] + elif 'PARALLAX_IVAR' in D.dtype.names: + parallax_error = 1./np.sqrt(D['PARALLAX_IVAR']) + else: + msg = "Either PARALLAX_ERROR or PARALLAX_IVAR must be passed!" + log.error(msg) + + return np.abs(dplx) < plx_sys + mult * parallax_error def read_data(swdir, rapol, decpol, ra_ref, mind, maxd, stream_name, - readcache=True, addnors=True): + readcache=True, addnors=True, test=False): """Assemble the data needed for a particular stream program. Example values for GD1: @@ -486,6 +497,9 @@ def read_data(swdir, rapol, decpol, ra_ref, mind, maxd, stream_name, versa, i.e. if `swdir` contains "south" add sweep files from the north by substituting "north" in place of "south"). + test : :class:`bool` + Read a subset of the data for testing purposes. + Returns ------- :class:`array_like` or `boolean` @@ -530,6 +544,7 @@ def read_data(swdir, rapol, decpol, ra_ref, mind, maxd, stream_name, # ADM read in the sweep files. infiles = io.list_sweepfiles(swdir) + # ADM read both the north and south directories, if requested. if addnors: if "south" in swdir: @@ -563,6 +578,13 @@ def read_data(swdir, rapol, decpol, ra_ref, mind, maxd, stream_name, # ADM determine which sweep files touch the relevant HEALPixels. filesperpixel, _, _ = sweep_files_touch_hp(nside, pixlist, infiles) infiles = list(np.unique(np.hstack([filesperpixel[pix] for pix in pixlist]))) + + # ADM read a subset of the data for testing purposes, if requested. + if test: + msg = "Limiting data to first 20 files for testing purposes" + log.info(msg) + infiles = infiles[:20] + # ADM loop through the sweep files and limit to objects in the stream. allobjs = [] for i, filename in enumerate(infiles): @@ -573,19 +595,31 @@ def read_data(swdir, rapol, decpol, ra_ref, mind, maxd, stream_name, # ADM only retain objects in the stream... ii = betw(sep.value, mind, maxd) - # ADM ...that aren't very faint (> 22.5 mag). + # ADM ...that aren't very faint (> 22.5 mag)... ii &= objs["FLUX_R"] > 1 objs = objs[ii] # ADM limit to northern objects in northern imaging and southern # ADM objects in southern imaging. - objs = resolve(objs) + LSobjs = resolve(objs) + + # ADM match to Gaia DR3. + # ADM catch the case where there are no objects meeting the cuts. + if len(LSobjs) > 0: + gaiaobjs = match_gaia_to_primary(LSobjs, matchrad=1., dr='dr3') + else: + gaiaobjs = LSobjs + + # ADM a (probably unnecessary) sanity check. + assert(len(gaiaobjs) == len(LSobjs)) # ADM only retain critical columns from the global data model. - data = np.zeros(len(objs), dtype=streamcols.dtype) - sharedcols = set(data.dtype.names).intersection(set(objs.dtype.names)) - for col in sharedcols: - data[col] = objs[col] + data = np.zeros(len(LSobjs), dtype=streamcols.dtype) + # ADM for both Gaia and Legacy Surveys, overwriting with Gaia. + for objs in LSobjs, gaiaobjs: + sharedcols = set(data.dtype.names).intersection(set(objs.dtype.names)) + for col in sharedcols: + data[col] = objs[col] # ADM retain the data from this part of the loop. allobjs.append(data) From 0957fed8b3e4c8b1606f7ce3cada6bac3fb8b6ca Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Mon, 26 Feb 2024 17:27:14 -0800 Subject: [PATCH 12/63] Begin writing selection cuts for GD1 --- py/desitarget/streams/cuts.py | 169 +++++++++++++++++++++++++++++ py/desitarget/streams/utilities.py | 78 +++++++------ 2 files changed, 212 insertions(+), 35 deletions(-) create mode 100644 py/desitarget/streams/cuts.py diff --git a/py/desitarget/streams/cuts.py b/py/desitarget/streams/cuts.py new file mode 100644 index 00000000..095f1fac --- /dev/null +++ b/py/desitarget/streams/cuts.py @@ -0,0 +1,169 @@ +""" +desitarget.streamcuts +===================== + +Target selection cuts for the DESI MWS Stellar Stream programs. + +Borrows heavily from Sergey Koposov. +""" +from time import time + +# ADM set up the DESI default logger +from desiutil.log import get_logger +log = get_logger() + +# ADM start the clock +start = time() + +import healpy +import datetime +import os +import scipy.interpolate +import numpy as np +import astropy.table as atpy + +from desitarget.cuts import _psflike +from desitarget.streams.utilities import read_data, sphere_rotate +from desitarget.streams.utilities import correct_pm, rotate_pm, betw +from desitarget.streams.utilities import pm12_sel_func, plx_sel_func + + +def is_in_GD1(objs): + """Whether a target lies within the GD1 stellar stream. + + Parameters + ---------- + objs : :class:`array_like` + Numpy rec array with at least the Legacy Surveys/Gaia columns: + RA, DEC, PARALLAX, PMRA, PMDEC, PARALLAX_IVAR, PMRA_IVAR, + PMDEC_IVAR, EBV, FLUX_G, FLUX_R, FLUX_Z, PSEUDOCOLOUR, TYPE, + ASTROMETRIC_PARAMS_SOLVED, NU_EFF_USED_IN_ASTROMETRY, + ECL_LAT, PHOT_G_MEAN_MAG. + + Returns + ------- + :class:`array_like` + ``True`` if the object is a BGS target of type ``targtype``. + + Notes + ----- + + """ + # ADM the parameters that define the coordinates of the stream. + rapol, decpol, ra_ref = 34.5987, 29.7331, 200 + + # ADM the parameters that define the extent of the stream. + mind, maxd = 80, 100 + + # ADM the name of the stream. + stream_name = "GD1" + + # ADM collect the data, including Gaia-matching to DR3. + D = read_data(swdir, rapol, decpol, ra_ref, mind, maxd, stream_name, + readcache=readcache) + + # ADM rotate the data into the coordinate system of the stream. + fi1, fi2 = sphere_rotate(objs['RA'], objs['DEC'], rapol, decpol, ra_ref) + + # ADM distance of the stream (similar to Koposov et al. 2010 paper). + dist = stream_distance(fi1, stream_name) + + # ADM heliocentric correction to proper motion. + xpmra, xpmdec = correct_pm(objs['RA'], objs['DEC'], + objs['PMRA'], objs['PMDEC'], dist) + + # ADM reflex correction for proper motions. + pmfi1, pmfi2 = rotate_pm(objs['RA'], objs['DEC'], xpmra, xpmdec, + rapol, decpol, ra_ref) + + # ADM derive the combined proper motion error. + pmra_error = 1./np.sqrt(objs['PMRA_IVAR']) + pmdec_error = 1./np.sqrt(objs['PMDEC_IVAR']) + pm_err = np.sqrt(0.5 * (pmra_error**2 + pmdec_error**2)) + + # ADM dust correction. + ext_coeff = dict(g=3.237, r=2.176, z=1.217) + eg, er, ez = [ext_coeff[_] * objs['EBV'] for _ in 'grz'] + ext = {} + ext['G'] = eg + ext['R'] = er + ext['Z'] = ez + + g, r, z = [22.5 - 2.5 * np.log10(objs['FLUX_' + _]) - ext[_] for _ in 'GRZ'] + + # ADM some spline function over which to interpolate. + TRACK = scipy.interpolate.CubicSpline([-90, -70, -50, -40, -20, 0, 20], + [-3, -1.5, -.2, -0., -.0, -1.2, -3]) + PM1TRACK = scipy.interpolate.UnivariateSpline( + [-90, -70, -50, -30, -15, 0, 20], + [-5.5, -6.3, -8, -6.5, -5.7, -5, -3.5]) + PM2TRACK = scipy.interpolate.UnivariateSpline([-90, -60, -45, -30, 0, 20], + [-.7, -.7, -0.35, -0.1, 0.2, .5], + s=0) + + # ADM create an interpolated set of phi2 coords (in stream coords). + dfi2 = fi2 - TRACK(fi1) + + # ADM derive the isochrone track for the stream. + CMD_II = get_CMD_interpolator(stream_name) + + # ADM how far the data lies from the isochrone. + delta_cmd = g - r - CMD_II(r - 5 * np.log10(dist * 1e3) + 5) + + # ADM necessary parameters are set up; perform the actual selection. + bright_limit, faint_limit = 16, 21 + + # ADM lies in the stream. + field_sel = betw(dfi2, -10, 10) & betw(fi1, -90, 21) + + # ADM Gaia-based selection (proper motion and parallax). + pm_pad = 2 # mas/yr padding in pm selection + gaia_astrom_sel = pm12_sel_func(fi1, pmfi1, pmfi2, pm_err, pm_pad, 2.5) + gaia_astrom_sel &= plx_sel_func(fi1, D, 2.5) + gaia_astrom_sel &= r > bright_limit + + log.info(f"Objects in the field of the stream: {field_sel.sum()}") + log.info(f"With correct astrometry: {(gaia_astrom_sel & field_sel).sum()}") + + # ADM padding around the isochrone. + bright_cmd_sel = betw(delta_cmd, -.2, .2) + + # ADM isochrone selection. + stellar_locus_blue_sel = ((betw(r - z - (-.17 + .67 * (g - r)), -0.2, 0.2) + & ((g - r) <= 1.1))) + stellar_locus_red_sel = (((g - r > 1.1) + & betw(g - r - (1.05 + .25 * (r - z)), -.2, .2))) + stellar_locus_sel = stellar_locus_blue_sel | stellar_locus_red_sel + + tot = np.sum(field_sel & gaia_astrom_sel & bright_cmd_sel) + print(f"With correct astrometry AND correct cmd: {tot}") + + # ADM selection for objects that lack Gaia astrometry. + # ADM has type PSF and in a reasonable isochrone window. + startyp = _psflike(objs["TYPE"]) + cmd_win = 0.1 + 10**(-2 + (r - 20) / 2.5) + + # ADM overall faint selection. + faint_sel = ~np.isfinite(objs['PMRA']) + faint_sel &= betw(r, 20, faint_limit) + faint_sel &= betw(np.abs(delta_cmd), 0, cmd_win)) + faint_sel &= startyp + faint_sel &= stellar_locus_sel + tot = np.sum(faint_sel & field_sel) + log.info(f"Objects in the field that meet the faint selection: {tot}") + + # ADM "filler" selections. + # (PSF type + blue in colour and not previously selected) + common_filler_sel &= betw(r, 19, faint_limit) + common_filler_sel &= startyp + common_filler_sel &= ~faint_sel + common_filler_set &= ~gaia_astrom_sel + common_filler_sel &= stellar_locus_sel + + filler_sel = common_filler_sel & betw(g - r, -.3, 1.2) + + filler_red_sel = common_filler_sel & betw(g - r, 1.2, 2.2) + tot = np.sum(filler_sel & field_sel) + log.info(f"Objects in the field that meet the filler selection: {tot}") + + return diff --git a/py/desitarget/streams/utilities.py b/py/desitarget/streams/utilities.py index 662a9442..86687d97 100644 --- a/py/desitarget/streams/utilities.py +++ b/py/desitarget/streams/utilities.py @@ -18,6 +18,7 @@ from pkg_resources import resource_filename from scipy.interpolate import UnivariateSpline from time import time +from zero_point import zero_point as gaia_zpt from desitarget import io from desitarget.geomask import pixarea2nside, add_hp_neighbors, sweep_files_touch_hp @@ -31,6 +32,9 @@ # ADM start the clock. start = time() +# ADM load the Gaia zeropoints. +gaia_zpt.zpt.load_tables() + # ADM Galactic reference frame. Use astropy v4.0 defaults. GCPARAMS = acoo.galactocentric_frame_defaults.get_from_registry( "v4.0")['parameters'] @@ -71,10 +75,8 @@ def betw(x, x1, x2): ---------- x : :class:`~numpy.ndarray` or `int` or `float` Value(s) that need checked against x1, x2. - x1 : :class:`~numpy.ndarray` or `int` or `float` Lower range to check against (inclusive). - x2 : :class:`~numpy.ndarray` or `int` or `float` Upper range to check against (exclusive). @@ -103,7 +105,6 @@ def torect(ra, dec): ---------- ra : :class:`~numpy.ndarray` or `float` Right Ascension in DEGREES. - dec : :class:`~numpy.ndarray` or `float` Declination in DEGREES. @@ -150,7 +151,6 @@ def rotation_matrix(rapol, decpol, ra0): ---------- rapol, decpol : :class:`float` Pole of the new coordinate system in DEGREES. - ra0 : :class:`float` Zero latitude of the new coordinate system in DEGREES. @@ -178,16 +178,12 @@ def sphere_rotate(ra, dec, rapol, decpol, ra0, revert=False): ---------- ra : :class:`~numpy.ndarray` or `float` Right Ascension in DEGREES. - dec : :class:`~numpy.ndarray` or `float` Declination in DEGREES. - rapol, decpol : :class:`float` Pole of the new coordinate system in DEGREES. - ra0 : :class:`float` Zero latitude of the new coordinate system in DEGREES. - revert : :class:`bool`, optional, defaults to ``False`` Reverse the rotation. @@ -225,19 +221,14 @@ def rotate_pm(ra, dec, pmra, pmdec, rapol, decpol, ra0, revert=False): ---------- ra, dec : :class:`~numpy.ndarray` or `float` Right Ascension, Declination in DEGREES. - pmra, pmdec : :class:`~numpy.ndarray` or `float` Proper motion in Right Ascension, Declination in mas/yr. - pmdec : :class:`~numpy.ndarray` or `float` Proper motion in Declination in mas/yr. - rapol, decpol : :class:`float` Pole of the new coordinate system in DEGREES. - ra0 : :class:`float` Zero latitude of the new coordinate system in DEGREES. - revert : :class:`bool`, optional, defaults to ``False`` Reverse the rotation. @@ -287,11 +278,9 @@ def correct_pm(ra, dec, pmra, pmdec, dist): ---------- ra, dec : :class:`~numpy.ndarray` or `float` Right Ascension, Declination in DEGREES. - pmra, pmdec : :class:`~numpy.ndarray` or `float` Proper motion in Right Ascension, Declination in mas/yr. `pmra` includes the cosine term. - dist : :class:`float` Distance in kpc. @@ -370,25 +359,19 @@ def pm12_sel_func(pm1track, pm2track, pmfi1, pmfi2, pm_err, pad=2, mult=2.5): ---------- pm1track : :class:`~numpy.ndarray` or `float` Allowed proper motions of stream targets, RA-sense. - pm2track : :class:`~numpy.ndarray` or `float` Allowed proper motions of stream targets, Dec-sense. - pmfi1 : :class:`~numpy.ndarray` or `float` Proper motion in stream coordinates of possible targets, derived from RA. - pmfi2 : :class:`~numpy.ndarray` or `float` Proper motion in stream coordinates of possible targets, derived from Dec. - pm_err : :class:`~numpy.ndarray` or `float` Proper motion error in stream coordinates of possible targets, combined across `pmfi1` and `pmfi2` errors. - pad: : :class:`float` or `int`, defaults to 2 Extra offset with which to pad `mult`*proper_motion_error. - mult : :class:`float` or `int`, defaults to 2.5 Multiple of the proper motion error to use for padding. @@ -409,17 +392,14 @@ def plx_sel_func(dist, D, mult, plx_sys=0.05): ---------- dist : :class:`~numpy.ndarray` or `float` Distance of possible stream members. - D : :class:`~numpy.ndarray` Numpy structured array of Gaia information that contains at least the columns `RA`, `ASTROMETRIC_PARAMS_SOLVED`, `PHOT_G_MEAN_MAG`, `NU_EFF_USED_IN_ASTRONOMY`, `PSEUDOCOLOUR`, `ECL_LAT`, `PARALLAX` `PARALLAX_ERROR`. `PARALLAX_IVAR` will be used instead of `PARALLAX_ERROR` if `PARALLAX_ERROR` is not present. - mult : :class:`float` or `int` Multiple of the parallax error to use for padding. - plx_sys : :class:`float` Extra offset with which to pad `mult`*parallax_error. @@ -454,6 +434,34 @@ def plx_sel_func(dist, D, mult, plx_sys=0.05): return np.abs(dplx) < plx_sys + mult * parallax_error +def stream_distance(fi1, stream_name): + """The distance to members of a stellar stream. + + Parameters + ---------- + fi1 : :class:`~numpy.ndarray` or `float` + Phi1 stream coordinate of possible targets, derived from RA. + stream_name : :class:`str` + Name of a stream, e.g. "GD1". + + Returns + ------- + :class:`array_like` or `float` + The distance to the passed members of the stream. + + Notes + ----- + - Output type is the same as that of the passed `fi1`. + """ + if stream_name.upper() == "GD1": + # ADM The distance to GD1 (similar to Koposov et al. 2010 paper). + dm = 18.82 + ((fi1 + 48) / 57)**2 - 4.45 + return 10**(dm / 5. - 2) + else: + msg = f"stream name {stream_name} not recognized" + log.error(msg) + + def read_data(swdir, rapol, decpol, ra_ref, mind, maxd, stream_name, readcache=True, addnors=True, test=False): """Assemble the data needed for a particular stream program. @@ -469,20 +477,15 @@ def read_data(swdir, rapol, decpol, ra_ref, mind, maxd, stream_name, Root directory of Legacy Surveys sweep files for a given data release for ONE of EITHER north or south, e.g. "/global/cfs/cdirs/cosmo/data/legacysurvey/dr9/south/sweep/9.0". - rapol, decpol : :class:`float` Pole in the stream coordinate system in DEGREES. - ra_ref : :class:`float` Zero latitude in the stream coordinate system in DEGREES. - mind, maxd : :class:`float` or `int` Minimum and maximum angular distance from the pole of the stream coordinate system to search for members in DEGREES. - stream_name : :class:`str` Name of a stream. Used to make the cached filename, e.g. "GD1". - readcache : :class:`bool` If ``True`` read from a previously constructed and cached file automatically, IF such a file exists. If ``False`` don't read @@ -490,13 +493,11 @@ def read_data(swdir, rapol, decpol, ra_ref, mind, maxd, stream_name, cached file is $TARG_DIR/streamcache/streamname-drX-cache.fits, where streamname is the lower-case passed `stream_name` and drX is the Legacy Surveys Data Release (parsed from `swdir`). - addnors : :class:`bool` If ``True`` then if `swdir` contains "north" add sweep files from the south by substituting "south" in place of "north" (and vice versa, i.e. if `swdir` contains "south" add sweep files from the north by substituting "north" in place of "south"). - test : :class:`bool` Read a subset of the data for testing purposes. @@ -510,6 +511,9 @@ def read_data(swdir, rapol, decpol, ra_ref, mind, maxd, stream_name, - The $TARG_DIR environment variable must be set to read/write from a cache. If $TARG_DIR is not set, caching is completely ignored. """ + # ADM The Gaia DR to which to match. + gaiadr = "dr3" + # ADM check whether $TARG_DIR exists. If it does, agree to read from # ADM and write to the cache. writecache = True @@ -603,10 +607,9 @@ def read_data(swdir, rapol, decpol, ra_ref, mind, maxd, stream_name, # ADM objects in southern imaging. LSobjs = resolve(objs) - # ADM match to Gaia DR3. # ADM catch the case where there are no objects meeting the cuts. if len(LSobjs) > 0: - gaiaobjs = match_gaia_to_primary(LSobjs, matchrad=1., dr='dr3') + gaiaobjs = match_gaia_to_primary(LSobjs, matchrad=1., dr=gaiadr) else: gaiaobjs = LSobjs @@ -630,11 +633,16 @@ def read_data(swdir, rapol, decpol, ra_ref, mind, maxd, stream_name, allobjs = np.concatenate(allobjs) log.info(f"Found {len(allobjs)} total objects...t={time()-start:.1f}s") - # ADM if cache was passed and $TARG_DIR was set write the data. + # ADM if cache was passed and $TARG_DIR was set then write the data. if writecache: # ADM if the file doesn't exist we may need to make the directory. log.info(f"Writing cache to {cachefile}...t={time()-start:.1f}s") os.makedirs(os.path.dirname(cachefile), exist_ok=True) - io.write_with_units(cachefile, allobjs, extname="STREAMCACHE") + # ADM at least add the Gaia DR used to the header. + hdr = fitsio.FITSHDR() + hdr.add_record(dict(name="GAIADR", value=gaiadr, + comment="GAIA Data Release matched to")) + io.write_with_units(cachefile, allobjs, + header=hdr, extname="STREAMCACHE") return allobjs From 6fbb2c89568a9a2536e36ef324f261e8fb02cbc6 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Tue, 27 Feb 2024 09:47:35 -0800 Subject: [PATCH 13/63] centralize all the shared stream parameters in a yaml file --- py/desitarget/data/streams.yaml | 9 +++++++- py/desitarget/streams/cuts.py | 28 +++++++++++----------- py/desitarget/streams/utilities.py | 37 +++++++++++++++++++++++------- 3 files changed, 51 insertions(+), 23 deletions(-) diff --git a/py/desitarget/data/streams.yaml b/py/desitarget/data/streams.yaml index fc0dc9a8..f4a17dab 100644 --- a/py/desitarget/data/streams.yaml +++ b/py/desitarget/data/streams.yaml @@ -1,7 +1,14 @@ -# ADM This yaml file stores isochrones for stellar streams. +# ADM This yaml file stores isochrones and coordinate information for stellar streams. # ADM the gd1 stream. GD1: + # ADM stream coordinates. + RAPOL: 34.5987 + DECPOL: 29.7331 + RA_REF: 200 + # ADM stream extent. + MIND: 80 + MAXD: 100 # ADM color offset. COLOFF: -0.01 # ADM magnitude offset. diff --git a/py/desitarget/streams/cuts.py b/py/desitarget/streams/cuts.py index 095f1fac..790c60f1 100644 --- a/py/desitarget/streams/cuts.py +++ b/py/desitarget/streams/cuts.py @@ -23,10 +23,9 @@ import astropy.table as atpy from desitarget.cuts import _psflike -from desitarget.streams.utilities import read_data, sphere_rotate -from desitarget.streams.utilities import correct_pm, rotate_pm, betw -from desitarget.streams.utilities import pm12_sel_func, plx_sel_func - +from desitarget.streams.utilities import read_data, sphere_rotate, correct_pm, \ + rotate_pm, betw, pm12_sel_func, plx_sel_func, get_CMD_interpolator, \ + get_stream_parameters def is_in_GD1(objs): """Whether a target lies within the GD1 stellar stream. @@ -49,15 +48,16 @@ def is_in_GD1(objs): ----- """ - # ADM the parameters that define the coordinates of the stream. - rapol, decpol, ra_ref = 34.5987, 29.7331, 200 - - # ADM the parameters that define the extent of the stream. - mind, maxd = 80, 100 - # ADM the name of the stream. stream_name = "GD1" + # ADM look up the defining parameters of the stream. + stream = get_stream_parameters(stream_name) + # ADM the parameters that define the coordinates of the stream. + rapol, decpol, ra_ref = stream["RAPOL"], stream["DECPOL"], stream["RA_REF"] + # ADM the parameters that define the extent of the stream. + mind, maxd = stream["MIND"], stream["MAXD"] + # ADM collect the data, including Gaia-matching to DR3. D = read_data(swdir, rapol, decpol, ra_ref, mind, maxd, stream_name, readcache=readcache) @@ -67,7 +67,7 @@ def is_in_GD1(objs): # ADM distance of the stream (similar to Koposov et al. 2010 paper). dist = stream_distance(fi1, stream_name) - + # ADM heliocentric correction to proper motion. xpmra, xpmdec = correct_pm(objs['RA'], objs['DEC'], objs['PMRA'], objs['PMDEC'], dist) @@ -80,7 +80,7 @@ def is_in_GD1(objs): pmra_error = 1./np.sqrt(objs['PMRA_IVAR']) pmdec_error = 1./np.sqrt(objs['PMDEC_IVAR']) pm_err = np.sqrt(0.5 * (pmra_error**2 + pmdec_error**2)) - + # ADM dust correction. ext_coeff = dict(g=3.237, r=2.176, z=1.217) eg, er, ez = [ext_coeff[_] * objs['EBV'] for _ in 'grz'] @@ -151,7 +151,7 @@ def is_in_GD1(objs): faint_sel &= stellar_locus_sel tot = np.sum(faint_sel & field_sel) log.info(f"Objects in the field that meet the faint selection: {tot}") - + # ADM "filler" selections. # (PSF type + blue in colour and not previously selected) common_filler_sel &= betw(r, 19, faint_limit) @@ -165,5 +165,5 @@ def is_in_GD1(objs): filler_red_sel = common_filler_sel & betw(g - r, 1.2, 2.2) tot = np.sum(filler_sel & field_sel) log.info(f"Objects in the field that meet the filler selection: {tot}") - + return diff --git a/py/desitarget/streams/utilities.py b/py/desitarget/streams/utilities.py index 86687d97..4b943fd9 100644 --- a/py/desitarget/streams/utilities.py +++ b/py/desitarget/streams/utilities.py @@ -311,8 +311,8 @@ def correct_pm(ra, dec, pmra, pmdec, dist): (C.pm_dec - C1.pm_dec).to_value(masyr)) -def get_CMD_interpolator(stream_name): - """Isochrones via interpolating over points in color-magnitude space. +def get_stream_parameters(stream_name): + """Look up information for a given stream. Parameters ---------- @@ -322,7 +322,9 @@ def get_CMD_interpolator(stream_name): Returns ------- - A scipy interpolated UnivariateSpline. + :class:`~dict` + A dictionary of stream parameters for the passed `stream_name`. + Includes isochrones and positional information. Notes ----- @@ -336,13 +338,32 @@ def get_CMD_interpolator(stream_name): with open(fn) as f: stream = yaml.safe_load(f) + return stream[stream_name] + + +def get_CMD_interpolator(stream_name): + """Isochrones via interpolating over points in color-magnitude space. + + Parameters + ---------- + stream_name : :class:`str` + Name of a stream that appears in the ../data/streams.yaml file. + Possibilities include 'GD1'. + + Returns + ------- + A scipy interpolated UnivariateSpline. + """ + # ADM get information for the stream of interest. + stream = get_stream_parameters(stream_name) + # ADM retrieve the color and magnitude offsets. - coloff = stream[stream_name]["COLOFF"] - magoff = stream[stream_name]["MAGOFF"] + coloff = stream["COLOFF"] + magoff = stream["MAGOFF"] # ADM the isochrones to interpolate over. - iso_dartmouth_g = np.array(stream[stream_name]["ISO_G"]) - iso_dartmouth_r = np.array(stream[stream_name]["ISO_R"]) + iso_dartmouth_g = np.array(stream["ISO_G"]) + iso_dartmouth_r = np.array(stream["ISO_R"]) # ADM UnivariateSpline is from scipy.interpolate. CMD_II = UnivariateSpline(iso_dartmouth_r[::-1] + magoff, @@ -415,7 +436,7 @@ def plx_sel_func(dist, D, mult, plx_sys=0.05): D['NU_EFF_USED_IN_ASTROMETRY'][subset], D['PSEUDOCOLOUR'][subset], D['ECL_LAT'][subset], - D['ASTROMETRIC_PARAMS+SOLVED'][subset], + D['ASTROMETRIC_PARAMS_SOLVED'][subset], _warnings=False) plx_zpt = np.zeros(len(D['RA'])) plx_zpt_tmp[~np.isfinite(plx_zpt_tmp)] = 0 From eb6baea99a20071eb371b7a7a81d48ab5b7e1e4d Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Wed, 28 Feb 2024 09:17:13 -0800 Subject: [PATCH 14/63] clean-up, documentation, set up the Boolean arrays that will form the targeting bitmasks --- py/desitarget/io.py | 2 +- py/desitarget/streams/cuts.py | 83 ++++++++++++++++++------------ py/desitarget/streams/utilities.py | 8 ++- 3 files changed, 57 insertions(+), 36 deletions(-) diff --git a/py/desitarget/io.py b/py/desitarget/io.py index 250e6b46..4f5708e8 100644 --- a/py/desitarget/io.py +++ b/py/desitarget/io.py @@ -423,7 +423,7 @@ def write_with_units(filename, data, extname=None, header=None, ecsv=False): Returns ------- - Nothing, but writes the `data` to the `filename` in chunks with units + Nothing, but writes the `data` to the `filename` with units added from the desitarget units yaml file (see `/data/units.yaml`). Notes diff --git a/py/desitarget/streams/cuts.py b/py/desitarget/streams/cuts.py index 790c60f1..9b59d313 100644 --- a/py/desitarget/streams/cuts.py +++ b/py/desitarget/streams/cuts.py @@ -1,4 +1,5 @@ """ + desitarget.streamcuts ===================== @@ -8,13 +9,6 @@ """ from time import time -# ADM set up the DESI default logger -from desiutil.log import get_logger -log = get_logger() - -# ADM start the clock -start = time() - import healpy import datetime import os @@ -23,10 +17,18 @@ import astropy.table as atpy from desitarget.cuts import _psflike -from desitarget.streams.utilities import read_data, sphere_rotate, correct_pm, \ - rotate_pm, betw, pm12_sel_func, plx_sel_func, get_CMD_interpolator, \ +from desitarget.streams.utilities import sphere_rotate, correct_pm, rotate_pm, \ + betw, pm12_sel_func, plx_sel_func, get_CMD_interpolator, stream_distance, \ get_stream_parameters +# ADM set up the DESI default logger +from desiutil.log import get_logger +log = get_logger() + +# ADM start the clock +start = time() + + def is_in_GD1(objs): """Whether a target lies within the GD1 stellar stream. @@ -42,15 +44,17 @@ def is_in_GD1(objs): Returns ------- :class:`array_like` - ``True`` if the object is a BGS target of type ``targtype``. - - Notes - ----- - + ``True`` if the object is a bright "BRIGHT_PM" target. + :class:`array_like` + ``True`` if the object is a faint "FAINT_NO_PM" target. + :class:`array_like` + ``True`` if the object is a white dwarf "FILLER" target. """ # ADM the name of the stream. stream_name = "GD1" + log.info(f"Starting selection for {stream_name}...t={time()-start:.1f}s") + # ADM look up the defining parameters of the stream. stream = get_stream_parameters(stream_name) # ADM the parameters that define the coordinates of the stream. @@ -58,10 +62,6 @@ def is_in_GD1(objs): # ADM the parameters that define the extent of the stream. mind, maxd = stream["MIND"], stream["MAXD"] - # ADM collect the data, including Gaia-matching to DR3. - D = read_data(swdir, rapol, decpol, ra_ref, mind, maxd, stream_name, - readcache=readcache) - # ADM rotate the data into the coordinate system of the stream. fi1, fi2 = sphere_rotate(objs['RA'], objs['DEC'], rapol, decpol, ra_ref) @@ -77,8 +77,13 @@ def is_in_GD1(objs): rapol, decpol, ra_ref) # ADM derive the combined proper motion error. - pmra_error = 1./np.sqrt(objs['PMRA_IVAR']) - pmdec_error = 1./np.sqrt(objs['PMDEC_IVAR']) + # ADM guard against dividing by zero. + pmra_error = np.zeros_like(objs["PMRA_IVAR"]) + 1e8 + ii = objs['PMRA_IVAR'] != 0 + pmra_error[ii] = 1./np.sqrt(objs[ii]['PMRA_IVAR']) + pmdec_error = np.zeros_like(objs["PMDEC_IVAR"]) + 1e8 + ii = objs['PMDEC_IVAR'] != 0 + pmdec_error[ii] = 1./np.sqrt(objs[ii]['PMDEC_IVAR']) pm_err = np.sqrt(0.5 * (pmra_error**2 + pmdec_error**2)) # ADM dust correction. @@ -97,9 +102,9 @@ def is_in_GD1(objs): PM1TRACK = scipy.interpolate.UnivariateSpline( [-90, -70, -50, -30, -15, 0, 20], [-5.5, -6.3, -8, -6.5, -5.7, -5, -3.5]) - PM2TRACK = scipy.interpolate.UnivariateSpline([-90, -60, -45, -30, 0, 20], - [-.7, -.7, -0.35, -0.1, 0.2, .5], - s=0) + PM2TRACK = scipy.interpolate.UnivariateSpline( + [-90, -60, -45, -30, 0, 20], + [-.7, -.7, -0.35, -0.1, 0.2, .5], s=0) # ADM create an interpolated set of phi2 coords (in stream coords). dfi2 = fi2 - TRACK(fi1) @@ -119,10 +124,10 @@ def is_in_GD1(objs): # ADM Gaia-based selection (proper motion and parallax). pm_pad = 2 # mas/yr padding in pm selection gaia_astrom_sel = pm12_sel_func(fi1, pmfi1, pmfi2, pm_err, pm_pad, 2.5) - gaia_astrom_sel &= plx_sel_func(fi1, D, 2.5) + gaia_astrom_sel &= plx_sel_func(fi1, objs, 2.5) gaia_astrom_sel &= r > bright_limit - log.info(f"Objects in the field of the stream: {field_sel.sum()}") + log.info(f"Objects in the field: {field_sel.sum()}...t={time()-start:.1f}s") log.info(f"With correct astrometry: {(gaia_astrom_sel & field_sel).sum()}") # ADM padding around the isochrone. @@ -136,7 +141,7 @@ def is_in_GD1(objs): stellar_locus_sel = stellar_locus_blue_sel | stellar_locus_red_sel tot = np.sum(field_sel & gaia_astrom_sel & bright_cmd_sel) - print(f"With correct astrometry AND correct cmd: {tot}") + print(f"With correct astrometry AND cmd: {tot}...t={time()-start:.1f}s") # ADM selection for objects that lack Gaia astrometry. # ADM has type PSF and in a reasonable isochrone window. @@ -144,26 +149,38 @@ def is_in_GD1(objs): cmd_win = 0.1 + 10**(-2 + (r - 20) / 2.5) # ADM overall faint selection. - faint_sel = ~np.isfinite(objs['PMRA']) + faint_sel = objs['PMRA'] == 0 faint_sel &= betw(r, 20, faint_limit) - faint_sel &= betw(np.abs(delta_cmd), 0, cmd_win)) + faint_sel &= betw(np.abs(delta_cmd), 0, cmd_win) faint_sel &= startyp faint_sel &= stellar_locus_sel tot = np.sum(faint_sel & field_sel) - log.info(f"Objects in the field that meet the faint selection: {tot}") + log.info(f"Objects that meet faint selection: {tot}...t={time()-start:.1f}s") # ADM "filler" selections. # (PSF type + blue in colour and not previously selected) - common_filler_sel &= betw(r, 19, faint_limit) + common_filler_sel = betw(r, 19, faint_limit) common_filler_sel &= startyp common_filler_sel &= ~faint_sel - common_filler_set &= ~gaia_astrom_sel + common_filler_sel &= ~gaia_astrom_sel common_filler_sel &= stellar_locus_sel filler_sel = common_filler_sel & betw(g - r, -.3, 1.2) filler_red_sel = common_filler_sel & betw(g - r, 1.2, 2.2) tot = np.sum(filler_sel & field_sel) - log.info(f"Objects in the field that meet the filler selection: {tot}") + log.info(f"Objects meeting filler selection: {tot}...t={time()-start:.1f}s") + + log.info(f"Finished selection for {stream_name}...t={time()-start:.1f}s") + + bright_pm = bright_cmd_sel & gaia_astrom_sel & field_sel + faint_no_pm = faint_sel & field_sel + filler = filler_sel & field_sel + + # ADM sanity check that selections do not overlap. + check = bright_pm.astype(int) + faint_no_pm.astype(int) + filler.astype(int) + if np.max(check) > 1: + msg = "Selections should be unique but they overlap!" + log.error(msg) - return + return bright_pm, faint_no_pm, filler diff --git a/py/desitarget/streams/utilities.py b/py/desitarget/streams/utilities.py index 4b943fd9..9061d890 100644 --- a/py/desitarget/streams/utilities.py +++ b/py/desitarget/streams/utilities.py @@ -620,8 +620,12 @@ def read_data(swdir, rapol, decpol, ra_ref, mind, maxd, stream_name, # ADM only retain objects in the stream... ii = betw(sep.value, mind, maxd) - # ADM ...that aren't very faint (> 22.5 mag)... + # ADM ...that aren't very faint (> 22.5 mag in r). ii &= objs["FLUX_R"] > 1 + # ADM Also guard against negative fluxes in g/r. + ii &= objs["FLUX_G"] > 0. + ii &= objs["FLUX_Z"] > 0. + objs = objs[ii] # ADM limit to northern objects in northern imaging and southern @@ -647,7 +651,7 @@ def read_data(swdir, rapol, decpol, ra_ref, mind, maxd, stream_name, # ADM retain the data from this part of the loop. allobjs.append(data) - if i % 10 == 9: + if i % 5 == 4: log.info(f"Ran {i+1}/{len(infiles)} files...t={time()-start:.1f}s") # ADM assemble all of the relevant objects. From a816c47e9cab7597199b74e504436377f5ee6fa3 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Fri, 1 Mar 2024 17:05:18 -0800 Subject: [PATCH 15/63] add actual target classes, add infrastructure for selecting stream targets and finalizing the output data array --- py/desitarget/data/targetmask.yaml | 9 ++ py/desitarget/streams/cuts.py | 131 ++++++++++++++++++++++++++++- py/desitarget/streams/targets.py | 122 +++++++++++++++++++++++++++ py/desitarget/streams/utilities.py | 9 +- 4 files changed, 264 insertions(+), 7 deletions(-) create mode 100644 py/desitarget/streams/targets.py diff --git a/py/desitarget/data/targetmask.yaml b/py/desitarget/data/targetmask.yaml index 529c0ee3..b77a7e97 100644 --- a/py/desitarget/data/targetmask.yaml +++ b/py/desitarget/data/targetmask.yaml @@ -174,6 +174,9 @@ scnd_mask: - [DWF_BRIGHT_LO, 52, "See $SCND_DIR/docs/DWF_BRIGHT_LO.txt", {obsconditions: BRIGHT, filename: 'DWF_BRIGHT_LO', flavor: 'SPARE', updatemws: True, downsample: 1}] - [DWF_DARK_HI, 53, "See $SCND_DIR/docs/DWF_DARK_HI.txt", {obsconditions: DARK, filename: 'DWF_DARK_HI', flavor: 'SPARE', updatemws: True, downsample: 1}] - [DWF_DARK_LO, 54, "See $SCND_DIR/docs/DWF_DARK_LO.txt", {obsconditions: DARK, filename: 'DWF_DARK_LO', flavor: 'SPARE', updatemws: True, downsample: 1}] + - [GD1_BRIGHT_PM, 55, "Bright targets with Gaia in GD1 stream", {obsconditions: BRIGHT, filename: None, flavor: 'SPARE', updatemws: True, downsample: 1}] + - [GD1_FAINT_NO_PM, 56, "Faint targets without Gaia in GD1 stream", {obsconditions: BRIGHT, filename: None, flavor: 'SPARE', updatemws: True, downsample: 1}] + - [GD1_FILLER, 57, "Filler targets in GD1 stream", {obsconditions: BRIGHT, filename: None, flavor: 'SPARE', updatemws: True, downsample: 1}] # ADM reserve 59-62 in scnd_mask for Targets of Opportunity in both SV and the Main Survey. @@ -414,6 +417,9 @@ priorities: BRIGHT_TOO_HIP: {UNOBS: 9500, MORE_ZGOOD: 2, MORE_ZWARN: 2, DONE: 2, OBS: 1, DONOTOBSERVE: 0, MORE_MIDZQSO: 0} DARK_TOO_LOP: SAME_AS_BRIGHT_TOO_LOP DARK_TOO_HIP: SAME_AS_BRIGHT_TOO_HIP + GD1_BRIGHT_PM: {UNOBS: 1520, MORE_ZGOOD: 2, MORE_ZWARN: 2, DONE: 2, OBS: 1, DONOTOBSERVE: 0, MORE_MIDZQSO: 0} + GD1_FAINT_NO_PM: {UNOBS: 1420, MORE_ZGOOD: 2, MORE_ZWARN: 2, DONE: 2, OBS: 1, DONOTOBSERVE: 0, MORE_MIDZQSO: 0} + GD1_FILLER: {UNOBS: 100, MORE_ZGOOD: 2, MORE_ZWARN: 2, DONE: 2, OBS: 1, DONOTOBSERVE: 0, MORE_MIDZQSO: 0} # ADM INITIAL number of observations (NUMOBS_INIT) for each target bit # ADM SAME_AS_XXX means to use the NUMOBS_INIT for bitname XXX @@ -565,3 +571,6 @@ numobs: DWF_BRIGHT_LO: 10 DWF_DARK_HI: 10 DWF_DARK_LO: 10 + GD1_BRIGHT_PM: 1 + GD1_FAINT_NO_PM: 1 + GD1_FILLER: 1 diff --git a/py/desitarget/streams/cuts.py b/py/desitarget/streams/cuts.py index 9b59d313..2dcf19bd 100644 --- a/py/desitarget/streams/cuts.py +++ b/py/desitarget/streams/cuts.py @@ -1,7 +1,6 @@ """ - -desitarget.streamcuts -===================== +desitarget.streams.cuts +======================= Target selection cuts for the DESI MWS Stellar Stream programs. @@ -19,7 +18,9 @@ from desitarget.cuts import _psflike from desitarget.streams.utilities import sphere_rotate, correct_pm, rotate_pm, \ betw, pm12_sel_func, plx_sel_func, get_CMD_interpolator, stream_distance, \ - get_stream_parameters + get_stream_parameters, read_data +from desitarget.targets import resolve +from desitarget.streams.targets import finalize # ADM set up the DESI default logger from desiutil.log import get_logger @@ -184,3 +185,125 @@ def is_in_GD1(objs): log.error(msg) return bright_pm, faint_no_pm, filler + + +def set_target_bits(objs, stream_names=["GD1"]): + """Select stream targets, returning target mask arrays. + + Parameters + ---------- + objects : :class:`~numpy.ndarray` + numpy structured array with UPPERCASE columns needed for + stream target selection. See, e.g., + :func:`~desitarget.stream.cuts.is_in_GD1` for column names. + stream_names : :class:`list` + A list of stream names to process. Defaults to all streams. + + Returns + ------- + :class:`~numpy.ndarray` + (desi_target, bgs_target, mws_target, scnd_target) where each + element is an array of target selection bitmasks for each object. + + Notes + ----- + - See ../data/targetmask.yaml for the definition of each bit. + """ + from desitarget.targetmask import desi_mask, scnd_mask + + # ADM set up a zerod scnd_target array to |= with later. + scnd_target = np.zeros_like(objs["RA"], dtype='int64') + + # ADM might be able to make this more general by putting the + # ADM bit names in the data/yaml file and using globals() + # ADM to recover the is_in() functions. + + if "GD1" in stream_names: + gd1_bright_pm, gd1_faint_no_pm, gd1_filler = is_in_GD1(objs) + + scnd_target |= gd1_bright_pm * scnd_mask.GD1_BRIGHT_PM + scnd_target |= gd1_faint_no_pm * scnd_mask.GD1_FAINT_NO_PM + scnd_target |= gd1_filler * scnd_mask.GD1_FILLER + + # ADM tell DESI_TARGET where SCND_ANY was updated. + desi_target = (scnd_target != 0) * desi_mask.SCND_ANY + + # ADM set BGS_TARGET and MWS_TARGET to zeros. + bgs_target = np.zeros_like(scnd_target) + mws_target = np.zeros_like(scnd_target) + + return desi_target, bgs_target, mws_target, scnd_target + + +def select_targets(swdir, stream_names=["GD1"], readcache=True): + """Process files from an input directory to select targets. + + Parameters + ---------- + swdir : :class:`str` + Root directory of Legacy Surveys sweep files for a given data + release for ONE of EITHER north or south, e.g. + "/global/cfs/cdirs/cosmo/data/legacysurvey/dr9/south/sweep/9.0". + stream_names : :class:`list` + A list of stream names to process. Defaults to all streams. + readcache : :class:`bool` + If ``True`` read all data from previously made cache files, + in cases where such files exist. If ``False`` don't read + from caches AND OVERWRITE any cached files, if they exist. Cache + files are named $TARG_DIR/streamcache/streamname-drX-cache.fits, + where streamname is the lower-case name from `stream_names` and + drX is the Legacy Surveys Data Release (parsed from `swdir`). + + Returns + ------- + :class:`~numpy.ndarray` + Targets in the input `swdir` which pass the cuts with added + targeting columns such as ``TARGETID``, and ``DESI_TARGET``, + ``BGS_TARGET``, ``MWS_TARGET``, ``SCND_TARGET`` (i.e. target + selection bitmasks). + """ + # ADM loop over streams and read in the data. + # ADM eventually, for multiple streams, we would likely switch this + # ADM to reading in each sweep file and parallelizing across files. + allobjs = [] + for stream_name in stream_names: + # ADM read in the data. + strm = get_stream_parameters(stream_name) + # ADM the parameters that define the coordinates of the stream. + rapol, decpol, ra_ref = strm["RAPOL"], strm["DECPOL"], strm["RA_REF"] + # ADM the parameters that define the extent of the stream. + mind, maxd = strm["MIND"], strm["MAXD"] + # ADM read in the data. + objs = read_data(swdir, rapol, decpol, ra_ref, mind, maxd, stream_name, + readcache=readcache) + allobjs.append(objs) + objects = np.concatenate(allobjs) + + # ADM process the targets. + desi_target, bgs_target, mws_target, scnd_target = set_target_bits( + objs, stream_names=[stream_name]) + + # ADM finalize the targets. + # ADM anything with DESI_TARGET !=0 is truly a target. + ii = (desi_target != 0) + objects = objects[ii] + desi_target = desi_target[ii] + bgs_target = bgs_target[ii] + mws_target = mws_target[ii] + scnd_target = scnd_target[ii] + + # ADM add TARGETID and targeting bitmask columns. + targets = finalize(objects, desi_target, bgs_target, mws_target, scnd_target) + + # ADM resolve any duplicates between imaging data releases. + targets = resolve(targets) + + # ADM we'll definitely need to update the read_data loop if we ever + # ADM have overlapping targets in overlapping streams. + if len(np.unique(targets["TARGETID"])) != len(targets): + msg = ("Targets are not unique. The code needs updated to read in the " + "sweep files one-by-one (as in desitarget.cuts.select_targets()) " + "rather than caching each individual stream") + log.error(msg) + + return targets diff --git a/py/desitarget/streams/targets.py b/py/desitarget/streams/targets.py new file mode 100644 index 00000000..f0fffe41 --- /dev/null +++ b/py/desitarget/streams/targets.py @@ -0,0 +1,122 @@ +""" +desitarget.streams.targets +=========================== + +Manipulate/add target bitmasks/priorities/numbers of observations for +stream targets. +""" +import numpy as np +import numpy.lib.recfunctions as rfn + +from desitarget.targets import initial_priority_numobs, set_obsconditions, \ + encode_targetid + +# ADM set up the DESI default logger. +from desiutil.log import get_logger +log = get_logger() + + +def finalize(targets, desi_target, bgs_target, mws_target, scnd_target): + """Return new targets array with added/renamed columns + + Parameters + ---------- + targets : :class:`~numpy.ndarray` + numpy structured array of targets. + desi_target : :class:`~numpy.ndarray` + 1D array of target selection bit flags. + bgs_target : :class:`~numpy.ndarray` + 1D array of target selection bit flags. + mws_target : :class:`~numpy.ndarray` + 1D array of target selection bit flags. + scnd_target : :class:`~numpy.ndarray` + 1D array of target selection bit flags. + + Returns + ------- + :class:`~numpy.ndarray` + new targets structured array with the following additions: + * renaming OBJID -> BRICK_OBJID (it is only unique within a brick). + * renaming TYPE -> MORPHTYPE (used downstream in other contexts). + * Adding new columns: + - TARGETID: unique ID across all bricks or Gaia files. + - DESI_TARGET: dark time survey target selection flags. + - MWS_TARGET: bright time MWS target selection flags. + - BGS_TARGET: bright time BGS target selection flags. + - SCND_TARGET: stream secondary target selection flags. + - PRIORITY_INIT: initial priority for observing target. + - SUBPRIORITY: a placeholder column that is set to zero. + - NUMOBS_INIT: initial number of observations for target. + - OBSCONDITIONS: bitmask of observation conditions. + + Notes + ----- + - SUBPRIORITY is the only column that isn't populated. This is + because it's easier to populate it in a reproducible fashion + when collecting targets rather than on a per-brick basis + when this function is called. It's set to all zeros. + - NUMOBS_INIT and PRIORITY_INIT are split into DARK/BRIGHT/BACKUP + versions, as these surveys are effectively separate. + """ + # ADM some straightforward checks that inputs are the same length. + ntargets = len(targets) + assert ntargets == len(desi_target) + assert ntargets == len(bgs_target) + assert ntargets == len(mws_target) + assert ntargets == len(scnd_target) + + # - OBJID in tractor files is only unique within the brick; rename and + # - create a new unique TARGETID + targets = rfn.rename_fields(targets, + {'OBJID': 'BRICK_OBJID', 'TYPE': 'MORPHTYPE'}) + + + targetid = encode_targetid(objid=targets['BRICK_OBJID'], + brickid=targets['BRICKID'], + release=targets['RELEASE']) + + + nodata = np.zeros(ntargets, dtype='int')-1 + subpriority = np.zeros(ntargets, dtype='float') + + # ADM the columns to write out and their values and formats. + cols = ["TARGETID", "DESI_TARGET", "BGS_TARGET", "MWS_TARGET", "SCND_TARGET", + "SUBPRIORITY", "OBSCONDITIONS"] + vals = [targetid, desi_target, bgs_target, mws_target, scnd_target, + subpriority, nodata] + forms = [">i8", ">i8", ">i8", ">i8", ">i8", ">f8", ">i8"] + + # ADM set the initial PRIORITY and NUMOBS. + # ADM populate bright/dark/backup separately. + ender = ["_DARK", "_BRIGHT", "_BACKUP"] + obscon = ["DARK|GRAY", "BRIGHT", "BACKUP"] + for edr, oc in zip(ender, obscon): + cols += ["{}_INIT{}".format(pn, edr) for pn in ["PRIORITY", "NUMOBS"]] + vals += [nodata, nodata] + forms += ['>i8', '>i8'] + + # ADM write the output array. + newdt = [dt for dt in zip(cols, forms)] + done = np.array(np.zeros(len(targets)), dtype=targets.dtype.descr+newdt) + for col in targets.dtype.names: + done[col] = targets[col] + for col, val in zip(cols, vals): + done[col] = val + + # ADM add PRIORITY/NUMOBS columns. + for edr, oc in zip(ender, obscon): + pc, nc = "PRIORITY_INIT"+edr, "NUMOBS_INIT"+edr + done[pc], done[nc] = initial_priority_numobs(done, obscon=oc, scnd=True) + + # ADM set the OBSCONDITIONS. + done["OBSCONDITIONS"] = set_obsconditions(done, scnd=True) + + # ADM some final checks that the targets conform to expectations... + # ADM check that each target has a unique ID. + if len(done["TARGETID"]) != len(np.unique(done["TARGETID"])): + msg = ("Targets are not unique. The code might need updated to read the " + "sweep files one-by-one (as in desitarget.cuts.select_targets()) " + "rather than caching each individual stream") + log.critical(msg) + + return done diff --git a/py/desitarget/streams/utilities.py b/py/desitarget/streams/utilities.py index 9061d890..41521b57 100644 --- a/py/desitarget/streams/utilities.py +++ b/py/desitarget/streams/utilities.py @@ -1,6 +1,6 @@ """ -desitarget.streamutilties -========================= +desitarget.streams.utilties +=========================== Utilities for the DESI MWS Stellar Stream programs. @@ -447,7 +447,10 @@ def plx_sel_func(dist, D, mult, plx_sys=0.05): if 'PARALLAX_ERROR' in D.dtype.names: parallax_error = D['PARALLAX_ERROR'] elif 'PARALLAX_IVAR' in D.dtype.names: - parallax_error = 1./np.sqrt(D['PARALLAX_IVAR']) + # ADM guard against dividing by zero. + parallax_error = np.zeros_like(D["PARALLAX_IVAR"]) + 1e8 + ii = D['PARALLAX_IVAR'] != 0 + parallax_error[ii] = 1./np.sqrt(D[ii]['PARALLAX_IVAR']) else: msg = "Either PARALLAX_ERROR or PARALLAX_IVAR must be passed!" log.error(msg) From 7576594da1a6c157a6a466051bf792b4983b93e3 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Mon, 4 Mar 2024 11:30:42 -0800 Subject: [PATCH 16/63] switch reading/writing functions to their own I/O module --- py/desitarget/streams/cuts.py | 53 ++++--- py/desitarget/streams/io.py | 228 +++++++++++++++++++++++++++++ py/desitarget/streams/targets.py | 2 +- py/desitarget/streams/utilities.py | 208 -------------------------- 4 files changed, 262 insertions(+), 229 deletions(-) create mode 100644 py/desitarget/streams/io.py diff --git a/py/desitarget/streams/cuts.py b/py/desitarget/streams/cuts.py index 2dcf19bd..885f7435 100644 --- a/py/desitarget/streams/cuts.py +++ b/py/desitarget/streams/cuts.py @@ -18,15 +18,16 @@ from desitarget.cuts import _psflike from desitarget.streams.utilities import sphere_rotate, correct_pm, rotate_pm, \ betw, pm12_sel_func, plx_sel_func, get_CMD_interpolator, stream_distance, \ - get_stream_parameters, read_data + get_stream_parameters +from desitarget.streams.io import read_data_per_stream from desitarget.targets import resolve from desitarget.streams.targets import finalize -# ADM set up the DESI default logger +# ADM set up the DESI default logger. from desiutil.log import get_logger log = get_logger() -# ADM start the clock +# ADM start the clock. start = time() @@ -235,7 +236,8 @@ def set_target_bits(objs, stream_names=["GD1"]): return desi_target, bgs_target, mws_target, scnd_target -def select_targets(swdir, stream_names=["GD1"], readcache=True): +def select_targets(swdir, stream_names=["GD1"], readperstream=True, + readcache=True): """Process files from an input directory to select targets. Parameters @@ -246,7 +248,12 @@ def select_targets(swdir, stream_names=["GD1"], readcache=True): "/global/cfs/cdirs/cosmo/data/legacysurvey/dr9/south/sweep/9.0". stream_names : :class:`list` A list of stream names to process. Defaults to all streams. - readcache : :class:`bool` + readperstream : :class:`bool`, optional, defaults to ``True`` + When set, read each stream's data individually instead of looping + through all possible sweeps files. This is likely quickest and + most useful when working with a single stream. For multiple + streams it may cause issues when duplicate targets are selected. + readcache : :class:`bool`, optional, defaults to ``True`` If ``True`` read all data from previously made cache files, in cases where such files exist. If ``False`` don't read from caches AND OVERWRITE any cached files, if they exist. Cache @@ -262,22 +269,28 @@ def select_targets(swdir, stream_names=["GD1"], readcache=True): ``BGS_TARGET``, ``MWS_TARGET``, ``SCND_TARGET`` (i.e. target selection bitmasks). """ - # ADM loop over streams and read in the data. - # ADM eventually, for multiple streams, we would likely switch this - # ADM to reading in each sweep file and parallelizing across files. - allobjs = [] - for stream_name in stream_names: - # ADM read in the data. - strm = get_stream_parameters(stream_name) - # ADM the parameters that define the coordinates of the stream. - rapol, decpol, ra_ref = strm["RAPOL"], strm["DECPOL"], strm["RA_REF"] - # ADM the parameters that define the extent of the stream. - mind, maxd = strm["MIND"], strm["MAXD"] - # ADM read in the data. - objs = read_data(swdir, rapol, decpol, ra_ref, mind, maxd, stream_name, - readcache=readcache) + if readperstream: + # ADM loop over streams and read in the data per-stream. + # ADM eventually, for multiple streams, we would likely switch + # ADM to read in each sweep file and parallelizing across files. + allobjs = [] + for stream_name in stream_names: + # ADM read in the data. + strm = get_stream_parameters(stream_name) + # ADM the parameters that define the coordinates of the stream. + rapol, decpol, ra_ref = strm["RAPOL"], strm["DECPOL"], strm["RA_REF"] + # ADM the parameters that define the extent of the stream. + mind, maxd = strm["MIND"], strm["MAXD"] + # ADM read in the data. + objs = read_data_per_stream(swdir, rapol, decpol, ra_ref, mind, maxd, + stream_name, readcache=readcache) allobjs.append(objs) - objects = np.concatenate(allobjs) + objects = np.concatenate(allobjs) + else: + # ADM --TODO-- write loop across sweeps instead of streams. + msg = ("readperstream must be True until we implement looping " + "over sweeps instead of streams") + log.error(msg) # ADM process the targets. desi_target, bgs_target, mws_target, scnd_target = set_target_bits( diff --git a/py/desitarget/streams/io.py b/py/desitarget/streams/io.py new file mode 100644 index 00000000..2dd495a7 --- /dev/null +++ b/py/desitarget/streams/io.py @@ -0,0 +1,228 @@ +""" +desitarget.streams.io +===================== + +Reading/writing data for the MWS Stellar Stream programs. +""" +from time import time + +import os +import fitsio +import numpy as np + +from desitarget import io +from desitarget.geomask import pixarea2nside, add_hp_neighbors, sweep_files_touch_hp +from desitarget.gaiamatch import match_gaia_to_primary +from desitarget.targets import resolve + + +# ADM set up the DESI default logger. +from desiutil.log import get_logger +log = get_logger() + +# ADM start the clock. +start = time() + +# ADM the standard data model for working with streams. +streamcols = np.array([], dtype=[ + ('RELEASE', '>i2'), ('BRICKID', '>i4'), ('TYPE', 'S4'), + ('OBJID', '>i4'), ('RA', '>f8'), ('DEC', '>f8'), ('EBV', '>f4'), + ('FLUX_G', '>f4'), ('FLUX_R', '>f4'), ('FLUX_Z', '>f4'), + ('REF_EPOCH', '>f4'), ('PARALLAX', '>f4'), ('PARALLAX_IVAR', '>f4'), + ('PMRA', '>f4'), ('PMRA_IVAR', '>f4'), + ('PMDEC', '>f4'), ('PMDEC_IVAR', '>f4'), + ('ASTROMETRIC_PARAMS_SOLVED', '>i1'), ('NU_EFF_USED_IN_ASTROMETRY', '>f4'), + ('PSEUDOCOLOUR', '>f4'), ('PHOT_G_MEAN_MAG', '>f4'), ('ECL_LAT', '>f8') +]) + +# ADM the Gaia Data Release for matching throughout this module. +gaiadr = "dr3" + + +def read_data_per_stream(swdir, rapol, decpol, ra_ref, mind, maxd, stream_name, + readcache=True, addnors=True, test=False): + """Assemble the data needed for a particular stream program. + + Parameters + ---------- + swdir : :class:`str` + Root directory of Legacy Surveys sweep files for a given data + release for ONE of EITHER north or south, e.g. + "/global/cfs/cdirs/cosmo/data/legacysurvey/dr9/south/sweep/9.0". + rapol, decpol : :class:`float` + Pole in the stream coordinate system in DEGREES. + ra_ref : :class:`float` + Zero latitude in the stream coordinate system in DEGREES. + mind, maxd : :class:`float` or `int` + Minimum and maximum angular distance from the pole of the stream + coordinate system to search for members in DEGREES. + stream_name : :class:`str` + Name of a stream. Used to make the cached filename, e.g. "GD1". + readcache : :class:`bool` + If ``True`` read from a previously constructed and cached file + automatically, IF such a file exists. If ``False`` don't read + from the cache AND OVERWRITE the cached file, if it exists. The + cached file is $TARG_DIR/streamcache/streamname-drX-cache.fits, + where streamname is the lower-case passed `stream_name` and drX + is the Legacy Surveys Data Release (parsed from `swdir`). + addnors : :class:`bool` + If ``True`` then if `swdir` contains "north" add sweep files from + the south by substituting "south" in place of "north" (and vice + versa, i.e. if `swdir` contains "south" add sweep files from the + north by substituting "north" in place of "south"). + test : :class:`bool` + Read a subset of the data for testing purposes. + + Returns + ------- + :class:`array_like` or `boolean` + ``True`` for stream members. + + Notes + ----- + - Example values for, e.g., GD1: + swdir = "/global/cfs/cdirs/cosmo/data/legacysurvey/dr9/south/sweep/9.0" + rapol, decpol, ra_ref = 34.5987, 29.7331, 200 + mind, maxd = 80, 100 + - The $TARG_DIR environment variable must be set to read/write from + a cache. If $TARG_DIR is not set, caching is completely ignored. + - This is useful for a single stream. The :func:`~read_data` function + is likely a better choice for looping over the entire LS sweeps + data when targeting multiple streams. + """ + # ADM check whether $TARG_DIR exists. If it does, agree to read from + # ADM and write to the cache. + writecache = True + targdir = os.environ.get("TARG_DIR") + if targdir is None: + msg = "Set $TARG_DIR environment variable to use the cache!" + log.info(msg) + readcache = False + writecache = False + else: + # ADM retrieve the data release from the passed sweep directory. + dr = [i for i in swdir.split(os.sep) if "dr" in i] + # ADM fail if this doesn't look like a standard sweep directory. + if len(dr) != 1: + msg = 'swdir not parsed: should include a construction like ' + msg += '"dr9" or "dr10"' + raise ValueError(msg) + cachefile = os.path.join(os.getenv("TARG_DIR"), "streamcache", + f"{stream_name.lower()}-{dr[0]}-cache.fits") + + # ADM if we have a cache, read it if requested and return the data. + if readcache: + if os.path.isfile(cachefile): + objs = fitsio.read(cachefile, ext="STREAMCACHE") + msg = f"Read {len(objs)} objects from {cachefile} cache file" + log.info(msg) + return objs + else: + msg = f"{cachefile} cache file doesn't exist. " + msg += f"Proceeding as if readcache=False" + log.info(msg) + + # ADM read in the sweep files. + infiles = io.list_sweepfiles(swdir) + + # ADM read both the north and south directories, if requested. + if addnors: + if "south" in swdir: + infiles2 = swdir.replace("south", "north") + elif "north" in swdir: + infiles2 = swdir.replace("north", "south") + else: + msg = "addnors passed but swdir does not contain north or south!" + raise ValueError(msg) + infiles += io.list_sweepfiles(infiles2) + + # ADM calculate nside for HEALPixel of approximately 1o to limit + # ADM number of sweeps files that need to be read. + nside = pixarea2nside(1) + + # ADM determine RA, Dec of all HEALPixels at this nside. + allpix = np.arange(hp.nside2npix(nside)) + theta, phi = hp.pix2ang(nside, allpix, nest=True) + ra, dec = np.degrees(phi), 90-np.degrees(theta) + + # ADM only retain HEALPixels in the stream, based on mind and maxd. + cpix = acoo.SkyCoord(ra*auni.degree, dec*auni.degree) + cstream = acoo.SkyCoord(rapol*auni.degree, decpol*auni.degree) + sep = cpix.separation(cstream) + ii = betw(sep.value, mind, maxd) + pixlist = allpix[ii] + + # ADM pad with neighboring pixels to ensure stream is fully covered. + newpixlist = add_hp_neighbors(nside, pixlist) + + # ADM determine which sweep files touch the relevant HEALPixels. + filesperpixel, _, _ = sweep_files_touch_hp(nside, pixlist, infiles) + infiles = list(np.unique(np.hstack([filesperpixel[pix] for pix in pixlist]))) + + # ADM read a subset of the data for testing purposes, if requested. + if test: + msg = "Limiting data to first 20 files for testing purposes" + log.info(msg) + infiles = infiles[:20] + + # ADM loop through the sweep files and limit to objects in the stream. + allobjs = [] + for i, filename in enumerate(infiles): + objs = io.read_tractor(filename) + cobjs = acoo.SkyCoord(objs["RA"]*auni.degree, objs["DEC"]*auni.degree) + sep = cobjs.separation(cstream) + + # ADM only retain objects in the stream... + ii = betw(sep.value, mind, maxd) + + # ADM ...that aren't very faint (> 22.5 mag in r). + ii &= objs["FLUX_R"] > 1 + # ADM Also guard against negative fluxes in g/r. + ii &= objs["FLUX_G"] > 0. + ii &= objs["FLUX_Z"] > 0. + + objs = objs[ii] + + # ADM limit to northern objects in northern imaging and southern + # ADM objects in southern imaging. + LSobjs = resolve(objs) + + # ADM catch the case where there are no objects meeting the cuts. + if len(LSobjs) > 0: + gaiaobjs = match_gaia_to_primary(LSobjs, matchrad=1., dr=gaiadr) + else: + gaiaobjs = LSobjs + + # ADM a (probably unnecessary) sanity check. + assert(len(gaiaobjs) == len(LSobjs)) + + # ADM only retain critical columns from the global data model. + data = np.zeros(len(LSobjs), dtype=streamcols.dtype) + # ADM for both Gaia and Legacy Surveys, overwriting with Gaia. + for objs in LSobjs, gaiaobjs: + sharedcols = set(data.dtype.names).intersection(set(objs.dtype.names)) + for col in sharedcols: + data[col] = objs[col] + + # ADM retain the data from this part of the loop. + allobjs.append(data) + if i % 5 == 4: + log.info(f"Ran {i+1}/{len(infiles)} files...t={time()-start:.1f}s") + + # ADM assemble all of the relevant objects. + allobjs = np.concatenate(allobjs) + log.info(f"Found {len(allobjs)} total objects...t={time()-start:.1f}s") + + # ADM if cache was passed and $TARG_DIR was set then write the data. + if writecache: + # ADM if the file doesn't exist we may need to make the directory. + log.info(f"Writing cache to {cachefile}...t={time()-start:.1f}s") + os.makedirs(os.path.dirname(cachefile), exist_ok=True) + # ADM at least add the Gaia DR used to the header. + hdr = fitsio.FITSHDR() + hdr.add_record(dict(name="GAIADR", value=gaiadr, + comment="GAIA Data Release matched to")) + io.write_with_units(cachefile, allobjs, + header=hdr, extname="STREAMCACHE") + + return allobjs diff --git a/py/desitarget/streams/targets.py b/py/desitarget/streams/targets.py index f0fffe41..7dfc345c 100644 --- a/py/desitarget/streams/targets.py +++ b/py/desitarget/streams/targets.py @@ -1,6 +1,6 @@ """ desitarget.streams.targets -=========================== +========================== Manipulate/add target bitmasks/priorities/numbers of observations for stream targets. diff --git a/py/desitarget/streams/utilities.py b/py/desitarget/streams/utilities.py index 41521b57..0b604ca1 100644 --- a/py/desitarget/streams/utilities.py +++ b/py/desitarget/streams/utilities.py @@ -10,9 +10,7 @@ """ import yaml import os -import fitsio import numpy as np -import healpy as hp import astropy.coordinates as acoo import astropy.units as auni from pkg_resources import resource_filename @@ -20,11 +18,6 @@ from time import time from zero_point import zero_point as gaia_zpt -from desitarget import io -from desitarget.geomask import pixarea2nside, add_hp_neighbors, sweep_files_touch_hp -from desitarget.gaiamatch import match_gaia_to_primary -from desitarget.targets import resolve - # ADM set up the DESI default logger. from desiutil.log import get_logger log = get_logger() @@ -43,18 +36,6 @@ kms = auni.km / auni.s masyr = auni.mas / auni.year -# ADM the standard data model for working with streams. -streamcols = np.array([], dtype=[ - ('RELEASE', '>i2'), ('BRICKID', '>i4'), ('TYPE', 'S4'), - ('OBJID', '>i4'), ('RA', '>f8'), ('DEC', '>f8'), ('EBV', '>f4'), - ('FLUX_G', '>f4'), ('FLUX_R', '>f4'), ('FLUX_Z', '>f4'), - ('REF_EPOCH', '>f4'), ('PARALLAX', '>f4'), ('PARALLAX_IVAR', '>f4'), - ('PMRA', '>f4'), ('PMRA_IVAR', '>f4'), - ('PMDEC', '>f4'), ('PMDEC_IVAR', '>f4'), - ('ASTROMETRIC_PARAMS_SOLVED', '>i1'), ('NU_EFF_USED_IN_ASTROMETRY', '>f4'), - ('PSEUDOCOLOUR', '>f4'), ('PHOT_G_MEAN_MAG', '>f4'), ('ECL_LAT', '>f8') -]) - def cosd(x): """Return cos(x) for an angle x in degrees. @@ -485,192 +466,3 @@ def stream_distance(fi1, stream_name): msg = f"stream name {stream_name} not recognized" log.error(msg) - -def read_data(swdir, rapol, decpol, ra_ref, mind, maxd, stream_name, - readcache=True, addnors=True, test=False): - """Assemble the data needed for a particular stream program. - - Example values for GD1: - swdir = "/global/cfs/cdirs/cosmo/data/legacysurvey/dr9/south/sweep/9.0" - rapol, decpol, ra_ref = 34.5987, 29.7331, 200 - mind, maxd = 80, 100 - - Parameters - ---------- - swdir : :class:`str` - Root directory of Legacy Surveys sweep files for a given data - release for ONE of EITHER north or south, e.g. - "/global/cfs/cdirs/cosmo/data/legacysurvey/dr9/south/sweep/9.0". - rapol, decpol : :class:`float` - Pole in the stream coordinate system in DEGREES. - ra_ref : :class:`float` - Zero latitude in the stream coordinate system in DEGREES. - mind, maxd : :class:`float` or `int` - Minimum and maximum angular distance from the pole of the stream - coordinate system to search for members in DEGREES. - stream_name : :class:`str` - Name of a stream. Used to make the cached filename, e.g. "GD1". - readcache : :class:`bool` - If ``True`` read from a previously constructed and cached file - automatically, IF such a file exists. If ``False`` don't read - from the cache AND OVERWRITE the cached file, if it exists. The - cached file is $TARG_DIR/streamcache/streamname-drX-cache.fits, - where streamname is the lower-case passed `stream_name` and drX - is the Legacy Surveys Data Release (parsed from `swdir`). - addnors : :class:`bool` - If ``True`` then if `swdir` contains "north" add sweep files from - the south by substituting "south" in place of "north" (and vice - versa, i.e. if `swdir` contains "south" add sweep files from the - north by substituting "north" in place of "south"). - test : :class:`bool` - Read a subset of the data for testing purposes. - - Returns - ------- - :class:`array_like` or `boolean` - ``True`` for stream members. - - Notes - ----- - - The $TARG_DIR environment variable must be set to read/write from - a cache. If $TARG_DIR is not set, caching is completely ignored. - """ - # ADM The Gaia DR to which to match. - gaiadr = "dr3" - - # ADM check whether $TARG_DIR exists. If it does, agree to read from - # ADM and write to the cache. - writecache = True - targdir = os.environ.get("TARG_DIR") - if targdir is None: - msg = "Set $TARG_DIR environment variable to use the cache!" - log.info(msg) - readcache = False - writecache = False - else: - # ADM retrieve the data release from the passed sweep directory. - dr = [i for i in swdir.split(os.sep) if "dr" in i] - # ADM fail if this doesn't look like a standard sweep directory. - if len(dr) != 1: - msg = 'swdir not parsed: should include a construction like ' - msg += '"dr9" or "dr10"' - raise ValueError(msg) - cachefile = os.path.join(os.getenv("TARG_DIR"), "streamcache", - f"{stream_name.lower()}-{dr[0]}-cache.fits") - - # ADM if we have a cache, read it if requested and return the data. - if readcache: - if os.path.isfile(cachefile): - objs = fitsio.read(cachefile, ext="STREAMCACHE") - msg = f"Read {len(objs)} objects from {cachefile} cache file" - log.info(msg) - return objs - else: - msg = f"{cachefile} cache file doesn't exist. " - msg += f"Proceeding as if readcache=False" - log.info(msg) - - # ADM read in the sweep files. - infiles = io.list_sweepfiles(swdir) - - # ADM read both the north and south directories, if requested. - if addnors: - if "south" in swdir: - infiles2 = swdir.replace("south", "north") - elif "north" in swdir: - infiles2 = swdir.replace("north", "south") - else: - msg = "addnors passed but swdir does not contain north or south!" - raise ValueError(msg) - infiles += io.list_sweepfiles(infiles2) - - # ADM calculate nside for HEALPixel of approximately 1o to limit - # ADM number of sweeps files that need to be read. - nside = pixarea2nside(1) - - # ADM determine RA, Dec of all HEALPixels at this nside. - allpix = np.arange(hp.nside2npix(nside)) - theta, phi = hp.pix2ang(nside, allpix, nest=True) - ra, dec = np.degrees(phi), 90-np.degrees(theta) - - # ADM only retain HEALPixels in the stream, based on mind and maxd. - cpix = acoo.SkyCoord(ra*auni.degree, dec*auni.degree) - cstream = acoo.SkyCoord(rapol*auni.degree, decpol*auni.degree) - sep = cpix.separation(cstream) - ii = betw(sep.value, mind, maxd) - pixlist = allpix[ii] - - # ADM pad with neighboring pixels to ensure stream is fully covered. - newpixlist = add_hp_neighbors(nside, pixlist) - - # ADM determine which sweep files touch the relevant HEALPixels. - filesperpixel, _, _ = sweep_files_touch_hp(nside, pixlist, infiles) - infiles = list(np.unique(np.hstack([filesperpixel[pix] for pix in pixlist]))) - - # ADM read a subset of the data for testing purposes, if requested. - if test: - msg = "Limiting data to first 20 files for testing purposes" - log.info(msg) - infiles = infiles[:20] - - # ADM loop through the sweep files and limit to objects in the stream. - allobjs = [] - for i, filename in enumerate(infiles): - objs = io.read_tractor(filename) - cobjs = acoo.SkyCoord(objs["RA"]*auni.degree, objs["DEC"]*auni.degree) - sep = cobjs.separation(cstream) - - # ADM only retain objects in the stream... - ii = betw(sep.value, mind, maxd) - - # ADM ...that aren't very faint (> 22.5 mag in r). - ii &= objs["FLUX_R"] > 1 - # ADM Also guard against negative fluxes in g/r. - ii &= objs["FLUX_G"] > 0. - ii &= objs["FLUX_Z"] > 0. - - objs = objs[ii] - - # ADM limit to northern objects in northern imaging and southern - # ADM objects in southern imaging. - LSobjs = resolve(objs) - - # ADM catch the case where there are no objects meeting the cuts. - if len(LSobjs) > 0: - gaiaobjs = match_gaia_to_primary(LSobjs, matchrad=1., dr=gaiadr) - else: - gaiaobjs = LSobjs - - # ADM a (probably unnecessary) sanity check. - assert(len(gaiaobjs) == len(LSobjs)) - - # ADM only retain critical columns from the global data model. - data = np.zeros(len(LSobjs), dtype=streamcols.dtype) - # ADM for both Gaia and Legacy Surveys, overwriting with Gaia. - for objs in LSobjs, gaiaobjs: - sharedcols = set(data.dtype.names).intersection(set(objs.dtype.names)) - for col in sharedcols: - data[col] = objs[col] - - # ADM retain the data from this part of the loop. - allobjs.append(data) - if i % 5 == 4: - log.info(f"Ran {i+1}/{len(infiles)} files...t={time()-start:.1f}s") - - # ADM assemble all of the relevant objects. - allobjs = np.concatenate(allobjs) - log.info(f"Found {len(allobjs)} total objects...t={time()-start:.1f}s") - - # ADM if cache was passed and $TARG_DIR was set then write the data. - if writecache: - # ADM if the file doesn't exist we may need to make the directory. - log.info(f"Writing cache to {cachefile}...t={time()-start:.1f}s") - os.makedirs(os.path.dirname(cachefile), exist_ok=True) - # ADM at least add the Gaia DR used to the header. - hdr = fitsio.FITSHDR() - hdr.add_record(dict(name="GAIADR", value=gaiadr, - comment="GAIA Data Release matched to")) - io.write_with_units(cachefile, allobjs, - header=hdr, extname="STREAMCACHE") - - return allobjs From 01944ae1d0af2d2fb4d0c9eb7afc2100fedd47db Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Mon, 4 Mar 2024 12:05:06 -0800 Subject: [PATCH 17/63] update units in desitarget files to be FITS compliant --- py/desitarget/data/units.yaml | 64 ++++++++++++++++---------------- py/desitarget/test/test_units.py | 8 ++-- 2 files changed, 36 insertions(+), 36 deletions(-) diff --git a/py/desitarget/data/units.yaml b/py/desitarget/data/units.yaml index 791ad2de..f02f7421 100644 --- a/py/desitarget/data/units.yaml +++ b/py/desitarget/data/units.yaml @@ -13,7 +13,7 @@ BRICKID: BRICKNAME: DCHISQ: DEC: deg -DEC_IVAR: 1/deg^2 +DEC_IVAR: deg^-2 EBV: mag FIBERFLUX_G: nanomaggy FIBERFLUX_R: nanomaggy @@ -22,13 +22,13 @@ FIBERTOTFLUX_G: nanomaggy FIBERTOTFLUX_R: nanomaggy FIBERTOTFLUX_Z: nanomaggy FLUX_G: nanomaggy -FLUX_IVAR_G: 1/nanomaggy^2 -FLUX_IVAR_R: 1/nanomaggy^2 -FLUX_IVAR_W1: 1/nanomaggy^2 -FLUX_IVAR_W2: 1/nanomaggy^2 -FLUX_IVAR_W3: 1/nanomaggy^2 -FLUX_IVAR_W4: 1/nanomaggy^2 -FLUX_IVAR_Z: 1/nanomaggy^2 +FLUX_IVAR_G: nanomaggy^-2 +FLUX_IVAR_R: nanomaggy^-2 +FLUX_IVAR_W1: nanomaggy^-2 +FLUX_IVAR_W2: nanomaggy^-2 +FLUX_IVAR_W3: nanomaggy^-2 +FLUX_IVAR_W4: nanomaggy^-2 +FLUX_IVAR_Z: nanomaggy^-2 FLUX_R: nanomaggy FLUX_W1: nanomaggy FLUX_W2: nanomaggy @@ -46,11 +46,11 @@ FRACIN_Z: FRACMASKED_G: FRACMASKED_R: FRACMASKED_Z: -GALDEPTH_G: 1/nanomaggy^2 -GALDEPTH_R: 1/nanomaggy^2 -GALDEPTH_Z: 1/nanomaggy^2 -LC_FLUX_IVAR_W1: 1/nanomaggy^2 -LC_FLUX_IVAR_W2: 1/nanomaggy^2 +GALDEPTH_G: nanomaggy^-2 +GALDEPTH_R: nanomaggy^-2 +GALDEPTH_Z: nanomaggy^-2 +LC_FLUX_IVAR_W1: nanomaggy^-2 +LC_FLUX_IVAR_W2: nanomaggy^-2 LC_FLUX_W1: nanomaggy LC_FLUX_W2: nanomaggy LC_MJD_W1: @@ -69,11 +69,11 @@ NOBS_G: NOBS_R: NOBS_Z: OBJID: -PSFDEPTH_G: 1/nanomaggy^2 -PSFDEPTH_R: 1/nanomaggy^2 -PSFDEPTH_Z: 1/nanomaggy^2 +PSFDEPTH_G: nanomaggy^-2 +PSFDEPTH_R: nanomaggy^-2 +PSFDEPTH_Z: nanomaggy^-2 RA: deg -RA_IVAR: 1/deg^2 +RA_IVAR: deg^-2 REF_EPOCH: yr RELEASE: SERSIC: @@ -83,13 +83,13 @@ SHAPEDEV_E1_IVAR: SHAPEDEV_E2: SHAPEDEV_E2_IVAR: SHAPEDEV_R: arcsec -SHAPEDEV_R_IVAR: 1/arcsec^2 +SHAPEDEV_R_IVAR: arcsec^-2 SHAPEEXP_E1: SHAPEEXP_E1_IVAR: SHAPEEXP_E2: SHAPEEXP_E2_IVAR: SHAPEEXP_R: arcsec -SHAPEEXP_R_IVAR: 1/arcsec^2 +SHAPEEXP_R_IVAR: arcsec^-2 SHAPE_E1: SHAPE_E1_IVAR: SHAPE_E2: @@ -117,11 +117,11 @@ GAIA_DUPLICATED_SOURCE: GAIA_ASTROMETRIC_SIGMA5D_MAX: GAIA_ASTROMETRIC_PARAMS_SOLVED: PARALLAX: mas -PARALLAX_IVAR: 1/mas^2 +PARALLAX_IVAR: mas^-2 PMRA: mas / yr -PMRA_IVAR: yr^2 / mas^2 +PMRA_IVAR: yr2 / mas2 PMDEC: mas / yr -PMDEC_IVAR: yr^2 / mas^2 +PMDEC_IVAR: yr2 / mas2 # ADM columns specifically added by target selection. BGS_TARGET: @@ -140,9 +140,9 @@ TARGETID: # ADM sky-related units: BLOBDIST: pix -FIBERFLUX_IVAR_G: 1/nanomaggy^2 -FIBERFLUX_IVAR_R: 1/nanomaggy^2 -FIBERFLUX_IVAR_Z: 1/nanomaggy^2 +FIBERFLUX_IVAR_G: nanomaggy^-2 +FIBERFLUX_IVAR_R: nanomaggy^-2 +FIBERFLUX_IVAR_Z: nanomaggy^-2 # ADM GFA-related units: URAT_ID: @@ -150,15 +150,15 @@ URAT_SEP: arcsec # ADM randoms-related units: APFLUX_G: nanomaggy -APFLUX_IVAR_G: 1/nanomaggy^2 -APFLUX_IVAR_R: 1/nanomaggy^2 -APFLUX_IVAR_Z: 1/nanomaggy^2 +APFLUX_IVAR_G: nanomaggy^-2 +APFLUX_IVAR_R: nanomaggy^-2 +APFLUX_IVAR_Z: nanomaggy^-2 APFLUX_R: nanomaggy APFLUX_Z: nanomaggy NUMOBS_MORE: PRIORITY: -PSFDEPTH_W1: 1/nanomaggy^2 -PSFDEPTH_W2: 1/nanomaggy^2 +PSFDEPTH_W1: nanomaggy^-2 +PSFDEPTH_W2: nanomaggy^-2 PSFSIZE_G: arcsec PSFSIZE_R: arcsec PSFSIZE_Z: arcsec @@ -182,5 +182,5 @@ TILEID: # ADM ToO-related units: CHECKER: -MJD_BEGIN: day -MJD_END: day +MJD_BEGIN: d +MJD_END: d diff --git a/py/desitarget/test/test_units.py b/py/desitarget/test/test_units.py index a8a7ab0b..b636e469 100644 --- a/py/desitarget/test/test_units.py +++ b/py/desitarget/test/test_units.py @@ -42,12 +42,12 @@ def test_fits_units(self): uniq = set(self.units.values()) uniq.remove(None) - # ADM nmgy isn't an allowed unit in earlier versions of astropy. - if astropyversion.major < 4: - uniq = set([i for i in list(uniq) if 'nanomaggy' not in i]) + # ADM allowed versions of units that aren't FITS-compliant. + allowed = ["nanomaggy", "nanomaggy^-2"] + uniq = set([i for i in list(uniq) if i not in allowed]) # ADM parse the units to check they're allowed astropy units. - parsed = [u.Unit(unit) for unit in uniq] + parsed = [u.Unit(unit, format="fits") for unit in uniq] # ADM these should be equivalent, even though, formally, parsed # ADM contains items of type astropy.units.core.Unit. From 038181e5bae38700ae2a632cb44c3b3c740d3e06 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Mon, 4 Mar 2024 15:33:54 -0800 Subject: [PATCH 18/63] add write function (that also checks for too-bright targets); add executable script --- bin/select_stream_targets | 76 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100755 bin/select_stream_targets diff --git a/bin/select_stream_targets b/bin/select_stream_targets new file mode 100755 index 00000000..955e1059 --- /dev/null +++ b/bin/select_stream_targets @@ -0,0 +1,76 @@ +#!/usr/bin/env python + +from __future__ import print_function, division + +import os, sys +import numpy as np +import fitsio + +from desitarget.streams import io, cuts + +from time import time +start = time() + +from desiutil.log import get_logger +log = get_logger() + +from argparse import ArgumentParser +ap = ArgumentParser(description=("Generates DESI target bits from Legacy Surveys" + " sweeps files") + ) +ap.add_argument("sweepdir", + help=("Root directory of LS sweeps for a given data release for " + "ONE of EITHER north or south") + ) +ap.add_argument("dest", + help=("Output target selection directory (the file name is built" + "on-the-fly from other inputs") + ) +ap.add_argument('-s','--streamnames', default="GD1", + help=("Comma-separated names of streams to run (e.g. x,y,z). " + "Default is to just run GD1)") + ) +ap.add_argument("--donotaddnors", action="store_true", + help=("Both the south/north LS files are read by default. Pass " + "this to read only the specific files in passed sweepdir.") + ) +ap.add_argument("--donotreadperstream", action="store_true", + help=("Default is to read targets in a loop per-stream instead " + "of in a loor per-sweeps file. Pass this to loop over " + " sweep files instead of streams.") + ) +ap.add_argument("--donotreadcache", action="store_true", + help=("Default is to read from previously cached data files " + "where possible. Pass this to start from scratch (and " + "overwrite caches). Only relevant for --readperstream.") + ) + +ns = ap.parse_args() + +# ADM build the list of command line arguments to +# ADM write to the output header. +hdr = fitsio.FITSHDR() +nsdict = vars(ns) +for k in nsdict: + hdr[k.upper()] = nsdict[k] + +# ADM change the input comma-separated string of stream names to a list. +sn = ns.streamnames +if sn is not None: + streamnames = sn.split(',') + +print(streamnames, hdr) + +# ADM flip the sense of some inputs. +addnors = not(ns.donotaddnors) +readperstream = not(ns.donotreadperstream) +readcache = not(ns.donotreadcache) + +targets = cuts.select_targets( + ns.sweepdir, stream_names=streamnames, readperstream=readperstream, + addnors=addnors, readcache=readcache +) + +ntargs, filename = io.write_targets(ns.dest, targets, header) + +log.info(f"{ntargs} targets written to {outfile}...t={time()-start:.1f}s") From 794cee4a6177c59a3f91c4a329cf2d4409be6629 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Tue, 5 Mar 2024 05:44:16 -0800 Subject: [PATCH 19/63] add standalone write function. Check if targets are too bright before writing them to file --- bin/run_target_qa | 2 +- py/desitarget/streams/cuts.py | 10 ++++- py/desitarget/streams/io.py | 62 ++++++++++++++++++++++++++++-- py/desitarget/streams/targets.py | 2 - py/desitarget/streams/utilities.py | 1 - 5 files changed, 67 insertions(+), 10 deletions(-) diff --git a/bin/run_target_qa b/bin/run_target_qa index db224705..82c6142a 100755 --- a/bin/run_target_qa +++ b/bin/run_target_qa @@ -79,7 +79,7 @@ else: max_bin_area = 1.0 if ns.prepare: - fn = os.path.join(os.getenv("CSCRATCH"), ns.dest) + fn = os.path.join(os.getenv("SCRATCH"), ns.dest) objs, _, _, hdr = read_data(ns.src, header=True, downsample=ns.downsample) fitsio.write(fn+'.tmp', objs, extname='TARGETS', header=hdr, clobber=True) os.rename(fn+'.tmp', fn) diff --git a/py/desitarget/streams/cuts.py b/py/desitarget/streams/cuts.py index 885f7435..efe5af05 100644 --- a/py/desitarget/streams/cuts.py +++ b/py/desitarget/streams/cuts.py @@ -237,7 +237,7 @@ def set_target_bits(objs, stream_names=["GD1"]): def select_targets(swdir, stream_names=["GD1"], readperstream=True, - readcache=True): + addnors=True, readcache=True): """Process files from an input directory to select targets. Parameters @@ -253,6 +253,11 @@ def select_targets(swdir, stream_names=["GD1"], readperstream=True, through all possible sweeps files. This is likely quickest and most useful when working with a single stream. For multiple streams it may cause issues when duplicate targets are selected. + addnors : :class:`bool` + If ``True`` then if `swdir` contains "north" add sweep files from + the south by substituting "south" in place of "north" (and vice + versa, i.e. if `swdir` contains "south" add sweep files from the + north by substituting "north" in place of "south"). readcache : :class:`bool`, optional, defaults to ``True`` If ``True`` read all data from previously made cache files, in cases where such files exist. If ``False`` don't read @@ -283,7 +288,8 @@ def select_targets(swdir, stream_names=["GD1"], readperstream=True, mind, maxd = strm["MIND"], strm["MAXD"] # ADM read in the data. objs = read_data_per_stream(swdir, rapol, decpol, ra_ref, mind, maxd, - stream_name, readcache=readcache) + stream_name, + addnors=addnors, readcache=readcache) allobjs.append(objs) objects = np.concatenate(allobjs) else: diff --git a/py/desitarget/streams/io.py b/py/desitarget/streams/io.py index 2dd495a7..50f339a7 100644 --- a/py/desitarget/streams/io.py +++ b/py/desitarget/streams/io.py @@ -9,12 +9,15 @@ import os import fitsio import numpy as np +import healpy as hp +import astropy.coordinates as acoo +import astropy.units as auni from desitarget import io from desitarget.geomask import pixarea2nside, add_hp_neighbors, sweep_files_touch_hp from desitarget.gaiamatch import match_gaia_to_primary from desitarget.targets import resolve - +from desitarget.streams.utilities import betw # ADM set up the DESI default logger. from desiutil.log import get_logger @@ -27,12 +30,15 @@ streamcols = np.array([], dtype=[ ('RELEASE', '>i2'), ('BRICKID', '>i4'), ('TYPE', 'S4'), ('OBJID', '>i4'), ('RA', '>f8'), ('DEC', '>f8'), ('EBV', '>f4'), - ('FLUX_G', '>f4'), ('FLUX_R', '>f4'), ('FLUX_Z', '>f4'), + ('FLUX_G', '>f4'), ('FIBERTOTFLUX_G', '>f4'), + ('FLUX_R', '>f4'), ('FIBERTOTFLUX_R', '>f4'), + ('FLUX_Z', '>f4'), ('FIBERTOTFLUX_Z', '>f4'), ('REF_EPOCH', '>f4'), ('PARALLAX', '>f4'), ('PARALLAX_IVAR', '>f4'), ('PMRA', '>f4'), ('PMRA_IVAR', '>f4'), ('PMDEC', '>f4'), ('PMDEC_IVAR', '>f4'), ('ASTROMETRIC_PARAMS_SOLVED', '>i1'), ('NU_EFF_USED_IN_ASTROMETRY', '>f4'), - ('PSEUDOCOLOUR', '>f4'), ('PHOT_G_MEAN_MAG', '>f4'), ('ECL_LAT', '>f8') + ('PSEUDOCOLOUR', '>f4'), ('ECL_LAT', '>f8'), ('PHOT_G_MEAN_MAG', '>f4'), + ('PHOT_BP_MEAN_MAG', '>f4'), ('PHOT_RP_MEAN_MAG', '>f4') ]) # ADM the Gaia Data Release for matching throughout this module. @@ -40,7 +46,7 @@ def read_data_per_stream(swdir, rapol, decpol, ra_ref, mind, maxd, stream_name, - readcache=True, addnors=True, test=False): + readcache=True, addnors=True, test=False): """Assemble the data needed for a particular stream program. Parameters @@ -226,3 +232,51 @@ def read_data_per_stream(swdir, rapol, decpol, ra_ref, mind, maxd, stream_name, header=hdr, extname="STREAMCACHE") return allobjs + + +def write_targets(filename, targs, header): + """Write stream targets to a FITS file. + + Parameters + ---------- + filename : :class:`str` + The output filename. + targs : :class:`~numpy.ndarray` + The numpy structured array of data to write. + header : :class:`dict` optional + Header for output file. Can be a FITShdr object or dictionary. + + Returns + ------- + :class:`int` + The number of targets that were written to file. + :class:`str` + The name of the file to which targets were written. + + Notes + ----- + - Must contain at least the columns: + PHOT_G_MEAN_MAG, PHOT_BP_MEAN_MAG, PHOT_RP_MEAN_MAG and + FIBERTOTFLUX_G, FIBERTOTFLUX_R, FIBERTOTFLUX_Z + - Always OVERWRITES existing files! + - Writes atomically. Any files that died mid-write will be + appended by ".tmp". + - Units are automatically added from the desitarget units yaml file + (see `/data/units.yaml`). + - Currently mostly wraps :func:`~desitarget.io.write_with_units`. + """ + # ADM check if any targets are too bright. + maglim = 15 + fluxlim = 10**((22.5-maglim)/2.5) + toobright = np.zeros(len(targs), dtype="?") + for col in ["PHOT_G_MEAN_MAG", "PHOT_BP_MEAN_MAG", "PHOT_RP_MEAN_MAG"]: + toobright |= (targs[col] != 0) & (targs[col] < maglim) + for col in ["FIBERTOTFLUX_G", "FIBERTOTFLUX_R", "FIBERTOTFLUX_Z"]: + toobright |= (targs[col] != 0) & (targs[col] > fluxlim) + if np.any(toobright): + tids = targs["TARGETID"][toobright] + log.warning(f"Targets TOO BRIGHT to be written to {filename}: {tids}") + + io.write_with_units(filename, targs, extname="STREAMTARGETS", header=header) + + return len(targs), filename diff --git a/py/desitarget/streams/targets.py b/py/desitarget/streams/targets.py index 7dfc345c..3609df2d 100644 --- a/py/desitarget/streams/targets.py +++ b/py/desitarget/streams/targets.py @@ -70,12 +70,10 @@ def finalize(targets, desi_target, bgs_target, mws_target, scnd_target): targets = rfn.rename_fields(targets, {'OBJID': 'BRICK_OBJID', 'TYPE': 'MORPHTYPE'}) - targetid = encode_targetid(objid=targets['BRICK_OBJID'], brickid=targets['BRICKID'], release=targets['RELEASE']) - nodata = np.zeros(ntargets, dtype='int')-1 subpriority = np.zeros(ntargets, dtype='float') diff --git a/py/desitarget/streams/utilities.py b/py/desitarget/streams/utilities.py index 0b604ca1..6a3658ed 100644 --- a/py/desitarget/streams/utilities.py +++ b/py/desitarget/streams/utilities.py @@ -465,4 +465,3 @@ def stream_distance(fi1, stream_name): else: msg = f"stream name {stream_name} not recognized" log.error(msg) - From 3813e35169575f5b9e0698a668c594cb236bfbc5 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Tue, 5 Mar 2024 10:23:31 -0800 Subject: [PATCH 20/63] debugging and bookkeeping --- bin/select_stream_targets | 18 +++++------ py/desitarget/streams/io.py | 60 +++++++++++++++++++++++++++++-------- 2 files changed, 57 insertions(+), 21 deletions(-) diff --git a/bin/select_stream_targets b/bin/select_stream_targets index 955e1059..f8a6499c 100755 --- a/bin/select_stream_targets +++ b/bin/select_stream_targets @@ -16,7 +16,7 @@ log = get_logger() from argparse import ArgumentParser ap = ArgumentParser(description=("Generates DESI target bits from Legacy Surveys" - " sweeps files") + " sweep files") ) ap.add_argument("sweepdir", help=("Root directory of LS sweeps for a given data release for " @@ -24,25 +24,25 @@ ap.add_argument("sweepdir", ) ap.add_argument("dest", help=("Output target selection directory (the file name is built" - "on-the-fly from other inputs") + " on-the-fly from other inputs)") ) ap.add_argument('-s','--streamnames', default="GD1", help=("Comma-separated names of streams to run (e.g. x,y,z). " - "Default is to just run GD1)") + "Default is to just run GD1") ) ap.add_argument("--donotaddnors", action="store_true", help=("Both the south/north LS files are read by default. Pass " - "this to read only the specific files in passed sweepdir.") + "this to read only the specific files in passed sweepdir") ) ap.add_argument("--donotreadperstream", action="store_true", help=("Default is to read targets in a loop per-stream instead " "of in a loor per-sweeps file. Pass this to loop over " - " sweep files instead of streams.") + " sweep files instead of streams") ) ap.add_argument("--donotreadcache", action="store_true", help=("Default is to read from previously cached data files " "where possible. Pass this to start from scratch (and " - "overwrite caches). Only relevant for --readperstream.") + "overwrite caches). Only relevant for --readperstream") ) ns = ap.parse_args() @@ -53,14 +53,14 @@ hdr = fitsio.FITSHDR() nsdict = vars(ns) for k in nsdict: hdr[k.upper()] = nsdict[k] +# ADM also explicitly add the command used, just in case. +hdr["CMDLINE"] = ' '.join(sys.argv) # ADM change the input comma-separated string of stream names to a list. sn = ns.streamnames if sn is not None: streamnames = sn.split(',') -print(streamnames, hdr) - # ADM flip the sense of some inputs. addnors = not(ns.donotaddnors) readperstream = not(ns.donotreadperstream) @@ -71,6 +71,6 @@ targets = cuts.select_targets( addnors=addnors, readcache=readcache ) -ntargs, filename = io.write_targets(ns.dest, targets, header) +ntargs, outfile = io.write_targets(ns.dest, targets, hdr, streamnames=sn) log.info(f"{ntargs} targets written to {outfile}...t={time()-start:.1f}s") diff --git a/py/desitarget/streams/io.py b/py/desitarget/streams/io.py index 50f339a7..c42b7b44 100644 --- a/py/desitarget/streams/io.py +++ b/py/desitarget/streams/io.py @@ -18,6 +18,7 @@ from desitarget.gaiamatch import match_gaia_to_primary from desitarget.targets import resolve from desitarget.streams.utilities import betw +from desiutil import depend # ADM set up the DESI default logger. from desiutil.log import get_logger @@ -234,17 +235,22 @@ def read_data_per_stream(swdir, rapol, decpol, ra_ref, mind, maxd, stream_name, return allobjs -def write_targets(filename, targs, header): +def write_targets(dirname, targs, header, streamnames=""): """Write stream targets to a FITS file. Parameters ---------- - filename : :class:`str` - The output filename. + dirname : :class:`str` + The output directory name. Filenames are constructed from other + inputs. targs : :class:`~numpy.ndarray` The numpy structured array of data to write. - header : :class:`dict` optional + header : :class:`dict` Header for output file. Can be a FITShdr object or dictionary. + Pass {} if you have no additional header information. + streamnames : :class:`str, optional + Information about stream names that correspond to the targets. + Included in the output filename. Returns ------- @@ -257,14 +263,27 @@ def write_targets(filename, targs, header): ----- - Must contain at least the columns: PHOT_G_MEAN_MAG, PHOT_BP_MEAN_MAG, PHOT_RP_MEAN_MAG and - FIBERTOTFLUX_G, FIBERTOTFLUX_R, FIBERTOTFLUX_Z + FIBERTOTFLUX_G, FIBERTOTFLUX_R, FIBERTOTFLUX_Z, RELEASE - Always OVERWRITES existing files! - - Writes atomically. Any files that died mid-write will be + - Writes atomically. Any output files that died mid-write will be appended by ".tmp". - Units are automatically added from the desitarget units yaml file (see `/data/units.yaml`). - - Currently mostly wraps :func:`~desitarget.io.write_with_units`. + - Mostly wraps :func:`~desitarget.io.write_with_units`. """ + # ADM construct the output filename. + drs = list(set(targs["RELEASE"]//1000)) + if len(drs) == 1: + drint = drs[0] + drstr = f"dr{drint}" + else: + log.info("Couldn't parse LS data release. Defaulting to drX.") + drint = "X" + drstr = "drX" + outfn = f"streamtargets-{streamnames.lower()}-bright.fits" + outfn = os.path.join(dirname, drstr, io.desitarget_version, + "streamtargets", "main", "resolve", "bright", outfn) + # ADM check if any targets are too bright. maglim = 15 fluxlim = 10**((22.5-maglim)/2.5) @@ -275,8 +294,25 @@ def write_targets(filename, targs, header): toobright |= (targs[col] != 0) & (targs[col] > fluxlim) if np.any(toobright): tids = targs["TARGETID"][toobright] - log.warning(f"Targets TOO BRIGHT to be written to {filename}: {tids}") - - io.write_with_units(filename, targs, extname="STREAMTARGETS", header=header) - - return len(targs), filename + log.warning(f"Targets TOO BRIGHT to be written to {outfn}: {tids}") + + # ADM add the DESI dependencies. + depend.add_dependencies(header) + # ADM some other useful header information. + depend.setdep(header, 'desitarget', io.desitarget_version) + depend.setdep(header, 'desitarget-git', io.gitversion()) + depend.setdep(header, 'photcat', drstr) + + # ADM add information to construct the filename to the header. + header["OBSCON"] = "bright" + header["SURVEY"] = "main" + header["RESOLVE"] = True + header["DR"] = drint + header["GAIADR"] = gaiadr + + # ADM create necessary directories, if they don't exist. + os.makedirs(os.path.dirname(outfn), exist_ok=True) + # ADM and, finally, write out the targets. + io.write_with_units(outfn, targs, extname="STREAMTARGETS", header=header) + + return len(targs), outfn From 35bf7c3b77d955a768d0cb8fa6b58f7a908ff2eb Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Tue, 5 Mar 2024 15:19:53 -0800 Subject: [PATCH 21/63] absorb a snapshot of the gaiadr3_zeropoint package into desitarget --- .../gaia_dr3_parallax_zero_point/LICENSE | 165 ++++++++++ .../gaia_dr3_parallax_zero_point/README.md | 19 ++ .../coefficients/z5_200720.txt | 15 + .../coefficients/z6_200720.txt | 15 + .../gaia_dr3_parallax_zero_point/zpt.py | 292 ++++++++++++++++++ py/desitarget/streams/utilities.py | 2 +- 6 files changed, 507 insertions(+), 1 deletion(-) create mode 100644 py/desitarget/streams/gaia_dr3_parallax_zero_point/LICENSE create mode 100644 py/desitarget/streams/gaia_dr3_parallax_zero_point/README.md create mode 100644 py/desitarget/streams/gaia_dr3_parallax_zero_point/coefficients/z5_200720.txt create mode 100644 py/desitarget/streams/gaia_dr3_parallax_zero_point/coefficients/z6_200720.txt create mode 100644 py/desitarget/streams/gaia_dr3_parallax_zero_point/zpt.py diff --git a/py/desitarget/streams/gaia_dr3_parallax_zero_point/LICENSE b/py/desitarget/streams/gaia_dr3_parallax_zero_point/LICENSE new file mode 100644 index 00000000..65c5ca88 --- /dev/null +++ b/py/desitarget/streams/gaia_dr3_parallax_zero_point/LICENSE @@ -0,0 +1,165 @@ + GNU LESSER GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + + This version of the GNU Lesser General Public License incorporates +the terms and conditions of version 3 of the GNU General Public +License, supplemented by the additional permissions listed below. + + 0. Additional Definitions. + + As used herein, "this License" refers to version 3 of the GNU Lesser +General Public License, and the "GNU GPL" refers to version 3 of the GNU +General Public License. + + "The Library" refers to a covered work governed by this License, +other than an Application or a Combined Work as defined below. + + An "Application" is any work that makes use of an interface provided +by the Library, but which is not otherwise based on the Library. +Defining a subclass of a class defined by the Library is deemed a mode +of using an interface provided by the Library. + + A "Combined Work" is a work produced by combining or linking an +Application with the Library. The particular version of the Library +with which the Combined Work was made is also called the "Linked +Version". + + The "Minimal Corresponding Source" for a Combined Work means the +Corresponding Source for the Combined Work, excluding any source code +for portions of the Combined Work that, considered in isolation, are +based on the Application, and not on the Linked Version. + + The "Corresponding Application Code" for a Combined Work means the +object code and/or source code for the Application, including any data +and utility programs needed for reproducing the Combined Work from the +Application, but excluding the System Libraries of the Combined Work. + + 1. Exception to Section 3 of the GNU GPL. + + You may convey a covered work under sections 3 and 4 of this License +without being bound by section 3 of the GNU GPL. + + 2. Conveying Modified Versions. + + If you modify a copy of the Library, and, in your modifications, a +facility refers to a function or data to be supplied by an Application +that uses the facility (other than as an argument passed when the +facility is invoked), then you may convey a copy of the modified +version: + + a) under this License, provided that you make a good faith effort to + ensure that, in the event an Application does not supply the + function or data, the facility still operates, and performs + whatever part of its purpose remains meaningful, or + + b) under the GNU GPL, with none of the additional permissions of + this License applicable to that copy. + + 3. Object Code Incorporating Material from Library Header Files. + + The object code form of an Application may incorporate material from +a header file that is part of the Library. You may convey such object +code under terms of your choice, provided that, if the incorporated +material is not limited to numerical parameters, data structure +layouts and accessors, or small macros, inline functions and templates +(ten or fewer lines in length), you do both of the following: + + a) Give prominent notice with each copy of the object code that the + Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the object code with a copy of the GNU GPL and this license + document. + + 4. Combined Works. + + You may convey a Combined Work under terms of your choice that, +taken together, effectively do not restrict modification of the +portions of the Library contained in the Combined Work and reverse +engineering for debugging such modifications, if you also do each of +the following: + + a) Give prominent notice with each copy of the Combined Work that + the Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the Combined Work with a copy of the GNU GPL and this license + document. + + c) For a Combined Work that displays copyright notices during + execution, include the copyright notice for the Library among + these notices, as well as a reference directing the user to the + copies of the GNU GPL and this license document. + + d) Do one of the following: + + 0) Convey the Minimal Corresponding Source under the terms of this + License, and the Corresponding Application Code in a form + suitable for, and under terms that permit, the user to + recombine or relink the Application with a modified version of + the Linked Version to produce a modified Combined Work, in the + manner specified by section 6 of the GNU GPL for conveying + Corresponding Source. + + 1) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (a) uses at run time + a copy of the Library already present on the user's computer + system, and (b) will operate properly with a modified version + of the Library that is interface-compatible with the Linked + Version. + + e) Provide Installation Information, but only if you would otherwise + be required to provide such information under section 6 of the + GNU GPL, and only to the extent that such information is + necessary to install and execute a modified version of the + Combined Work produced by recombining or relinking the + Application with a modified version of the Linked Version. (If + you use option 4d0, the Installation Information must accompany + the Minimal Corresponding Source and Corresponding Application + Code. If you use option 4d1, you must provide the Installation + Information in the manner specified by section 6 of the GNU GPL + for conveying Corresponding Source.) + + 5. Combined Libraries. + + You may place library facilities that are a work based on the +Library side by side in a single library together with other library +facilities that are not Applications and are not covered by this +License, and convey such a combined library under terms of your +choice, if you do both of the following: + + a) Accompany the combined library with a copy of the same work based + on the Library, uncombined with any other library facilities, + conveyed under the terms of this License. + + b) Give prominent notice with the combined library that part of it + is a work based on the Library, and explaining where to find the + accompanying uncombined form of the same work. + + 6. Revised Versions of the GNU Lesser General Public License. + + The Free Software Foundation may publish revised and/or new versions +of the GNU Lesser General Public License from time to time. Such new +versions will be similar in spirit to the present version, but may +differ in detail to address new problems or concerns. + + Each version is given a distinguishing version number. If the +Library as you received it specifies that a certain numbered version +of the GNU Lesser General Public License "or any later version" +applies to it, you have the option of following the terms and +conditions either of that published version or of any later version +published by the Free Software Foundation. If the Library as you +received it does not specify a version number of the GNU Lesser +General Public License, you may choose any version of the GNU Lesser +General Public License ever published by the Free Software Foundation. + + If the Library as you received it specifies that a proxy can decide +whether future versions of the GNU Lesser General Public License shall +apply, that proxy's public statement of acceptance of any version is +permanent authorization for you to choose that version for the +Library. diff --git a/py/desitarget/streams/gaia_dr3_parallax_zero_point/README.md b/py/desitarget/streams/gaia_dr3_parallax_zero_point/README.md new file mode 100644 index 00000000..79845320 --- /dev/null +++ b/py/desitarget/streams/gaia_dr3_parallax_zero_point/README.md @@ -0,0 +1,19 @@ +## Attribution + +The code and coefficient files in this directory are a snapshot of the +[gaiadr3_zeropoint](https://gitlab.com/icc-ub/public/gaiadr3_zeropoint/-/blob/master/) +package (downloaded on 2024-03-05). Please refer to +[the original code](https://gitlab.com/icc-ub/public/gaiadr3_zeropoint/) +for details and attribution. + +## License + +The contents of this (and only this) directory retain the original +GNU LESSER GENERAL PUBLIC LICENSE included herein. + +## Justification + +The gaiadr3_zeropoint package includes some dependencies that are +not supported by [desihub](https://github.com/desihub) and the DESI +project. We have therefore taken a snapshot of the package to +simplify maintenance of the DESI software stack. \ No newline at end of file diff --git a/py/desitarget/streams/gaia_dr3_parallax_zero_point/coefficients/z5_200720.txt b/py/desitarget/streams/gaia_dr3_parallax_zero_point/coefficients/z5_200720.txt new file mode 100644 index 00000000..615d0013 --- /dev/null +++ b/py/desitarget/streams/gaia_dr3_parallax_zero_point/coefficients/z5_200720.txt @@ -0,0 +1,15 @@ + -1, 0, 0, 0, 1, 1, 2, 3, 4 + -2, 0, 1, 2, 0, 1, 0, 0, 0 + 6.0, -26.98, -9.62, 27.40, -25.1, -0.0, -1257, 0, 0 +10.8, -27.23, -3.07, 23.04, 35.3, 15.7, -1257, 0, 0 +11.2, -30.33, -9.23, 9.08, -88.4, -11.8, -1257, 0, 0 +11.8, -33.54, -10.08, 13.28, -126.7, 11.6, -1257, 0, 0 +12.2, -13.65, -0.07, 9.35, -111.4, 40.6, -1257, 0, 0 +12.9, -19.53, -1.64, 15.86, -66.8, 20.6, -1257, 0, 0 +13.1, -37.99, +2.63, +16.14, -5.7, +14.0, -1257, +107.9, +104.3 +15.9, -38.33, +5.61, +15.42, 0, +18.7, -1189, +243.8, +155.2 +16.1, -31.05, +2.83, +8.59, 0, +15.5, -1404, +105.5, +170.7 +17.5, -29.18, -0.09, +2.41, 0, +24.5, -1165, +189.7, +325.0 +19.0, -18.40, +5.98, -6.46, 0, +5.5, 0, 0, +276.6 +20.0, -12.65, -4.57, -7.46, 0, +97.9, 0, 0, 0 +21.0, -18.22, -15.24, -18.54, 0, +128.2, 0, 0, 0 diff --git a/py/desitarget/streams/gaia_dr3_parallax_zero_point/coefficients/z6_200720.txt b/py/desitarget/streams/gaia_dr3_parallax_zero_point/coefficients/z6_200720.txt new file mode 100644 index 00000000..3c0dbb77 --- /dev/null +++ b/py/desitarget/streams/gaia_dr3_parallax_zero_point/coefficients/z6_200720.txt @@ -0,0 +1,15 @@ +-1, 0, 0, 0, 1, 1, 1, 2 +-2, 0, 1, 2, 0, 1, 2, 0 + 6.0, -27.85, -7.78, 27.47, -32.1, 14.4, 9.5, -67 +10.8, -28.91, -3.57, 22.92, 7.7, 12.6, 1.6, -572 +11.2, -26.72, -8.74, 9.36, -30.3, 5.6, 17.2, -1104 +11.8, -29.04, -9.69, 13.63, -49.4, 36.3, 17.7, -1129 +12.2, -12.39, -2.16, 10.23, -92.6, 19.8, 27.6, -365 +12.9, -18.99, -1.93, 15.90, -57.2, -8.0, 19.9, -554 +13.1, -38.29, 2.59, 16.20, -10.5, 1.4, 0.4, -960 +15.9, -36.83, 4.20, 15.76, 22.3, 11.1, 10.0, -1367 +16.1, -28.37, 1.99, 9.28, 50.4, 17.2, 13.7, -1351 +17.5, -24.68, -1.37, 3.52, 86.8, 19.8, 21.3, -1380 +19.0, -15.32, 4.01, -6.03, 29.2, 14.1, 0.4, -563 +20.0, -13.73, -10.92, -8.30, -74.4, 196.4, -42.0, 536 +21.0, -29.53, -20.34, -18.74, -39.5, 326.8, -262.3, 1598 diff --git a/py/desitarget/streams/gaia_dr3_parallax_zero_point/zpt.py b/py/desitarget/streams/gaia_dr3_parallax_zero_point/zpt.py new file mode 100644 index 00000000..837e552a --- /dev/null +++ b/py/desitarget/streams/gaia_dr3_parallax_zero_point/zpt.py @@ -0,0 +1,292 @@ +# created by: Pau Ramos (@brugalada) & Anthony Brown +# based on Matlab code by: Lennart Lindegren +# Date: 21/07/2020 (important for the table of coefficients) + +import os +import numpy as np +import warnings + +__mypath = os.path.dirname(os.path.abspath(__file__)) + +_file5_currentversion = __mypath + '/coefficients/z5_200720.txt' +_file6_currentversion = __mypath + '/coefficients/z6_200720.txt' + + +# Definition of functions that load the coefficient tables and initialize the global variables + +def _read_table(file, sep=','): + """ + Extract the coefficients and interpolation limits from the input file provided. + + The first and second rows are assumed to be the indices that govern, respectively, the colour and sinBeta interpolations. + + From the third rows onwards, all belong to the G magnitude interpolation: first column, the phot_g_mean_mag boundaries. The rest of columns, the interpolation coefficients. + """ + # reads the file (.txt) + input_file = np.genfromtxt(file, delimiter=sep) + + # auxiliary variables j and k + j = list(map(int, input_file[0, 1:])) + k = list(map(int, input_file[1, 1:])) + # g vector + g = input_file[2:, 0] + # coefficients + q_jk = input_file[2:, 1:] + # shape + n, m = q_jk.shape + + return j, k, g, q_jk, n, m + + +def load_tables(file5=_file5_currentversion, file6=_file6_currentversion, sep=','): + """ + Initialises the tables containing the coefficients of the interpolations for the Z5 and Z6 functions. + + NOTE: USE THE DEFAULT VALUES unless you are very sure of what you are doing. + + Inputs + file5: path to the file with the Z5 coefficients (.txt or similar) + file6: path to the file with the Z6 coefficients (.txt or similar) + sep (optional): separator used to split the lines (default, comma) + """ + global j_5, k_5, g_5, q_jk5, n_5, m_5, j_6, k_6, g_6, q_jk6, n_6, m_6 + + j_5, k_5, g_5, q_jk5, n_5, m_5 = _read_table(file5, sep=sep) + j_6, k_6, g_6, q_jk6, n_6, m_6 = _read_table(file6, sep=sep) + + return None + + +# Auxiliary function: calculates the zero-point only for an array of stars with the same number of +# astrometric_params_solved + +def _calc_zpt(phot_g_mean_mag, nu_eff_used_in_astrometry, pseudocolour, sinBeta, source_type): + """ + Compute the zero-point parallax for an array of stars. + + WARNING! This function is meant to be auxiliary, therefore it assumes that the inputs are well formatted (see + get_zpt()) and that all the sources have the same value for astrometric_params_solved. That is, either all are 5p + (source_type: 5) or 6p (source_type: 6). Never 2p. + """ + + # load the right coefficients: + if source_type == 5: + colour = nu_eff_used_in_astrometry + j, k, g, q_jk, n, m = j_5, k_5, g_5, q_jk5, n_5, m_5 + elif source_type == 6: + colour = pseudocolour + j, k, g, q_jk, n, m = j_6, k_6, g_6, q_jk6, n_6, m_6 + + # basis functions evaluated at colour and ecl_lat + c = [np.ones_like(colour), + np.max((-0.24 * np.ones_like(colour), np.min((0.24 * np.ones_like(colour), colour - 1.48), axis=0)), axis=0), + np.min((0.24 * np.ones_like(colour), np.max((np.zeros_like(colour), 1.48 - colour), axis=0)), axis=0) ** 3, + np.min((np.zeros_like(colour), colour - 1.24), axis=0), + np.max((np.zeros_like(colour), colour - 1.72), axis=0)] + b = [np.ones_like(sinBeta), sinBeta, sinBeta ** 2 - 1. / 3] + + # coefficients must be interpolated between g(left) and g(left+1) + # find the bin in g where gMag is + ig = np.max((np.zeros_like(phot_g_mean_mag), + np.min((np.ones_like(phot_g_mean_mag) * (n - 2), np.digitize(phot_g_mean_mag, g, right=False) - 1), + axis=0)), axis=0).astype(int) + + # interpolate coefficients to gMag: + h = np.max((np.zeros_like(phot_g_mean_mag), + np.min((np.ones_like(phot_g_mean_mag), (phot_g_mean_mag - g[ig]) / (g[ig + 1] - g[ig])), axis=0)), + axis=0) + + # sum over the product of the coefficients to get the zero-point + zpt = np.sum([((1 - h) * q_jk[ig, i] + h * q_jk[ig + 1, i]) * c[j[i]] * b[k[i]] for i in range(m)], axis=0) + + return zpt + + +# Main function: calculates the zero-point for any source in the Gaia catalogue + +def get_zpt(phot_g_mean_mag, nu_eff_used_in_astrometry, pseudocolour, ecl_lat, astrometric_params_solved, + _warnings=True): + """ + Returns the parallax zero point [mas] for a source of given G magnitude, effective wavenumber (nuEff) [1/micron], + pseudocolour (pc) [1/micron], and ecl_lat [degrees]. It also needs the astrometric_params_solved to discern + between 5-p and 6-p solutions. Valid for 5- and 6-parameter solutions with 6 2p, 31 -> 5p, 95 -> 6p) + + Output: + correction in mas (milliarcsecond, not micro). + """ + inputs_are_floats = False + + # check availability of the tables + try: + global j_5, k_5, g_5, q_jk5, n_5, m_5, j_6, k_6, g_6, q_jk6, n_6, m_6 + len(g_5) + len(g_6) + except: + raise ValueError("The table of coefficients have not been initialized!!\n Run load_tables().") + + # check input types + inputs = [phot_g_mean_mag, nu_eff_used_in_astrometry, pseudocolour, ecl_lat, astrometric_params_solved] + inputs_names = ['phot_g_mean_mag', 'nu_eff_used_in_astrometry', 'pseudocolour', 'ecl_lat', + 'astrometric_params_solved'] + for i, inp in enumerate(inputs): + # first: check is not an iterable + if not (isinstance(inp, np.ndarray) or isinstance(inp, list) or isinstance(inp, tuple)): + # if not an iterable, has to be int or float + if not (np.can_cast(inp, float) or np.can_cast(inp, int)): + raise ValueError( + """The input '{}' is of an unknown type. + Only types accepted are: float, int, ndarray, list or tuple.""".format(inputs_names[i])) + + # check coherence among inputs + if np.isscalar(phot_g_mean_mag): + inputs_are_floats = True + try: + phot_g_mean_mag = np.array([phot_g_mean_mag]) + nu_eff_used_in_astrometry = np.array([nu_eff_used_in_astrometry]) + pseudocolour = np.array([pseudocolour]) + ecl_lat = np.array([ecl_lat]) + astrometric_params_solved = np.array([astrometric_params_solved]) + + for inp in [phot_g_mean_mag, nu_eff_used_in_astrometry, pseudocolour, ecl_lat, astrometric_params_solved]: + inp[0] + except: + raise ValueError("The variables are not well formated! The types are not coherent among the inputs.") + + else: + phot_g_mean_mag = np.array(phot_g_mean_mag) + nu_eff_used_in_astrometry = np.array(nu_eff_used_in_astrometry) + pseudocolour = np.array(pseudocolour) + ecl_lat = np.array(ecl_lat) + astrometric_params_solved = np.array(astrometric_params_solved) + + if not ( + phot_g_mean_mag.shape == nu_eff_used_in_astrometry.shape == pseudocolour.shape == ecl_lat.shape == + astrometric_params_solved.shape): + raise ValueError("Dimension mismatch! At least one of the inputs has a different shape than the rest.") + + # ###### HERE ALL VARIABLES SHOULD BE CORRECT ######## + + # check astrometric_params_solved + if not np.all((astrometric_params_solved == 31) | (astrometric_params_solved == 95)): + raise ValueError( + """Some of the sources have an invalid number of the astrometric_params_solved and are not one of the two + possible values (31,95). Please provide an acceptable value.""") + + # define 5p and 6p sources + sources_5p = np.where(astrometric_params_solved == 31) + sources_6p = np.where(astrometric_params_solved == 95) + + # check magnitude and colour ranges + if not _warnings: + # initialise filterning arrays + gmag_outofrange_ind = None + nueff_outofrange_ind = None + pseudocolor_outofrange_ind = None + + if np.any(phot_g_mean_mag >= 21) or np.any(phot_g_mean_mag <= 6): + if _warnings: + warnings.warn( + """The apparent magnitude of one or more of the sources is outside the expected range (6-21 mag). + Outside this range, there is no further interpolation, thus the values at 6 or 21 are returned.""", + UserWarning) + # raise ValueError('The apparent magnitude of the source is outside the valid range (6-21 mag)') + else: + if inputs_are_floats: + return np.nan + else: + # return np.ones_like(phot_g_mean_mag) * np.nan + gmag_outofrange_ind = np.where((phot_g_mean_mag >= 21) | (phot_g_mean_mag <= 6)) + + if (np.any(nu_eff_used_in_astrometry[sources_5p] >= 1.9) or np.any( + nu_eff_used_in_astrometry[sources_5p] <= 1.1)): + if _warnings: + warnings.warn( + """The nu_eff_used_in_astrometry of some of the 5p source(s) is outside the expected range (1.1-1.9 + mag). Outside this range, the zero-point calculated can be seriously wrong.""", + UserWarning) + else: + if inputs_are_floats: + return np.nan + else: + nueff_outofrange_ind = np.where((astrometric_params_solved == 31) & ( + (nu_eff_used_in_astrometry >= 1.9) | (nu_eff_used_in_astrometry <= 1.1))) + + if np.any(pseudocolour[sources_6p] >= 1.72) or np.any(pseudocolour[sources_6p] <= 1.24): + if _warnings: + warnings.warn( + """The pseudocolour of some of the 6p source(s) is outside the expected range (1.24-1.72 mag). + The maximum corrections are reached already at 1.24 and 1.72""", + UserWarning) + else: + if inputs_are_floats: + return np.nan + else: + pseudocolor_outofrange_ind = np.where( + (astrometric_params_solved == 95) & ((pseudocolour >= 1.72) | (pseudocolour <= 1.24))) + + # initialise answer + zpt = np.zeros_like(phot_g_mean_mag) + + # compute zero-point for 5p + zpt[sources_5p] = _calc_zpt(phot_g_mean_mag[sources_5p], nu_eff_used_in_astrometry[sources_5p], + pseudocolour[sources_5p], np.sin(np.deg2rad(ecl_lat[sources_5p])), 5) + + # compute zero-point for 5p + zpt[sources_6p] = _calc_zpt(phot_g_mean_mag[sources_6p], nu_eff_used_in_astrometry[sources_6p], + pseudocolour[sources_6p], np.sin(np.deg2rad(ecl_lat[sources_6p])), 6) + + if inputs_are_floats: + return np.round(zpt * 0.001, 6)[0] # convert to mas + else: + zpt = np.round(zpt * 0.001, 6) # convert to mas + # if warnings are turned off, turn to NaN the sources out of range + if not _warnings: + if gmag_outofrange_ind is not None: + zpt[gmag_outofrange_ind] = np.nan * np.ones_like(gmag_outofrange_ind) + if nueff_outofrange_ind is not None: + zpt[nueff_outofrange_ind] = np.nan * np.ones_like(nueff_outofrange_ind) + if pseudocolor_outofrange_ind is not None: + zpt[pseudocolor_outofrange_ind] = np.nan * np.ones_like(pseudocolor_outofrange_ind) + return zpt + + +# A simple pandas wrapper: it should take around 2e-4 seconds/star + +def zpt_wrapper(pandas_row): + """ + Compute the parallax zero-point with get_zpt function for each row of the pandas DataFrame. It assumes that the + DataFrame has: + + - phot_g_mean_mag: apparent magnitude in the G band + - nu_eff_used_in_astrometry: effective wavenumber for a 5-parameter solution + - pseudocolour: effective wavenumber for a 6-parameter solution + - ecl_lat: ecliptic latitude in degrees + - astrometric_params_solved (3 -> 2p, 31 -> 5p, 95 -> 6p) + + Errors are set to False, therefore stars that are NOT inside the valid range of the interpolators will receive a + NaN. + + Example: df.apply(zpt_wrapper,axis=1) + """ + + return get_zpt(pandas_row.phot_g_mean_mag, pandas_row.nu_eff_used_in_astrometry, + pandas_row.pseudocolour,pandas_row.ecl_lat, + pandas_row.astrometric_params_solved, + _warnings=False) diff --git a/py/desitarget/streams/utilities.py b/py/desitarget/streams/utilities.py index 6a3658ed..107ff7f2 100644 --- a/py/desitarget/streams/utilities.py +++ b/py/desitarget/streams/utilities.py @@ -16,7 +16,7 @@ from pkg_resources import resource_filename from scipy.interpolate import UnivariateSpline from time import time -from zero_point import zero_point as gaia_zpt +import desitarget.streams.gaia_dr3_parallax_zero_point.zpt as gaia_zpt # ADM set up the DESI default logger. from desiutil.log import get_logger From 21ffde44db5998b04cbdb744d3aa93bce57ee40b Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Tue, 5 Mar 2024 15:24:09 -0800 Subject: [PATCH 22/63] link directly to the license --- py/desitarget/streams/gaia_dr3_parallax_zero_point/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py/desitarget/streams/gaia_dr3_parallax_zero_point/README.md b/py/desitarget/streams/gaia_dr3_parallax_zero_point/README.md index 79845320..be7ea5a4 100644 --- a/py/desitarget/streams/gaia_dr3_parallax_zero_point/README.md +++ b/py/desitarget/streams/gaia_dr3_parallax_zero_point/README.md @@ -9,7 +9,7 @@ for details and attribution. ## License The contents of this (and only this) directory retain the original -GNU LESSER GENERAL PUBLIC LICENSE included herein. +[GNU LESSER GENERAL PUBLIC LICENSE included herein](./LICENSE). ## Justification From 1d7f466d5fc9664af5be15e6623cd65545dfa071 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Tue, 26 Mar 2024 10:39:07 -0700 Subject: [PATCH 23/63] remove from future import boilerplate --- bin/select_stream_targets | 2 -- 1 file changed, 2 deletions(-) diff --git a/bin/select_stream_targets b/bin/select_stream_targets index f8a6499c..2ab05aa3 100755 --- a/bin/select_stream_targets +++ b/bin/select_stream_targets @@ -1,7 +1,5 @@ #!/usr/bin/env python -from __future__ import print_function, division - import os, sys import numpy as np import fitsio From 42d2b1f05afae741bd406481e71f9317abe1ceca Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Tue, 26 Mar 2024 10:41:43 -0700 Subject: [PATCH 24/63] moving clock start to inside function --- py/desitarget/streams/cuts.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/py/desitarget/streams/cuts.py b/py/desitarget/streams/cuts.py index efe5af05..9ada26a9 100644 --- a/py/desitarget/streams/cuts.py +++ b/py/desitarget/streams/cuts.py @@ -27,9 +27,6 @@ from desiutil.log import get_logger log = get_logger() -# ADM start the clock. -start = time() - def is_in_GD1(objs): """Whether a target lies within the GD1 stellar stream. @@ -52,6 +49,9 @@ def is_in_GD1(objs): :class:`array_like` ``True`` if the object is a white dwarf "FILLER" target. """ + # ADM start the clock. + start = time() + # ADM the name of the stream. stream_name = "GD1" From a24656c7b1c62f3f19ce2d1cfdca11b906cd6966 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Tue, 26 Mar 2024 11:32:15 -0700 Subject: [PATCH 25/63] set up coefficients files using importlib.resources.files --- .../streams/gaia_dr3_parallax_zero_point/zpt.py | 9 ++++++--- py/desitarget/streams/utilities.py | 16 ++++++++-------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/py/desitarget/streams/gaia_dr3_parallax_zero_point/zpt.py b/py/desitarget/streams/gaia_dr3_parallax_zero_point/zpt.py index 837e552a..cb596930 100644 --- a/py/desitarget/streams/gaia_dr3_parallax_zero_point/zpt.py +++ b/py/desitarget/streams/gaia_dr3_parallax_zero_point/zpt.py @@ -6,10 +6,13 @@ import numpy as np import warnings -__mypath = os.path.dirname(os.path.abspath(__file__)) +from importlib import resources as importlib_resources -_file5_currentversion = __mypath + '/coefficients/z5_200720.txt' -_file6_currentversion = __mypath + '/coefficients/z6_200720.txt' +__mypath = importlib_resources.files('desitarget') +__mypath = __mypath / 'streams' / 'gaia_dr3_parallax_zero_point' + +_file5_currentversion = __mypath / 'coefficients' / 'z5_200720.txt' +_file6_currentversion = __mypath / 'coefficients' / 'z6_200720.txt' # Definition of functions that load the coefficient tables and initialize the global variables diff --git a/py/desitarget/streams/utilities.py b/py/desitarget/streams/utilities.py index 107ff7f2..62f78139 100644 --- a/py/desitarget/streams/utilities.py +++ b/py/desitarget/streams/utilities.py @@ -25,8 +25,8 @@ # ADM start the clock. start = time() -# ADM load the Gaia zeropoints. -gaia_zpt.zpt.load_tables() +# ADM load the Gaia zero points. +gaia_zpt.load_tables() # ADM Galactic reference frame. Use astropy v4.0 defaults. GCPARAMS = acoo.galactocentric_frame_defaults.get_from_registry( @@ -413,12 +413,12 @@ def plx_sel_func(dist, D, mult, plx_sys=0.05): # extra plx systematic error padding plx_sys = 0.05 subset = np.in1d(D['ASTROMETRIC_PARAMS_SOLVED'], [31, 95]) - plx_zpt_tmp = gaia_zpt.zpt.get_zpt(D['PHOT_G_MEAN_MAG'][subset], - D['NU_EFF_USED_IN_ASTROMETRY'][subset], - D['PSEUDOCOLOUR'][subset], - D['ECL_LAT'][subset], - D['ASTROMETRIC_PARAMS_SOLVED'][subset], - _warnings=False) + plx_zpt_tmp = gaia_zpt.get_zpt(D['PHOT_G_MEAN_MAG'][subset], + D['NU_EFF_USED_IN_ASTROMETRY'][subset], + D['PSEUDOCOLOUR'][subset], + D['ECL_LAT'][subset], + D['ASTROMETRIC_PARAMS_SOLVED'][subset], + _warnings=False) plx_zpt = np.zeros(len(D['RA'])) plx_zpt_tmp[~np.isfinite(plx_zpt_tmp)] = 0 plx_zpt[subset] = plx_zpt_tmp From 8712eddafc0c9e5f8bf03e2036c86f2ebce839b6 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Tue, 26 Mar 2024 11:35:00 -0700 Subject: [PATCH 26/63] moving another clock start from the package level to the function level --- py/desitarget/streams/io.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/py/desitarget/streams/io.py b/py/desitarget/streams/io.py index c42b7b44..b5127fd0 100644 --- a/py/desitarget/streams/io.py +++ b/py/desitarget/streams/io.py @@ -24,9 +24,6 @@ from desiutil.log import get_logger log = get_logger() -# ADM start the clock. -start = time() - # ADM the standard data model for working with streams. streamcols = np.array([], dtype=[ ('RELEASE', '>i2'), ('BRICKID', '>i4'), ('TYPE', 'S4'), @@ -97,6 +94,9 @@ def read_data_per_stream(swdir, rapol, decpol, ra_ref, mind, maxd, stream_name, is likely a better choice for looping over the entire LS sweeps data when targeting multiple streams. """ + # ADM start the clock. + start = time() + # ADM check whether $TARG_DIR exists. If it does, agree to read from # ADM and write to the cache. writecache = True From 50e8e42b2aece7d9ebf90e2d9baae2499f2a01cb Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Tue, 26 Mar 2024 14:04:34 -0700 Subject: [PATCH 27/63] add coefficients files for Gaia zero points to setup.py --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 12a4c89d..7daaa3fd 100755 --- a/setup.py +++ b/setup.py @@ -68,7 +68,8 @@ 'desitarget.sv3': ['data/*',], 'desitarget.test': ['t/*',], 'desitarget.mock': [os.path.relpath(_,'py/desitarget/mock') for _ in [os.path.join(_[0],'*') for _ in os.walk('py/desitarget/mock/data')]], - } + 'desitarget.streams.gaia_dr3_parallax_zero_point': ['coefficients/*',], + } # # Run setup command. # From 283feeb51089b44dff389254d0685bf4f4f5c191 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Fri, 29 Mar 2024 17:13:27 -0700 Subject: [PATCH 28/63] fix incorrect call in pm12_sel_func. Should be a call to PM1TRACK(fi1), PM2TRACK(fi1) --- py/desitarget/streams/cuts.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/py/desitarget/streams/cuts.py b/py/desitarget/streams/cuts.py index 9ada26a9..6d2679d2 100644 --- a/py/desitarget/streams/cuts.py +++ b/py/desitarget/streams/cuts.py @@ -125,7 +125,8 @@ def is_in_GD1(objs): # ADM Gaia-based selection (proper motion and parallax). pm_pad = 2 # mas/yr padding in pm selection - gaia_astrom_sel = pm12_sel_func(fi1, pmfi1, pmfi2, pm_err, pm_pad, 2.5) + gaia_astrom_sel = pm12_sel_func(PM1TRACK(fi1), PM2TRACK(fi1), pmfi1, pmfi2, + pm_err, pm_pad, 2.5) gaia_astrom_sel &= plx_sel_func(fi1, objs, 2.5) gaia_astrom_sel &= r > bright_limit From e39d3a8d62f8892325c41baac160344fbb414e8c Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Fri, 29 Mar 2024 17:23:17 -0700 Subject: [PATCH 29/63] Limit to Declinations of > -20o --- py/desitarget/streams/io.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/py/desitarget/streams/io.py b/py/desitarget/streams/io.py index b5127fd0..0004346a 100644 --- a/py/desitarget/streams/io.py +++ b/py/desitarget/streams/io.py @@ -182,6 +182,9 @@ def read_data_per_stream(swdir, rapol, decpol, ra_ref, mind, maxd, stream_name, # ADM only retain objects in the stream... ii = betw(sep.value, mind, maxd) + # ADM at a declination of > -20o + ii &= objs["DEC"] > -20. + # ADM ...that aren't very faint (> 22.5 mag in r). ii &= objs["FLUX_R"] > 1 # ADM Also guard against negative fluxes in g/r. From 02ca2083528d3860d792ed480260eeae361e8e98 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Sun, 31 Mar 2024 19:47:34 -0700 Subject: [PATCH 30/63] take more care to assign Gaia and LS columns separately when constructing cache --- py/desitarget/streams/io.py | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/py/desitarget/streams/io.py b/py/desitarget/streams/io.py index 0004346a..2b68339a 100644 --- a/py/desitarget/streams/io.py +++ b/py/desitarget/streams/io.py @@ -24,13 +24,17 @@ from desiutil.log import get_logger log = get_logger() -# ADM the standard data model for working with streams. -streamcols = np.array([], dtype=[ +# ADM the Legacy Surveys part of the data model for working with streams. +streamcolsLS = np.array([], dtype=[ ('RELEASE', '>i2'), ('BRICKID', '>i4'), ('TYPE', 'S4'), ('OBJID', '>i4'), ('RA', '>f8'), ('DEC', '>f8'), ('EBV', '>f4'), ('FLUX_G', '>f4'), ('FIBERTOTFLUX_G', '>f4'), ('FLUX_R', '>f4'), ('FIBERTOTFLUX_R', '>f4'), ('FLUX_Z', '>f4'), ('FIBERTOTFLUX_Z', '>f4'), +]) + +# ADM the Gaia part of the data model for working with streams. +streamcolsGaia = np.array([], dtype=[ ('REF_EPOCH', '>f4'), ('PARALLAX', '>f4'), ('PARALLAX_IVAR', '>f4'), ('PMRA', '>f4'), ('PMRA_IVAR', '>f4'), ('PMDEC', '>f4'), ('PMDEC_IVAR', '>f4'), @@ -135,13 +139,13 @@ def read_data_per_stream(swdir, rapol, decpol, ra_ref, mind, maxd, stream_name, # ADM read both the north and south directories, if requested. if addnors: if "south" in swdir: - infiles2 = swdir.replace("south", "north") + swdir2 = swdir.replace("south", "north") elif "north" in swdir: - infiles2 = swdir.replace("north", "south") + swdir2 = swdir.replace("north", "south") else: msg = "addnors passed but swdir does not contain north or south!" raise ValueError(msg) - infiles += io.list_sweepfiles(infiles2) + infiles += io.list_sweepfiles(swdir2) # ADM calculate nside for HEALPixel of approximately 1o to limit # ADM number of sweeps files that need to be read. @@ -182,7 +186,7 @@ def read_data_per_stream(swdir, rapol, decpol, ra_ref, mind, maxd, stream_name, # ADM only retain objects in the stream... ii = betw(sep.value, mind, maxd) - # ADM at a declination of > -20o + # ADM ...at a declination of > -20o... ii &= objs["DEC"] > -20. # ADM ...that aren't very faint (> 22.5 mag in r). @@ -207,12 +211,14 @@ def read_data_per_stream(swdir, rapol, decpol, ra_ref, mind, maxd, stream_name, assert(len(gaiaobjs) == len(LSobjs)) # ADM only retain critical columns from the global data model. - data = np.zeros(len(LSobjs), dtype=streamcols.dtype) - # ADM for both Gaia and Legacy Surveys, overwriting with Gaia. - for objs in LSobjs, gaiaobjs: - sharedcols = set(data.dtype.names).intersection(set(objs.dtype.names)) - for col in sharedcols: - data[col] = objs[col] + data = np.zeros(len(LSobjs), dtype= + streamcolsLS.dtype.descr + streamcolsGaia.dtype.descr) + # ADM add data for the Legacy Surveys columns. + for col in streamcolsLS.dtype.names: + data[col] = LSobjs[col] + # ADM add data for the Gaia columns. + for col in streamcolsGaia.dtype.names: + data[col] = gaiaobjs[col] # ADM retain the data from this part of the loop. allobjs.append(data) From 6f3e99ffded2f95209b703457ca1660d2395a3a5 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Sun, 31 Mar 2024 20:00:35 -0700 Subject: [PATCH 31/63] smarter method of catching case where there are no Legacy Surveys objects that meet the basic criteria --- py/desitarget/streams/io.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/py/desitarget/streams/io.py b/py/desitarget/streams/io.py index 2b68339a..083ee3ee 100644 --- a/py/desitarget/streams/io.py +++ b/py/desitarget/streams/io.py @@ -204,24 +204,24 @@ def read_data_per_stream(swdir, rapol, decpol, ra_ref, mind, maxd, stream_name, # ADM catch the case where there are no objects meeting the cuts. if len(LSobjs) > 0: gaiaobjs = match_gaia_to_primary(LSobjs, matchrad=1., dr=gaiadr) - else: - gaiaobjs = LSobjs - - # ADM a (probably unnecessary) sanity check. - assert(len(gaiaobjs) == len(LSobjs)) - - # ADM only retain critical columns from the global data model. - data = np.zeros(len(LSobjs), dtype= - streamcolsLS.dtype.descr + streamcolsGaia.dtype.descr) - # ADM add data for the Legacy Surveys columns. - for col in streamcolsLS.dtype.names: - data[col] = LSobjs[col] - # ADM add data for the Gaia columns. - for col in streamcolsGaia.dtype.names: - data[col] = gaiaobjs[col] - - # ADM retain the data from this part of the loop. - allobjs.append(data) + + # ADM a (probably unnecessary) sanity check. + assert(len(gaiaobjs) == len(LSobjs)) + + # ADM only retain critical columns from the global data model. + data = np.zeros(len(LSobjs), dtype=streamcolsLS.dtype.descr + + streamcolsGaia.dtype.descr) + + # ADM add data for the Legacy Surveys columns. + for col in streamcolsLS.dtype.names: + data[col] = LSobjs[col] + # ADM add data for the Gaia columns. + for col in streamcolsGaia.dtype.names: + data[col] = gaiaobjs[col] + + # ADM retain the data from this part of the loop. + allobjs.append(data) + if i % 5 == 4: log.info(f"Ran {i+1}/{len(infiles)} files...t={time()-start:.1f}s") From e4e9914e5c8040b0a6bfbad3d8b93bf6e1f1e23b Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Mon, 1 Apr 2024 09:31:15 -0700 Subject: [PATCH 32/63] correct concatenation of objects in case we move to multiple streams --- py/desitarget/streams/cuts.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/py/desitarget/streams/cuts.py b/py/desitarget/streams/cuts.py index 6d2679d2..cce10a76 100644 --- a/py/desitarget/streams/cuts.py +++ b/py/desitarget/streams/cuts.py @@ -194,7 +194,7 @@ def set_target_bits(objs, stream_names=["GD1"]): Parameters ---------- - objects : :class:`~numpy.ndarray` + objs : :class:`~numpy.ndarray` numpy structured array with UPPERCASE columns needed for stream target selection. See, e.g., :func:`~desitarget.stream.cuts.is_in_GD1` for column names. @@ -291,7 +291,7 @@ def select_targets(swdir, stream_names=["GD1"], readperstream=True, objs = read_data_per_stream(swdir, rapol, decpol, ra_ref, mind, maxd, stream_name, addnors=addnors, readcache=readcache) - allobjs.append(objs) + allobjs.append(objs) objects = np.concatenate(allobjs) else: # ADM --TODO-- write loop across sweeps instead of streams. @@ -301,7 +301,7 @@ def select_targets(swdir, stream_names=["GD1"], readperstream=True, # ADM process the targets. desi_target, bgs_target, mws_target, scnd_target = set_target_bits( - objs, stream_names=[stream_name]) + objects, stream_names=stream_names) # ADM finalize the targets. # ADM anything with DESI_TARGET !=0 is truly a target. From efd5c68466831d607fceb9c91ac27768487a8ccb Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Mon, 1 Apr 2024 17:17:56 -0700 Subject: [PATCH 33/63] first attempt at incorporating new targets (such as GD1 stream targets) into primary/secondary MTL ledgers --- bin/select_stream_targets | 5 +- py/desitarget/mtl.py | 100 ++++++++++++++++++++++++++++++++++ py/desitarget/streams/cuts.py | 6 +- py/desitarget/streams/io.py | 14 ++++- 4 files changed, 120 insertions(+), 5 deletions(-) diff --git a/bin/select_stream_targets b/bin/select_stream_targets index 2ab05aa3..ad69907b 100755 --- a/bin/select_stream_targets +++ b/bin/select_stream_targets @@ -69,6 +69,9 @@ targets = cuts.select_targets( addnors=addnors, readcache=readcache ) -ntargs, outfile = io.write_targets(ns.dest, targets, hdr, streamnames=sn) +# ADM note that we hardcode for bright conditions. This could change in +# ADM the future if we add new streams or programs. +ntargs, outfile = io.write_targets(ns.dest, targets, hdr, + streamnames=sn, obscon="BRIGHT") log.info(f"{ntargs} targets written to {outfile}...t={time()-start:.1f}s") diff --git a/py/desitarget/mtl.py b/py/desitarget/mtl.py index c8cf983a..ea0b642b 100644 --- a/py/desitarget/mtl.py +++ b/py/desitarget/mtl.py @@ -1093,6 +1093,106 @@ def purge_tiles(tiles, obscon, mtldir=None, secondary=False, verbose=True): return Table(np.concatenate(gonetargs)), gonetiles +def add_to_ledgers_in_hp(targets, nside, pixlist, mtldir=None, obscon="DARK", + verbose=True): + """ + Add new targets to an existing set of ledgers. + + Parameters + ---------- + targets : :class:`~numpy.array` + Targets made by, e.g. `desitarget.streams.cuts.select_targets()`. + nside : :class:`int` + (NESTED) HEALPixel nside that corresponds to `pixlist`. + pixlist : :class:`list` or `int` + HEALPixels at `nside` at which to write the MTLs. + mtldir : :class:`str`, optional, defaults to ``None`` + Full path to the directory that hosts the MTL ledgers and the MTL + tile file. If ``None``, then look up the MTL directory from the + $MTL_DIR environment variable. + obscon : :class:`str`, optional, defaults to "DARK" + A string matching ONE obscondition in the desitarget bitmask yaml + file (i.e. in `desitarget.targetmask.obsconditions`). + Governs the sub-directory for which the ledgers are appended. + verbose : :class:`bool`, optional, defaults to ``True`` + If ``True`` then log target and file information. + + Returns + ------- + Nothing, but appends the `targets` to the appropriate ledgers in + `outdirname`. + + Notes + ----- + - Where there is a matching TARGETID, the priority and number + of observations are both set to the higher number and the + bits are merged across target classes. + """ + t0 = time() + + # ADM grab the MTL directory (in case we're relying on $MTL_DIR). + mtldir = get_mtl_dir(mtldir) + + # ADM in case an integer was passed. + pixlist = np.atleast_1d(pixlist) + + # ADM execute MTL. + mtl = make_mtl(targets, obscon, trimcols=True) + + # ADM the HEALPixel within which each target in the MTL lies. + theta, phi = np.radians(90-mtl["DEC"]), np.radians(mtl["RA"]) + mtlpix = hp.ang2pix(nside, theta, phi, nest=True) + + # ADM Run through each pixel and compare the existing and new MTLs. + for pix in pixlist: + inpix = mtlpix == pix + if np.any(inpix): + # ADM the new MTL information. + mtlinpix = mtl[inpix] + # ADM find existing targets in primary and secondary ledgers. + for resolve, scnd in zip([True, None], [False, True]): + # ADM read in the existing ledgers. + fn = io.find_target_files(mtldir, flavor="mtl", survey="main", + hp=pix, resolve=resolve, obscon=obscon, + ender="ecsv") + old = io.read_mtl_ledger(fn) + # ADM match targets in the new and existing primary MTLs. + iim, iio = match(mtlinpix["TARGETID"], old["TARGETID"]) + # ADM extract the matches and remove them from the new MTLs. + mtlmatches = mtlinpix[iim] + mtlinpix["TARGETID"][iim] = -1 + mtlinpix = mtlinpix[mtlinpix["TARGETID"] != -1] + # ADM also extract the primary/secondary ledger matches. + oldmatches = old[iio] + # ADM combine the target bits. + for col in ["DESI_TARGET", "BGS_TARGET", + "MWS_TARGET", "SCND_TARGET"]: + oldmatches[col] |= mtlmatches[col] + # ADM set NUMOBS_MORE to the larger number. + ii = mtlmatches["NUMOBS_MORE"] > oldmatches["NUMOBS_MORE"] + oldmatches["NUMOBS_MORE"][ii] = mtlmatches["NUMOBS_MORE"][ii] + # ADM set PRIORITY to larger number (+ inherit TARGET_STATE). + ii = mtlmatches["PRIORITY"] > oldmatches["PRIORITY"] + oldmatches["PRIORITY"][ii] = mtlmatches["PRIORITY"][ii] + oldmatches["TARGET_STATE"][ii] = mtlmatches["TARGET_STATE"][ii] + # ADM also need to inherit the new timestamp and code version. + oldmatches["TIMESTAMP"] = mtlmatches["TIMESTAMP"] + oldmatches["VERSION"] = mtlmatches["VERSION"] + if scnd==True: + # ADM when done matching to old secondary targets add all + # ADM the remaining new targets to the secondary ledgers. + oldmatches = np.concatenate([oldmatches, mtlinpix]) + # ADM append new state to bottom of existing file. + nt, fn = io.write_mtl( + mtldir, oldmatches, ecsv=True, survey="main", obscon=obscon, + nsidefile=nside, hpxlist=pix, scnd=scnd, append=True) + if verbose: + log.info( + f"{nt} targets appended to {fn}...t={time()-t0:.1f}s") + + return + + def make_ledger_in_hp(targets, outdirname, nside, pixlist, obscon="DARK", indirname=None, verbose=True, scnd=False, timestamp=None, append=False): diff --git a/py/desitarget/streams/cuts.py b/py/desitarget/streams/cuts.py index cce10a76..fef77f4c 100644 --- a/py/desitarget/streams/cuts.py +++ b/py/desitarget/streams/cuts.py @@ -98,7 +98,7 @@ def is_in_GD1(objs): g, r, z = [22.5 - 2.5 * np.log10(objs['FLUX_' + _]) - ext[_] for _ in 'GRZ'] - # ADM some spline function over which to interpolate. + # ADM some spline functions over which to interpolate. TRACK = scipy.interpolate.CubicSpline([-90, -70, -50, -40, -20, 0, 20], [-3, -1.5, -.2, -0., -.0, -1.2, -3]) PM1TRACK = scipy.interpolate.UnivariateSpline( @@ -144,7 +144,7 @@ def is_in_GD1(objs): stellar_locus_sel = stellar_locus_blue_sel | stellar_locus_red_sel tot = np.sum(field_sel & gaia_astrom_sel & bright_cmd_sel) - print(f"With correct astrometry AND cmd: {tot}...t={time()-start:.1f}s") + print(f"Objects meeting bright selection: {tot}...t={time()-start:.1f}s") # ADM selection for objects that lack Gaia astrometry. # ADM has type PSF and in a reasonable isochrone window. @@ -158,7 +158,7 @@ def is_in_GD1(objs): faint_sel &= startyp faint_sel &= stellar_locus_sel tot = np.sum(faint_sel & field_sel) - log.info(f"Objects that meet faint selection: {tot}...t={time()-start:.1f}s") + log.info(f"Objects meeting faint selection: {tot}...t={time()-start:.1f}s") # ADM "filler" selections. # (PSF type + blue in colour and not previously selected) diff --git a/py/desitarget/streams/io.py b/py/desitarget/streams/io.py index 083ee3ee..90985c1d 100644 --- a/py/desitarget/streams/io.py +++ b/py/desitarget/streams/io.py @@ -244,7 +244,7 @@ def read_data_per_stream(swdir, rapol, decpol, ra_ref, mind, maxd, stream_name, return allobjs -def write_targets(dirname, targs, header, streamnames=""): +def write_targets(dirname, targs, header, streamnames="", obscon=None): """Write stream targets to a FITS file. Parameters @@ -260,6 +260,13 @@ def write_targets(dirname, targs, header, streamnames=""): streamnames : :class:`str, optional Information about stream names that correspond to the targets. Included in the output filename. + obscon : :class:`str`, optional, defaults to `None` + Can pass one of "DARK" or "BRIGHT". If passed, don't write the + full set of data, rather only write targets appropriate for + "DARK" or "BRIGHT" observing conditions. The relevant + `PRIORITY_INIT` and `NUMOBS_INIT` columns will be derived from + `PRIORITY_INIT_DARK`, etc. and `filename` will have "bright" or + "dark" appended to the lowest DIRECTORY in the input `filename`. Returns ------- @@ -280,6 +287,11 @@ def write_targets(dirname, targs, header, streamnames=""): (see `/data/units.yaml`). - Mostly wraps :func:`~desitarget.io.write_with_units`. """ + # ADM limit to just BRIGHT or DARK targets, if requested. + # ADM Ignore the filename output, we'll build that on-the-fly. + if obscon is not None: + _, header, targs = io._bright_or_dark(dirname, header, targs, obscon) + # ADM construct the output filename. drs = list(set(targs["RELEASE"]//1000)) if len(drs) == 1: From 8ccc1fe87ef6da12af590ed650f7c8ba6ad163d0 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Tue, 2 Apr 2024 10:16:12 -0700 Subject: [PATCH 34/63] need to inherit SUBPRIORITY as well as PRIORITY, fix bug due to primary and secondary ledger sets having different column orders --- py/desitarget/mtl.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/py/desitarget/mtl.py b/py/desitarget/mtl.py index ea0b642b..006da4ea 100644 --- a/py/desitarget/mtl.py +++ b/py/desitarget/mtl.py @@ -1093,7 +1093,7 @@ def purge_tiles(tiles, obscon, mtldir=None, secondary=False, verbose=True): return Table(np.concatenate(gonetargs)), gonetiles -def add_to_ledgers_in_hp(targets, nside, pixlist, mtldir=None, obscon="DARK", +def add_to_ledgers_in_hp(targets, pixlist, mtldir=None, obscon="DARK", verbose=True): """ Add new targets to an existing set of ledgers. @@ -1102,10 +1102,8 @@ def add_to_ledgers_in_hp(targets, nside, pixlist, mtldir=None, obscon="DARK", ---------- targets : :class:`~numpy.array` Targets made by, e.g. `desitarget.streams.cuts.select_targets()`. - nside : :class:`int` - (NESTED) HEALPixel nside that corresponds to `pixlist`. pixlist : :class:`list` or `int` - HEALPixels at `nside` at which to write the MTLs. + HEALPixels at :func:`_get_mtl_nside()` in which to add to MTLs. mtldir : :class:`str`, optional, defaults to ``None`` Full path to the directory that hosts the MTL ledgers and the MTL tile file. If ``None``, then look up the MTL directory from the @@ -1130,6 +1128,9 @@ def add_to_ledgers_in_hp(targets, nside, pixlist, mtldir=None, obscon="DARK", """ t0 = time() + # ADM get the standard nside. + nside = _get_mtl_nside() + # ADM grab the MTL directory (in case we're relying on $MTL_DIR). mtldir = get_mtl_dir(mtldir) @@ -1149,6 +1150,8 @@ def add_to_ledgers_in_hp(targets, nside, pixlist, mtldir=None, obscon="DARK", if np.any(inpix): # ADM the new MTL information. mtlinpix = mtl[inpix] + if verbose: + log.info(f"Working with {len(mtlinpix)} targets in pixel {pix}") # ADM find existing targets in primary and secondary ledgers. for resolve, scnd in zip([True, None], [False, True]): # ADM read in the existing ledgers. @@ -1171,17 +1174,23 @@ def add_to_ledgers_in_hp(targets, nside, pixlist, mtldir=None, obscon="DARK", # ADM set NUMOBS_MORE to the larger number. ii = mtlmatches["NUMOBS_MORE"] > oldmatches["NUMOBS_MORE"] oldmatches["NUMOBS_MORE"][ii] = mtlmatches["NUMOBS_MORE"][ii] - # ADM set PRIORITY to larger number (+ inherit TARGET_STATE). + # ADM set PRIORITY to larger number + # ADM (+ inherit TARGET_STATE and SUBPRIORITY). ii = mtlmatches["PRIORITY"] > oldmatches["PRIORITY"] oldmatches["PRIORITY"][ii] = mtlmatches["PRIORITY"][ii] + oldmatches["SUBPRIORITY"][ii] = mtlmatches["SUBPRIORITY"][ii] oldmatches["TARGET_STATE"][ii] = mtlmatches["TARGET_STATE"][ii] # ADM also need to inherit the new timestamp and code version. oldmatches["TIMESTAMP"] = mtlmatches["TIMESTAMP"] oldmatches["VERSION"] = mtlmatches["VERSION"] - if scnd==True: - # ADM when done matching to old secondary targets add all - # ADM the remaining new targets to the secondary ledgers. - oldmatches = np.concatenate([oldmatches, mtlinpix]) + if scnd: + # ADM when done matching to old secondaries add all + # ADM remaining new targets to the secondary ledgers. + # ADM make sure to retain "secondary" column order. + reordered = np.zeros(len(mtlinpix), dtype=oldmatches.dtype.descr) + for col in reordered.dtype.names: + reordered[col] = mtlinpix[col] + oldmatches = np.concatenate([oldmatches, reordered]) # ADM append new state to bottom of existing file. nt, fn = io.write_mtl( mtldir, oldmatches, ecsv=True, survey="main", obscon=obscon, From 4bc86967df8cb05179020e38536177a524c8d88d Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Tue, 2 Apr 2024 14:34:10 -0700 Subject: [PATCH 35/63] we'll need to be able to force an identical timestamp for updates --- py/desitarget/mtl.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/py/desitarget/mtl.py b/py/desitarget/mtl.py index 006da4ea..409c5b86 100644 --- a/py/desitarget/mtl.py +++ b/py/desitarget/mtl.py @@ -1094,7 +1094,7 @@ def purge_tiles(tiles, obscon, mtldir=None, secondary=False, verbose=True): def add_to_ledgers_in_hp(targets, pixlist, mtldir=None, obscon="DARK", - verbose=True): + timestamp=None, verbose=True): """ Add new targets to an existing set of ledgers. @@ -1112,6 +1112,8 @@ def add_to_ledgers_in_hp(targets, pixlist, mtldir=None, obscon="DARK", A string matching ONE obscondition in the desitarget bitmask yaml file (i.e. in `desitarget.targetmask.obsconditions`). Governs the sub-directory for which the ledgers are appended. + timestamp : :class:`str`, optional + A timestamp to use in place of that assigned by `make_mtl`. verbose : :class:`bool`, optional, defaults to ``True`` If ``True`` then log target and file information. @@ -1127,6 +1129,8 @@ def add_to_ledgers_in_hp(targets, pixlist, mtldir=None, obscon="DARK", bits are merged across target classes. """ t0 = time() + # ADM a dictionary to hold header keywords for the ouput file. + hdr = {} # ADM get the standard nside. nside = _get_mtl_nside() @@ -1140,6 +1144,13 @@ def add_to_ledgers_in_hp(targets, pixlist, mtldir=None, obscon="DARK", # ADM execute MTL. mtl = make_mtl(targets, obscon, trimcols=True) + # ADM if requested, substitute a bespoke timestamp. + if timestamp is not None: + # ADM check the timestamp is valid. + _ = check_timestamp(timestamp) + hdr["TSFORCED"] = timestamp + mtl["TIMESTAMP"] = timestamp + # ADM the HEALPixel within which each target in the MTL lies. theta, phi = np.radians(90-mtl["DEC"]), np.radians(mtl["RA"]) mtlpix = hp.ang2pix(nside, theta, phi, nest=True) @@ -1194,7 +1205,8 @@ def add_to_ledgers_in_hp(targets, pixlist, mtldir=None, obscon="DARK", # ADM append new state to bottom of existing file. nt, fn = io.write_mtl( mtldir, oldmatches, ecsv=True, survey="main", obscon=obscon, - nsidefile=nside, hpxlist=pix, scnd=scnd, append=True) + nsidefile=nside, hpxlist=pix, scnd=scnd, extra=hdr, + append=True) if verbose: log.info( f"{nt} targets appended to {fn}...t={time()-t0:.1f}s") From 5408bcdbf4557a6e600264fab93c7b9126099172 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Tue, 2 Apr 2024 15:43:58 -0700 Subject: [PATCH 36/63] add a SUBPRIORITY to the stream targets --- py/desitarget/streams/io.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/py/desitarget/streams/io.py b/py/desitarget/streams/io.py index 90985c1d..429c8977 100644 --- a/py/desitarget/streams/io.py +++ b/py/desitarget/streams/io.py @@ -244,7 +244,8 @@ def read_data_per_stream(swdir, rapol, decpol, ra_ref, mind, maxd, stream_name, return allobjs -def write_targets(dirname, targs, header, streamnames="", obscon=None): +def write_targets(dirname, targs, header, streamnames="", obscon=None, + subpriority=True): """Write stream targets to a FITS file. Parameters @@ -267,6 +268,10 @@ def write_targets(dirname, targs, header, streamnames="", obscon=None): `PRIORITY_INIT` and `NUMOBS_INIT` columns will be derived from `PRIORITY_INIT_DARK`, etc. and `filename` will have "bright" or "dark" appended to the lowest DIRECTORY in the input `filename`. + subpriority : :class:`bool`, optional, defaults to ``True`` + If ``True`` and a `SUBPRIORITY` column is in the input `targs`, + then `SUBPRIORITY==0.0` entries are overwritten by a random float + in the range 0 to 1, using a seed of 816. Returns ------- @@ -316,6 +321,18 @@ def write_targets(dirname, targs, header, streamnames="", obscon=None): if np.any(toobright): tids = targs["TARGETID"][toobright] log.warning(f"Targets TOO BRIGHT to be written to {outfn}: {tids}") + # ADM remove the targets that are too bright. + targs = targs[~toobright] + + # ADM populate SUBPRIORITY with a reproducible random float. + if "SUBPRIORITY" in targs.dtype.names and subpriority: + subpseed = 816 + np.random.seed(subpseed) + # SB only set subpriorities that aren't already set, but keep + # original full random sequence order. + ii = targs["SUBPRIORITY"] == 0.0 + targs["SUBPRIORITY"][ii] = np.random.random(len(targs))[ii] + header["SUBPSEED"] = subpseed # ADM add the DESI dependencies. depend.add_dependencies(header) From 8bfa8793fb2222fe589bfc9c86313cb18e571e94 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Thu, 4 Apr 2024 12:19:11 -0700 Subject: [PATCH 37/63] add function to loop through and add to all of the ledgers in parallel --- py/desitarget/mtl.py | 185 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 178 insertions(+), 7 deletions(-) diff --git a/py/desitarget/mtl.py b/py/desitarget/mtl.py index 409c5b86..0c0ec99b 100644 --- a/py/desitarget/mtl.py +++ b/py/desitarget/mtl.py @@ -46,22 +46,49 @@ ('NUMOBS', '>i8'), ('NUMOBS_MORE', '>i8'), ('Z', '>f8'), ('ZWARN', '>i8'), ('TIMESTAMP', 'U25'), ('VERSION', 'U14'), ('TARGET_STATE', 'U30'), ('ZTILEID', '>i4') - ]) +]) + +# ADM at some point the primary and secondary data models for the MTLs +# ADM are trimmed and reordered. These record their exact format on disk. +mtlprimdatamodel = np.array([], dtype=[ + ('RA', 'f8'), ('IS_QSO_QN', '>i2'), ('DELTACHI2', '>f8'), - ]) +]) zcatdatamodel = np.array([], dtype=[ ('RA', '>f8'), ('DEC', '>f8'), ('TARGETID', '>i8'), ('NUMOBS', '>i4'), ('Z', '>f8'), ('ZWARN', '>i8'), ('ZTILEID', '>i4') - ]) +]) mtltilefiledm = np.array([], dtype=[ ('TILEID', '>i4'), ('TIMESTAMP', 'U25'), ('VERSION', 'U14'), ('PROGRAM', 'U6'), ('ZDATE', '>i8'), ('ARCHIVEDATE', '>i8') - ]) +]) # ADM when using basic or csv ascii writes, specifying the formats of @@ -1094,7 +1121,7 @@ def purge_tiles(tiles, obscon, mtldir=None, secondary=False, verbose=True): def add_to_ledgers_in_hp(targets, pixlist, mtldir=None, obscon="DARK", - timestamp=None, verbose=True): + timestamp=None, verbose=True, updatedonefiles=False): """ Add new targets to an existing set of ledgers. @@ -1116,11 +1143,14 @@ def add_to_ledgers_in_hp(targets, pixlist, mtldir=None, obscon="DARK", A timestamp to use in place of that assigned by `make_mtl`. verbose : :class:`bool`, optional, defaults to ``True`` If ``True`` then log target and file information. + updatedonefiles: :class:`bool`, optional, defaults to ``True`` + If ``False`` then do NOT write a timestamp to the MTL + override "done" files indicating an update has occurred. Returns ------- Nothing, but appends the `targets` to the appropriate ledgers in - `outdirname`. + the `mtldir`. Notes ----- @@ -1169,7 +1199,14 @@ def add_to_ledgers_in_hp(targets, pixlist, mtldir=None, obscon="DARK", fn = io.find_target_files(mtldir, flavor="mtl", survey="main", hp=pix, resolve=resolve, obscon=obscon, ender="ecsv") - old = io.read_mtl_ledger(fn) + try: + old = io.read_mtl_ledger(fn) + # ADM the file may not exist, in which case work with an + # ADM empty set of "old" information. + except FileNotFoundError: + old = np.zeros(0, dtype=mtlprimdatamodel.dtype) + if scnd: + old = np.zeros(0, dtype=mtlsecdatamodel.dtype) # ADM match targets in the new and existing primary MTLs. iim, iio = match(mtlinpix["TARGETID"], old["TARGETID"]) # ADM extract the matches and remove them from the new MTLs. @@ -1210,6 +1247,140 @@ def add_to_ledgers_in_hp(targets, pixlist, mtldir=None, obscon="DARK", if verbose: log.info( f"{nt} targets appended to {fn}...t={time()-t0:.1f}s") + else: + log.info(f"No targets in pixel {pix}") + + # ADM Note the update in the MTL tile file. + if updatedonefiles: + for sec in [True, False]: + mtltilefn = os.path.join( + mtldir, get_mtl_tile_file_name(secondary=sec, override=True)) + # ADM initialize the output array and add the tiles. + mocktiles = np.zeros(1, dtype=mtltilefiledm.dtype) + mocktiles["TIMESTAMP"] = timestamp + # ADM add the version of desitarget. + mocktiles["VERSION"] = dt_version + # ADM add the program/obscon. + mocktiles["PROGRAM"] = obscon + # ADM special numbers to indicate this type of update. + mocktiles["TILEID"] = -1 + mocktiles["ZDATE"] = -1 + mocktiles["ARCHIVEDATE"] = -1 + + # ADM write to file. + io.write_mtl_tile_file(mtltilefn, mocktiles) + + return + + +def add_to_ledgers(targs, mtldir=None, pixlist=None, obscon="DARK", + numproc=1, updatedonefiles=True): + """ + Make initial MTL ledger files for HEALPixels, in parallel. + + Parameters + ---------- + targs : :class:`str` or `~numpy.ndarray` + Full path to a file containing targets to be added to ledgers, or + the targets themselves. A string is interpreted as a filename, + anything else is interpreted as an array of targets. + mtldir : :class:`str`, optional, defaults to ``None`` + Full path to the directory that hosts the MTL ledgers and the MTL + tile file. If ``None``, then look up the MTL directory from the + $MTL_DIR environment variable. + pixlist : :class:`list` or `int`, defaults to ``None`` + (Nested) HEALPixels for which to write the MTLs at the default + `nside` (which is `_get_mtl_nside()`). Defaults to ``None``, + which runs all of the pixels at `_get_mtl_nside()` that are + present in the input `targs`. + obscon : :class:`str`, optional, defaults to "DARK" + A string matching ONE obscondition in the desitarget bitmask yaml + file (i.e. in `desitarget.targetmask.obsconditions`), e.g. "DARK" + Governs how priorities are set based on "obsconditions". Also + governs the sub-directory to which the ledger is written. + numproc : :class:`int`, optional, defaults to 1 for serial + Number of processes to parallelize across. + updatedonefiles: :class:`bool`, optional, defaults to ``True`` + If ``False`` then do NOT write a timestamp to the MTL + override "done" files indicating an update has occurred. + USE WITH CAUTION. In general, the alt-MTL schema needs to + know when updates happened. + + Returns + ------- + Nothing, but appends the `targs` to the appropriate ledgers in + the `mtldir`. + """ + # ADM a universal timestamp for use in all the new ledger entries. + timestamp = get_utc_date(survey="main") + + # ADM read in the targets, if necessary. + if isinstance(targs, str): + targs = fitsio.read(targs) + + # ADM grab the MTL directory (in case we're relying on $MTL_DIR). + mtldir = get_mtl_dir(mtldir) + + # ADM the nside at which to append to the MTLs. + mtlnside = _get_mtl_nside() + # ADM default to running pixels covered by the input targets. + if pixlist is None: + theta, phi = np.radians(90-targs["DEC"]), np.radians(targs["RA"]) + pixlist = np.unique(hp.ang2pix(mtlnside, theta, phi, nest=True)) + npixels = len(pixlist) + + # ADM the common function that is actually parallelized across. + def _add_to_ledgers_in_hp(pixnum): + """add to ledgers in a single HEALPixel""" + # ADM write MTLs for the targets split over HEALPixels in pixlist. + # ADM note, here that we don't want to update the done files for + # ADM EVERY pixel. Just once at the end. + return add_to_ledgers_in_hp(targs, pixnum, mtldir=mtldir, obscon=obscon, + timestamp=timestamp, updatedonefiles=False) + + # ADM this is just to count pixels in _update_status. + npix = np.ones((), dtype='i8') + t0 = time() + + def _update_status(result): + """wrap key reduction operation on the main parallel process""" + if npix % 2 == 0 and npix > 0: + rate = (time() - t0) / npix + log.info(f"Updated {npix}/{npixels} HEALPixels; {rate:.1f}" + f"secs/pixel...t = {(time()-t0)/60.:.1f} mins") + npix[...] += 1 + return result + + # ADM Parallel process across HEALPixels. + if numproc > 1: + pool = sharedmem.MapReduce(np=numproc) + with pool: + pool.map(_add_to_ledgers_in_hp, pixlist, reduce=_update_status) + else: + for pixel in pixlist: + _update_status(_add_to_ledgers_in_hp(pixel)) + + # ADM Note the update in the MTL tile file. + if updatedonefiles: + for sec in [True, False]: + mtltilefn = os.path.join( + mtldir, get_mtl_tile_file_name(secondary=sec, override=True)) + # ADM initialize the output array and add the tiles. + mocktiles = np.zeros(1, dtype=mtltilefiledm.dtype) + mocktiles["TIMESTAMP"] = timestamp + # ADM add the version of desitarget. + mocktiles["VERSION"] = dt_version + # ADM add the program/obscon. + mocktiles["PROGRAM"] = obscon + # ADM special numbers to indicate this type of update. + mocktiles["TILEID"] = -1 + mocktiles["ZDATE"] = -1 + mocktiles["ARCHIVEDATE"] = -1 + + # ADM write to file. + io.write_mtl_tile_file(mtltilefn, mocktiles) + + log.info(f"Updated ledgers in {mtldir}...t = {(time()-t0)/60.:.1f} mins") return From 7d965b9ee92c43092c6988fd6ad5b33feac1c418 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Thu, 4 Apr 2024 12:32:04 -0700 Subject: [PATCH 38/63] switch over to NUMOBS_MORE behavior being set by the highest-priority target --- py/desitarget/mtl.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/py/desitarget/mtl.py b/py/desitarget/mtl.py index 0c0ec99b..7e0b15a6 100644 --- a/py/desitarget/mtl.py +++ b/py/desitarget/mtl.py @@ -1219,13 +1219,11 @@ def add_to_ledgers_in_hp(targets, pixlist, mtldir=None, obscon="DARK", for col in ["DESI_TARGET", "BGS_TARGET", "MWS_TARGET", "SCND_TARGET"]: oldmatches[col] |= mtlmatches[col] - # ADM set NUMOBS_MORE to the larger number. - ii = mtlmatches["NUMOBS_MORE"] > oldmatches["NUMOBS_MORE"] - oldmatches["NUMOBS_MORE"][ii] = mtlmatches["NUMOBS_MORE"][ii] # ADM set PRIORITY to larger number - # ADM (+ inherit TARGET_STATE and SUBPRIORITY). + # ADM (+ inherit TARGET_STATE, SUBPRIORITY, NUMOBS_MORE). ii = mtlmatches["PRIORITY"] > oldmatches["PRIORITY"] oldmatches["PRIORITY"][ii] = mtlmatches["PRIORITY"][ii] + oldmatches["NUMOBS_MORE"][ii] = mtlmatches["NUMOBS_MORE"][ii] oldmatches["SUBPRIORITY"][ii] = mtlmatches["SUBPRIORITY"][ii] oldmatches["TARGET_STATE"][ii] = mtlmatches["TARGET_STATE"][ii] # ADM also need to inherit the new timestamp and code version. From 803924ae10b724243540e3b2af6986ee366dcb28 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Thu, 4 Apr 2024 12:47:04 -0700 Subject: [PATCH 39/63] better align inherited zero point code with PEP 8 standards --- .../gaia_dr3_parallax_zero_point/zpt.py | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/py/desitarget/streams/gaia_dr3_parallax_zero_point/zpt.py b/py/desitarget/streams/gaia_dr3_parallax_zero_point/zpt.py index cb596930..ef72f6a7 100644 --- a/py/desitarget/streams/gaia_dr3_parallax_zero_point/zpt.py +++ b/py/desitarget/streams/gaia_dr3_parallax_zero_point/zpt.py @@ -46,7 +46,7 @@ def load_tables(file5=_file5_currentversion, file6=_file6_currentversion, sep=', Initialises the tables containing the coefficients of the interpolations for the Z5 and Z6 functions. NOTE: USE THE DEFAULT VALUES unless you are very sure of what you are doing. - + Inputs file5: path to the file with the Z5 coefficients (.txt or similar) file6: path to the file with the Z6 coefficients (.txt or similar) @@ -64,15 +64,15 @@ def load_tables(file5=_file5_currentversion, file6=_file6_currentversion, sep=', # astrometric_params_solved def _calc_zpt(phot_g_mean_mag, nu_eff_used_in_astrometry, pseudocolour, sinBeta, source_type): - """ + """ Compute the zero-point parallax for an array of stars. - + WARNING! This function is meant to be auxiliary, therefore it assumes that the inputs are well formatted (see get_zpt()) and that all the sources have the same value for astrometric_params_solved. That is, either all are 5p (source_type: 5) or 6p (source_type: 6). Never 2p. """ - # load the right coefficients: + # load the right coefficients: if source_type == 5: colour = nu_eff_used_in_astrometry j, k, g, q_jk, n, m = j_5, k_5, g_5, q_jk5, n_5, m_5 @@ -141,7 +141,7 @@ def get_zpt(phot_g_mean_mag, nu_eff_used_in_astrometry, pseudocolour, ecl_lat, a try: global j_5, k_5, g_5, q_jk5, n_5, m_5, j_6, k_6, g_6, q_jk6, n_6, m_6 len(g_5) + len(g_6) - except: + except NameError: raise ValueError("The table of coefficients have not been initialized!!\n Run load_tables().") # check input types @@ -154,7 +154,7 @@ def get_zpt(phot_g_mean_mag, nu_eff_used_in_astrometry, pseudocolour, ecl_lat, a # if not an iterable, has to be int or float if not (np.can_cast(inp, float) or np.can_cast(inp, int)): raise ValueError( - """The input '{}' is of an unknown type. + """The input '{}' is of an unknown type. Only types accepted are: float, int, ndarray, list or tuple.""".format(inputs_names[i])) # check coherence among inputs @@ -189,7 +189,7 @@ def get_zpt(phot_g_mean_mag, nu_eff_used_in_astrometry, pseudocolour, ecl_lat, a # check astrometric_params_solved if not np.all((astrometric_params_solved == 31) | (astrometric_params_solved == 95)): raise ValueError( - """Some of the sources have an invalid number of the astrometric_params_solved and are not one of the two + """Some of the sources have an invalid number of the astrometric_params_solved and are not one of the two possible values (31,95). Please provide an acceptable value.""") # define 5p and 6p sources @@ -206,7 +206,7 @@ def get_zpt(phot_g_mean_mag, nu_eff_used_in_astrometry, pseudocolour, ecl_lat, a if np.any(phot_g_mean_mag >= 21) or np.any(phot_g_mean_mag <= 6): if _warnings: warnings.warn( - """The apparent magnitude of one or more of the sources is outside the expected range (6-21 mag). + """The apparent magnitude of one or more of the sources is outside the expected range (6-21 mag). Outside this range, there is no further interpolation, thus the values at 6 or 21 are returned.""", UserWarning) # raise ValueError('The apparent magnitude of the source is outside the valid range (6-21 mag)') @@ -221,7 +221,7 @@ def get_zpt(phot_g_mean_mag, nu_eff_used_in_astrometry, pseudocolour, ecl_lat, a nu_eff_used_in_astrometry[sources_5p] <= 1.1)): if _warnings: warnings.warn( - """The nu_eff_used_in_astrometry of some of the 5p source(s) is outside the expected range (1.1-1.9 + """The nu_eff_used_in_astrometry of some of the 5p source(s) is outside the expected range (1.1-1.9 mag). Outside this range, the zero-point calculated can be seriously wrong.""", UserWarning) else: @@ -282,14 +282,14 @@ def zpt_wrapper(pandas_row): - pseudocolour: effective wavenumber for a 6-parameter solution - ecl_lat: ecliptic latitude in degrees - astrometric_params_solved (3 -> 2p, 31 -> 5p, 95 -> 6p) - + Errors are set to False, therefore stars that are NOT inside the valid range of the interpolators will receive a NaN. - + Example: df.apply(zpt_wrapper,axis=1) """ return get_zpt(pandas_row.phot_g_mean_mag, pandas_row.nu_eff_used_in_astrometry, - pandas_row.pseudocolour,pandas_row.ecl_lat, + pandas_row.pseudocolour, pandas_row.ecl_lat, pandas_row.astrometric_params_solved, _warnings=False) From 005d2515ca759ed34844ae3f6d00c3941fbfb2e2 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Thu, 4 Apr 2024 13:49:33 -0700 Subject: [PATCH 40/63] First argument of plx_sel_func should be dist! --- py/desitarget/streams/cuts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py/desitarget/streams/cuts.py b/py/desitarget/streams/cuts.py index fef77f4c..5c39a13a 100644 --- a/py/desitarget/streams/cuts.py +++ b/py/desitarget/streams/cuts.py @@ -127,7 +127,7 @@ def is_in_GD1(objs): pm_pad = 2 # mas/yr padding in pm selection gaia_astrom_sel = pm12_sel_func(PM1TRACK(fi1), PM2TRACK(fi1), pmfi1, pmfi2, pm_err, pm_pad, 2.5) - gaia_astrom_sel &= plx_sel_func(fi1, objs, 2.5) + gaia_astrom_sel &= plx_sel_func(dist, objs, 2.5) gaia_astrom_sel &= r > bright_limit log.info(f"Objects in the field: {field_sel.sum()}...t={time()-start:.1f}s") From 5596137824308b174e8fbfae52c0a937be9a73d9 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Sun, 7 Apr 2024 14:22:38 -0700 Subject: [PATCH 41/63] add init.py to new directories to facilitate importing --- py/desitarget/streams/__init__.py | 0 py/desitarget/streams/gaia_dr3_parallax_zero_point/__init__.py | 0 py/desitarget/streams/io.py | 3 ++- 3 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 py/desitarget/streams/__init__.py create mode 100644 py/desitarget/streams/gaia_dr3_parallax_zero_point/__init__.py diff --git a/py/desitarget/streams/__init__.py b/py/desitarget/streams/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/py/desitarget/streams/gaia_dr3_parallax_zero_point/__init__.py b/py/desitarget/streams/gaia_dr3_parallax_zero_point/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/py/desitarget/streams/io.py b/py/desitarget/streams/io.py index 429c8977..ccb0c546 100644 --- a/py/desitarget/streams/io.py +++ b/py/desitarget/streams/io.py @@ -35,7 +35,8 @@ # ADM the Gaia part of the data model for working with streams. streamcolsGaia = np.array([], dtype=[ - ('REF_EPOCH', '>f4'), ('PARALLAX', '>f4'), ('PARALLAX_IVAR', '>f4'), + ('REF_EPOCH', '>f4'), ('SOURCE_ID', '>i8'), + ('PARALLAX', '>f4'), ('PARALLAX_IVAR', '>f4'), ('PMRA', '>f4'), ('PMRA_IVAR', '>f4'), ('PMDEC', '>f4'), ('PMDEC_IVAR', '>f4'), ('ASTROMETRIC_PARAMS_SOLVED', '>i1'), ('NU_EFF_USED_IN_ASTROMETRY', '>f4'), From ab477a898170fff424b4286b885e137d3081c8ed Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Fri, 12 Apr 2024 17:31:07 -0700 Subject: [PATCH 42/63] parallelize building the cache; try to make the cache resemble the original Gaia data --- py/desitarget/streams/io.py | 176 +++++++++++++++++++---------- py/desitarget/streams/utilities.py | 41 +++++++ 2 files changed, 160 insertions(+), 57 deletions(-) diff --git a/py/desitarget/streams/io.py b/py/desitarget/streams/io.py index ccb0c546..6608e44f 100644 --- a/py/desitarget/streams/io.py +++ b/py/desitarget/streams/io.py @@ -17,7 +17,9 @@ from desitarget.geomask import pixarea2nside, add_hp_neighbors, sweep_files_touch_hp from desitarget.gaiamatch import match_gaia_to_primary from desitarget.targets import resolve -from desitarget.streams.utilities import betw +from desitarget.streams.utilities import betw, ivars_to_errors +from desitarget.internal import sharedmem + from desiutil import depend # ADM set up the DESI default logger. @@ -35,10 +37,10 @@ # ADM the Gaia part of the data model for working with streams. streamcolsGaia = np.array([], dtype=[ - ('REF_EPOCH', '>f4'), ('SOURCE_ID', '>i8'), - ('PARALLAX', '>f4'), ('PARALLAX_IVAR', '>f4'), - ('PMRA', '>f4'), ('PMRA_IVAR', '>f4'), - ('PMDEC', '>f4'), ('PMDEC_IVAR', '>f4'), + ('REF_EPOCH', '>f4'), ('REF_ID', '>i8'), + ('PARALLAX', '>f4'), ('PARALLAX_ERROR', '>f4'), + ('PMRA', '>f4'), ('PMRA_ERROR', '>f4'), + ('PMDEC', '>f4'), ('PMDEC_ERROR', '>f4'), ('ASTROMETRIC_PARAMS_SOLVED', '>i1'), ('NU_EFF_USED_IN_ASTROMETRY', '>f4'), ('PSEUDOCOLOUR', '>f4'), ('ECL_LAT', '>f8'), ('PHOT_G_MEAN_MAG', '>f4'), ('PHOT_BP_MEAN_MAG', '>f4'), ('PHOT_RP_MEAN_MAG', '>f4') @@ -48,8 +50,87 @@ gaiadr = "dr3" -def read_data_per_stream(swdir, rapol, decpol, ra_ref, mind, maxd, stream_name, - readcache=True, addnors=True, test=False): +def read_data_per_stream_one_file(filename, rapol, decpol, mind, maxd): + """Assemble the data needed for a stream program from one file + + Parameters + ---------- + swdir : :class:`str` + Name of a Legacy Surveys sweep file. + rapol, decpol : :class:`float` + Pole in the stream coordinate system in DEGREES. + mind, maxd : :class:`float` or `int` + Minimum and maximum angular distance from the pole of the stream + coordinate system to search for members in DEGREES. + + Returns + ------- + :class:`array_like` + An array of objects from the filename that are in the stream, + with matched Gaia information. + """ + objs = io.read_tractor(filename) + + # ADM codinates of the stream. + cstream = acoo.SkyCoord(rapol*auni.degree, decpol*auni.degree) + cobjs = acoo.SkyCoord(objs["RA"]*auni.degree, objs["DEC"]*auni.degree) + + # ADM separation between the objects of interest and the stream. + sep = cobjs.separation(cstream) + + # ADM only retain objects in the stream... + ii = betw(sep.value, mind, maxd) + + # ADM ...at a declination of > -20o... + ii &= objs["DEC"] > -20. + + # ADM ...that aren't very faint (> 22.5 mag in r). + ii &= objs["FLUX_R"] > 1 + # ADM Also guard against negative fluxes in g/r. + ii &= objs["FLUX_G"] > 0. + ii &= objs["FLUX_Z"] > 0. + + objs = objs[ii] + + # ADM limit to northern objects in northern imaging and southern + # ADM objects in southern imaging. + LSobjs = resolve(objs) + + # ADM set up an an output array, and only retain critical columns + # ADM from the global data model. + data = np.zeros(len(LSobjs), dtype=streamcolsLS.dtype.descr + + streamcolsGaia.dtype.descr) + + # ADM catch the case where there are no objects meeting the cuts. + if len(LSobjs) > 0: + gaiaobjs = match_gaia_to_primary(LSobjs, matchrad=1., dr=gaiadr) + + # ADM to try and better resemble Gaia data, set zero + # ADM magnitudes and proper motions to NaNs and change + # ADM IVARs back to errors. + gaiaobjs = ivars_to_errors( + gaiaobjs, colnames=["PARALLAX_IVAR", "PMRA_IVAR", "PMDEC_IVAR"]) + + for col in ["PHOT_G_MEAN_MAG", "PHOT_BP_MEAN_MAG", "PHOT_RP_MEAN_MAG" + , "PMRA", "PMDEC"]: + ii = gaiaobjs[col] < 1e-16 + gaiaobjs[ii][col] = np.nan + + # ADM a (probably unnecessary) sanity check. + assert(len(gaiaobjs) == len(LSobjs)) + + # ADM add data for the Legacy Surveys columns. + for col in streamcolsLS.dtype.names: + data[col] = LSobjs[col] + # ADM add data for the Gaia columns. + for col in streamcolsGaia.dtype.names: + data[col] = gaiaobjs[col] + + return data + + +def read_data_per_stream(swdir, rapol, decpol, mind, maxd, stream_name, + readcache=True, addnors=True, test=False, numproc=1): """Assemble the data needed for a particular stream program. Parameters @@ -60,8 +141,6 @@ def read_data_per_stream(swdir, rapol, decpol, ra_ref, mind, maxd, stream_name, "/global/cfs/cdirs/cosmo/data/legacysurvey/dr9/south/sweep/9.0". rapol, decpol : :class:`float` Pole in the stream coordinate system in DEGREES. - ra_ref : :class:`float` - Zero latitude in the stream coordinate system in DEGREES. mind, maxd : :class:`float` or `int` Minimum and maximum angular distance from the pole of the stream coordinate system to search for members in DEGREES. @@ -81,6 +160,9 @@ def read_data_per_stream(swdir, rapol, decpol, ra_ref, mind, maxd, stream_name, north by substituting "north" in place of "south"). test : :class:`bool` Read a subset of the data for testing purposes. + numproc : :class:`int`, optional, defaults to 1 for serial + The number of parallel processes to use. `numproc` of 16 is a + good balance between speed and file I/O. Returns ------- @@ -91,7 +173,7 @@ def read_data_per_stream(swdir, rapol, decpol, ra_ref, mind, maxd, stream_name, ----- - Example values for, e.g., GD1: swdir = "/global/cfs/cdirs/cosmo/data/legacysurvey/dr9/south/sweep/9.0" - rapol, decpol, ra_ref = 34.5987, 29.7331, 200 + rapol, decpol = 34.5987, 29.7331 mind, maxd = 80, 100 - The $TARG_DIR environment variable must be set to read/write from a cache. If $TARG_DIR is not set, caching is completely ignored. @@ -177,54 +259,34 @@ def read_data_per_stream(swdir, rapol, decpol, ra_ref, mind, maxd, stream_name, log.info(msg) infiles = infiles[:20] - # ADM loop through the sweep files and limit to objects in the stream. - allobjs = [] - for i, filename in enumerate(infiles): - objs = io.read_tractor(filename) - cobjs = acoo.SkyCoord(objs["RA"]*auni.degree, objs["DEC"]*auni.degree) - sep = cobjs.separation(cstream) - - # ADM only retain objects in the stream... - ii = betw(sep.value, mind, maxd) - - # ADM ...at a declination of > -20o... - ii &= objs["DEC"] > -20. - - # ADM ...that aren't very faint (> 22.5 mag in r). - ii &= objs["FLUX_R"] > 1 - # ADM Also guard against negative fluxes in g/r. - ii &= objs["FLUX_G"] > 0. - ii &= objs["FLUX_Z"] > 0. - - objs = objs[ii] - - # ADM limit to northern objects in northern imaging and southern - # ADM objects in southern imaging. - LSobjs = resolve(objs) - - # ADM catch the case where there are no objects meeting the cuts. - if len(LSobjs) > 0: - gaiaobjs = match_gaia_to_primary(LSobjs, matchrad=1., dr=gaiadr) - - # ADM a (probably unnecessary) sanity check. - assert(len(gaiaobjs) == len(LSobjs)) - - # ADM only retain critical columns from the global data model. - data = np.zeros(len(LSobjs), dtype=streamcolsLS.dtype.descr + - streamcolsGaia.dtype.descr) - - # ADM add data for the Legacy Surveys columns. - for col in streamcolsLS.dtype.names: - data[col] = LSobjs[col] - # ADM add data for the Gaia columns. - for col in streamcolsGaia.dtype.names: - data[col] = gaiaobjs[col] - - # ADM retain the data from this part of the loop. - allobjs.append(data) + def _read_data_per_stream_one_file(filename): + """Determine the stream objects for a single sweep file""" + return read_data_per_stream_one_file(filename, rapol, decpol, mind, maxd) + + nbrick = np.zeros((), dtype='i8') + t0 = time() + def _update_status(result): + """wrapper for critical reduction operation on main parallel process""" + if nbrick % 5 == 0 and nbrick > 0: + elapsed = time() - t0 + rate = elapsed / nbrick + log.info('{}/{} files; {:.1f} secs/file; {:.1f} total mins elapsed' + .format(nbrick, len(infiles), rate, elapsed/60.)) + + nbrick[...] += 1 + return result + + # ADM parallel process sweep files, limit to objects in the stream. + if numproc > 1: + pool = sharedmem.MapReduce(np=numproc) + with pool: + allobjs = pool.map(_read_data_per_stream_one_file, infiles, + reduce=_update_status) + else: + allobjs = list() + for fn in infiles: + allobjs.append(_update_status(_read_data_per_stream_one_file(fn))) - if i % 5 == 4: - log.info(f"Ran {i+1}/{len(infiles)} files...t={time()-start:.1f}s") # ADM assemble all of the relevant objects. allobjs = np.concatenate(allobjs) diff --git a/py/desitarget/streams/utilities.py b/py/desitarget/streams/utilities.py index 62f78139..db8a4ea6 100644 --- a/py/desitarget/streams/utilities.py +++ b/py/desitarget/streams/utilities.py @@ -17,6 +17,7 @@ from scipy.interpolate import UnivariateSpline from time import time import desitarget.streams.gaia_dr3_parallax_zero_point.zpt as gaia_zpt +from numpy.lib import recfunctions as rfn # ADM set up the DESI default logger. from desiutil.log import get_logger @@ -37,6 +38,46 @@ masyr = auni.mas / auni.year +def ivars_to_errors(objs, colnames=[]): + """ + Convert inverse variances to errors without dividing by zero. + + Parameters + ---------- + objs : :class:`~numpy.ndarray` + Array that contains the columns to be converted from inverse + variances to errors. + colnames : :class:`list` + The names of the columns to convert. + + Returns + ------- + :class:`~numpy.ndarray` + The input `objs`, modified so that any columns in `colnames` are + converted from inverse variances to errors and occurrences of + "IVAR" in column names are converted to "ERROR". + + Notes + ----- + - Column names are assumed to be upper case for the conversion of + IVAR->ERROR in the column names. + - No copy is made to save memory. So `objs` will be modified in place + and calls like a = ivars_to_errors(b, colnames=["X"]) will alter + values in b as well as returning a. + """ + for colname in colnames: + # ADM guard against dividing by zero. + error = np.zeros_like(objs[colname]) + 1e8 + ii = objs[colname] != 0 + error[ii] = 1./np.sqrt(objs[ii][colname]) + objs[colname] = error + newcolname = colname.replace("IVAR", "ERROR") + # ADM rename any IVAR columns. + objs = rfn.rename_fields(objs, {colname:newcolname}) + + return objs + + def cosd(x): """Return cos(x) for an angle x in degrees. """ From 365aad63d3c85c1ffe49c5f9dc5f7f8cd50fdeca Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Mon, 15 Apr 2024 16:33:28 -0700 Subject: [PATCH 43/63] clean up assignation of NaNs for zero columns in the cache --- py/desitarget/streams/cuts.py | 11 ++--------- py/desitarget/streams/io.py | 3 ++- py/desitarget/streams/utilities.py | 2 +- 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/py/desitarget/streams/cuts.py b/py/desitarget/streams/cuts.py index 5c39a13a..038de28e 100644 --- a/py/desitarget/streams/cuts.py +++ b/py/desitarget/streams/cuts.py @@ -79,14 +79,7 @@ def is_in_GD1(objs): rapol, decpol, ra_ref) # ADM derive the combined proper motion error. - # ADM guard against dividing by zero. - pmra_error = np.zeros_like(objs["PMRA_IVAR"]) + 1e8 - ii = objs['PMRA_IVAR'] != 0 - pmra_error[ii] = 1./np.sqrt(objs[ii]['PMRA_IVAR']) - pmdec_error = np.zeros_like(objs["PMDEC_IVAR"]) + 1e8 - ii = objs['PMDEC_IVAR'] != 0 - pmdec_error[ii] = 1./np.sqrt(objs[ii]['PMDEC_IVAR']) - pm_err = np.sqrt(0.5 * (pmra_error**2 + pmdec_error**2)) + pm_err = np.sqrt(0.5 * (objs["PMRA_ERROR"]**2 + objs["PMDEC_ERROR"]**2)) # ADM dust correction. ext_coeff = dict(g=3.237, r=2.176, z=1.217) @@ -288,7 +281,7 @@ def select_targets(swdir, stream_names=["GD1"], readperstream=True, # ADM the parameters that define the extent of the stream. mind, maxd = strm["MIND"], strm["MAXD"] # ADM read in the data. - objs = read_data_per_stream(swdir, rapol, decpol, ra_ref, mind, maxd, + objs = read_data_per_stream(swdir, rapol, decpol, mind, maxd, stream_name, addnors=addnors, readcache=readcache) allobjs.append(objs) diff --git a/py/desitarget/streams/io.py b/py/desitarget/streams/io.py index 6608e44f..21645369 100644 --- a/py/desitarget/streams/io.py +++ b/py/desitarget/streams/io.py @@ -114,7 +114,8 @@ def read_data_per_stream_one_file(filename, rapol, decpol, mind, maxd): for col in ["PHOT_G_MEAN_MAG", "PHOT_BP_MEAN_MAG", "PHOT_RP_MEAN_MAG" , "PMRA", "PMDEC"]: ii = gaiaobjs[col] < 1e-16 - gaiaobjs[ii][col] = np.nan + ii &= gaiaobjs[col] > -1e-16 + gaiaobjs[col][ii] = np.nan # ADM a (probably unnecessary) sanity check. assert(len(gaiaobjs) == len(LSobjs)) diff --git a/py/desitarget/streams/utilities.py b/py/desitarget/streams/utilities.py index db8a4ea6..3e3c5118 100644 --- a/py/desitarget/streams/utilities.py +++ b/py/desitarget/streams/utilities.py @@ -67,7 +67,7 @@ def ivars_to_errors(objs, colnames=[]): """ for colname in colnames: # ADM guard against dividing by zero. - error = np.zeros_like(objs[colname]) + 1e8 + error = np.zeros_like(objs[colname]) + np.nan ii = objs[colname] != 0 error[ii] = 1./np.sqrt(objs[ii][colname]) objs[colname] = error From ac874a660e7d665e974e62648f3168ef156ea5bb Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Mon, 15 Apr 2024 17:07:13 -0700 Subject: [PATCH 44/63] include PARALLAX in the columns corrected to the Gaia convention of NaN; write inverse function to convert errors back to IVARs --- py/desitarget/streams/io.py | 2 +- py/desitarget/streams/utilities.py | 40 ++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/py/desitarget/streams/io.py b/py/desitarget/streams/io.py index 21645369..b021dd63 100644 --- a/py/desitarget/streams/io.py +++ b/py/desitarget/streams/io.py @@ -112,7 +112,7 @@ def read_data_per_stream_one_file(filename, rapol, decpol, mind, maxd): gaiaobjs, colnames=["PARALLAX_IVAR", "PMRA_IVAR", "PMDEC_IVAR"]) for col in ["PHOT_G_MEAN_MAG", "PHOT_BP_MEAN_MAG", "PHOT_RP_MEAN_MAG" - , "PMRA", "PMDEC"]: + , "PARALLAX", "PMRA", "PMDEC"]: ii = gaiaobjs[col] < 1e-16 ii &= gaiaobjs[col] > -1e-16 gaiaobjs[col][ii] = np.nan diff --git a/py/desitarget/streams/utilities.py b/py/desitarget/streams/utilities.py index 3e3c5118..df4448e9 100644 --- a/py/desitarget/streams/utilities.py +++ b/py/desitarget/streams/utilities.py @@ -78,6 +78,46 @@ def ivars_to_errors(objs, colnames=[]): return objs +def errors_to_ivars(objs, colnames=[]): + """ + Convert errors to ivars, correctly dealing with NaNs. + + Parameters + ---------- + objs : :class:`~numpy.ndarray` + Array that contains the columns to be converted from errors + to inverse variances. + colnames : :class:`list` + The names of the columns to convert. + + Returns + ------- + :class:`~numpy.ndarray` + The input `objs`, modified so that any columns in `colnames` are + converted from errors to inverse variances and occurrences of + "ERROR" in column names are converted to "IVAR". + + Notes + ----- + - Column names are assumed to be upper case for the conversion of + ERROR->IVAR in the column names. + - No copy is made to save memory. So `objs` will be modified in place + and calls like a = errors_to_ivars(b, colnames=["X"]) will alter + values in b as well as returning a. + """ + for colname in colnames: + # ADM remove any NaNs. + ivar = np.zeros_like(objs[colname]) + ii = ~np.isnan(objs[colname]) + ivar[ii] = 1./objs[ii][colname]**2 + objs[colname] = ivar + newcolname = colname.replace("ERROR", "IVAR") + # ADM rename any IVAR columns. + objs = rfn.rename_fields(objs, {colname:newcolname}) + + return objs + + def cosd(x): """Return cos(x) for an angle x in degrees. """ From ec8537d24b9cb4cd4df4c9e40616391a659c27c9 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Tue, 16 Apr 2024 08:50:10 -0700 Subject: [PATCH 45/63] finalize the targets by returning them to a more data-systems like data model (no NaNs, IVARs) --- py/desitarget/streams/targets.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/py/desitarget/streams/targets.py b/py/desitarget/streams/targets.py index 3609df2d..d13b26f5 100644 --- a/py/desitarget/streams/targets.py +++ b/py/desitarget/streams/targets.py @@ -10,6 +10,7 @@ from desitarget.targets import initial_priority_numobs, set_obsconditions, \ encode_targetid +from desitarget.streams.utilities import errors_to_ivars # ADM set up the DESI default logger. from desiutil.log import get_logger @@ -109,6 +110,16 @@ def finalize(targets, desi_target, bgs_target, mws_target, scnd_target): # ADM set the OBSCONDITIONS. done["OBSCONDITIONS"] = set_obsconditions(done, scnd=True) + # ADM replace any NaNs with zeros. + for col in ["PHOT_G_MEAN_MAG", "PHOT_BP_MEAN_MAG", "PHOT_RP_MEAN_MAG", + "PARALLAX", "PMRA", "PMDEC"]: + ii = np.isnan(done[col]) + done[col][ii] = 0. + + # ADM change errors to IVARs. + done = errors_to_ivars( + done, colnames=["PARALLAX_ERROR", "PMRA_ERROR", "PMDEC_ERROR"]) + # ADM some final checks that the targets conform to expectations... # ADM check that each target has a unique ID. if len(done["TARGETID"]) != len(np.unique(done["TARGETID"])): From 406fdf525ef152047530bae2c371e78e2892bcc3 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Mon, 22 Apr 2024 14:35:05 -0700 Subject: [PATCH 46/63] allow zero and negative g and z fluxes in the cache --- py/desitarget/streams/io.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/py/desitarget/streams/io.py b/py/desitarget/streams/io.py index b021dd63..1dc1cf6a 100644 --- a/py/desitarget/streams/io.py +++ b/py/desitarget/streams/io.py @@ -87,8 +87,8 @@ def read_data_per_stream_one_file(filename, rapol, decpol, mind, maxd): # ADM ...that aren't very faint (> 22.5 mag in r). ii &= objs["FLUX_R"] > 1 # ADM Also guard against negative fluxes in g/r. - ii &= objs["FLUX_G"] > 0. - ii &= objs["FLUX_Z"] > 0. + # ii &= objs["FLUX_G"] > 0. + # ii &= objs["FLUX_Z"] > 0. objs = objs[ii] From da4f511079b225bf754a7feceff82b39f099d1b8 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Wed, 24 Apr 2024 17:21:38 -0700 Subject: [PATCH 47/63] refine Gaia matching to make it as precise as possible for small-scale Gaia pairs --- py/desitarget/gaiamatch.py | 114 ++++++++++++++++++++++++++++++++++-- py/desitarget/streams/io.py | 4 +- 2 files changed, 110 insertions(+), 8 deletions(-) diff --git a/py/desitarget/gaiamatch.py b/py/desitarget/gaiamatch.py index e531a180..db208275 100644 --- a/py/desitarget/gaiamatch.py +++ b/py/desitarget/gaiamatch.py @@ -24,8 +24,8 @@ from desitarget import io from desitarget.io import check_fitsio_version, gitversion from desitarget.internal import sharedmem -from desitarget.geomask import hp_in_box, add_hp_neighbors, pixarea2nside -from desitarget.geomask import hp_beyond_gal_b, nside2nside, rewind_coords +from desitarget.geomask import hp_in_box, add_hp_neighbors, pixarea2nside, \ + hp_beyond_gal_b, nside2nside, rewind_coords, radec_match_to from desimodel.footprint import radec2pix from astropy.coordinates import SkyCoord from astropy import units as u @@ -1485,6 +1485,76 @@ def find_gaia_files_tiles(tiles=None, neighbors=True, dr="dr2"): return gaiafiles +def match_gaia_to_primary_post_dr3(objs, matchrad=0.2, dr="dr3"): + """Match objects to Gaia healpix files starting with Gaia DR3. + + Parameters + ---------- + objs : :class:`~numpy.ndarray` + Must contain at least "RA", "DEC". ASSUMED TO BE AT A REFERENCE + EPOCH OF 2015.5 and EQUINOX J2000/ICRS. + matchrad : :class:`float`, optional, defaults to 0.2 arcsec + The matching radius in arcseconds. + dr : :class:`str`, optional, defaults to "dr3" + Name of a Gaia data release. Specifies which REF_EPOCH to use. + + Returns + ------- + :class:`~numpy.ndarray` + Gaia information for each matching object, in a format like + `dr3datamodelfull`. + + Notes + ----- + - Returned objects correspond row-by-row to `objs`. + - For objects that do NOT have a match in Gaia, the "REF_ID" + column is set to -1, and all other columns are zero. + - This code fixes a number of minor issues related to close pairs + in Gaia that were discovered after running Main Survey targets + (i.e. starting with Gaia DR3). To reproduce DESI Main Survey + targets, :func:`match_gaia_to_primary()` should be used. + """ + # ADM issue a warning that this code is intended for use post-DR3. + if dr in ["dr2", "edr3"]: + log.warning("For Gaia Data Releases earlier than DR3, you may \ + want to use the function match_gaia_to_primary() instead") + + # ADM set up the output array for Gaia information. + gaiainfo = np.zeros(len(objs), dtype=dr3datamodelfull.dtype) + + # ADM objects without matches should have REF_ID of -1. + gaiainfo['REF_ID'] = -1 + + # ADM compile the list of relevant Gaia files. + gaiafiles = find_gaia_files(objs, dr=dr) + + gaia = [] + for fn in gaiafiles: + gaia.append(read_gaia_file(fn, dr=dr)) + gaia = np.concatenate(gaia) + + # ADM the name of the relevant RA/Dec columns in the Gaia data model. + gracol, gdeccol = "RA", "DEC" + gpmracol, gpmdeccol = "PMRA", "PMDEC" + + # ADM rewind coordinates from the Gaia DR3 2016.0 epoch to 2015.5. + rarew, decrew = rewind_coords(gaia[gracol], gaia[gdeccol], + gaia[gpmracol], gaia[gpmdeccol], + epochnow=2016.0, epochpast=2015.5) + gaia[gracol] = rarew + gaia[gdeccol] = decrew + gaia["REF_EPOCH"] = 2015.5 + + # ADM perform the match. + # ADM sense is key! Need unique Gaia match for each primary object. + idgaia, idobjs = radec_match_to(gaia, objs, matchrad) + + # ADM assign Gaia info to array corresponding to passed objects. + gaiainfo[idobjs] = gaia[idgaia] + + return gaiainfo + + def match_gaia_to_primary(objs, matchrad=0.2, retaingaia=False, gaiabounds=[0., 360., -90., 90.], dr="edr3"): """Match objects to Gaia healpix files and return Gaia information. @@ -1526,6 +1596,13 @@ def match_gaia_to_primary(objs, matchrad=0.2, retaingaia=False, - If `retaingaia` is ``True`` then objects after the first len(`objs`) objects are Gaia objects that do not have a sweeps match but are in the area bounded by `gaiabounds`. + - After running targets for the DESI Main Survey (i.e. starting + with Gaia DR3) a minor bug was identified where a match within + `matchrad` was returned that wasn't necessarily the CLOSEST + match for instances of very close Gaia pairs. Therefore, for + releases subsequent to Gaia DR3, this function is modified to + always find the CLOSEST Gaia match rather than just ANY match + that satisfies the `matchrad` constraint. """ # ADM retain all Gaia objects in a sweeps-like box. if retaingaia: @@ -1578,8 +1655,17 @@ def match_gaia_to_primary(objs, matchrad=0.2, retaingaia=False, gaia["REF_EPOCH"] = 2015.5 cgaia = SkyCoord(gaia[gracol]*u.degree, gaia[gdeccol]*u.degree) - idobjs, idgaia, _, _ = cgaia.search_around_sky(cobjs, matchrad*u.arcsec) - # ADM assign the Gaia info to the array that corresponds to the passed objects. + # ADM updated behavior after DR3 to always find the closest pair + # ADM rather than just a pair that meets the matchrad constraint. + if dr == 'dr2' or dr == 'edr3': + idobjs, idgaia, _, _ = cgaia.search_around_sky(cobjs, + matchrad*u.arcsec) + else: + # ADM sense is key! Need unique Gaia match for each primary. + idgaia, idobjs = radec_match_to([cgaia.ra.value, cgaia.dec.value], + [cobjs.ra.value, cobjs.dec.value], + matchrad, radec=True) + # ADM assign Gaia info to array corresponding to passed objects. gaiainfo[idobjs] = gaia[idgaia] # ADM if retaingaia was set, also build an array of Gaia objects that @@ -1629,6 +1715,13 @@ def match_gaia_to_primary_single(objs, matchrad=0.2, dr="edr3"): ----- - If the object does NOT have a match in the Gaia files, the "REF_ID" column is set to -1, and all other columns are zero + - After running targets for the DESI Main Survey (i.e. starting + with Gaia DR3) a minor bug was identified where a match within + `matchrad` was returned that wasn't necessarily the CLOSEST + match for instances of very close Gaia pairs. Therefore, for + releases subsequent to Gaia DR3, this function is modified to + always find the CLOSEST Gaia match rather than just ANY match + that satisfies the `matchrad` constraint. """ # ADM convert the coordinates of the input objects to a SkyCoord object. cobjs = SkyCoord(objs["RA"]*u.degree, objs["DEC"]*u.degree) @@ -1669,8 +1762,17 @@ def match_gaia_to_primary_single(objs, matchrad=0.2, dr="edr3"): gaia["REF_EPOCH"] = 2015.5 cgaia = SkyCoord(gaia[gracol]*u.degree, gaia[gdeccol]*u.degree) - sep = cobjs.separation(cgaia) - idgaia = np.where(sep < matchrad*u.arcsec)[0] + + # ADM updated behavior after DR3 to always find the closest pair + # ADM rather than just a pair that meets the matchrad constraint. + if dr == 'dr2' or dr == 'edr3': + sep = cobjs.separation(cgaia) + idgaia = np.where(sep < matchrad*u.arcsec)[0] + else: + # ADM sense is key! Need unique Gaia match for each primary. + idgaia, idobjs = radec_match_to([cgaia.ra.value, cgaia.dec.value], + [cobjs.ra.value, cobjs.dec.value], + matchrad, radec=True) # ADM assign the Gaia info to the array that corresponds to the passed object. if len(idgaia) > 0: gaiainfo = gaia[idgaia] diff --git a/py/desitarget/streams/io.py b/py/desitarget/streams/io.py index 1dc1cf6a..63650ce7 100644 --- a/py/desitarget/streams/io.py +++ b/py/desitarget/streams/io.py @@ -15,7 +15,7 @@ from desitarget import io from desitarget.geomask import pixarea2nside, add_hp_neighbors, sweep_files_touch_hp -from desitarget.gaiamatch import match_gaia_to_primary +from desitarget.gaiamatch import match_gaia_to_primary_post_dr3 from desitarget.targets import resolve from desitarget.streams.utilities import betw, ivars_to_errors from desitarget.internal import sharedmem @@ -103,7 +103,7 @@ def read_data_per_stream_one_file(filename, rapol, decpol, mind, maxd): # ADM catch the case where there are no objects meeting the cuts. if len(LSobjs) > 0: - gaiaobjs = match_gaia_to_primary(LSobjs, matchrad=1., dr=gaiadr) + gaiaobjs = match_gaia_to_primary_post_dr3(LSobjs, matchrad=1., dr=gaiadr) # ADM to try and better resemble Gaia data, set zero # ADM magnitudes and proper motions to NaNs and change From 491d6158a8b3b869ce42861e839c44603f150720 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Wed, 24 Apr 2024 19:27:21 -0700 Subject: [PATCH 48/63] bug fix wrapping string --- py/desitarget/gaiamatch.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/py/desitarget/gaiamatch.py b/py/desitarget/gaiamatch.py index db208275..41c632c9 100644 --- a/py/desitarget/gaiamatch.py +++ b/py/desitarget/gaiamatch.py @@ -1516,8 +1516,8 @@ def match_gaia_to_primary_post_dr3(objs, matchrad=0.2, dr="dr3"): """ # ADM issue a warning that this code is intended for use post-DR3. if dr in ["dr2", "edr3"]: - log.warning("For Gaia Data Releases earlier than DR3, you may \ - want to use the function match_gaia_to_primary() instead") + log.warning("For Gaia Data Releases earlier than DR3, you may want" + " to use the function match_gaia_to_primary() instead") # ADM set up the output array for Gaia information. gaiainfo = np.zeros(len(objs), dtype=dr3datamodelfull.dtype) From b69239bbab9bfd5af931b6141c46fd831e68e1c6 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Tue, 7 May 2024 14:17:56 -0700 Subject: [PATCH 49/63] correct faint selection now that we allow NaNs --- py/desitarget/streams/cuts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py/desitarget/streams/cuts.py b/py/desitarget/streams/cuts.py index 038de28e..fa9dff1c 100644 --- a/py/desitarget/streams/cuts.py +++ b/py/desitarget/streams/cuts.py @@ -145,7 +145,7 @@ def is_in_GD1(objs): cmd_win = 0.1 + 10**(-2 + (r - 20) / 2.5) # ADM overall faint selection. - faint_sel = objs['PMRA'] == 0 + faint_sel = ~np.isfinite(objs['PMRA']) faint_sel &= betw(r, 20, faint_limit) faint_sel &= betw(np.abs(delta_cmd), 0, cmd_win) faint_sel &= startyp From 56cd246df545bf4b73f2c2bc55c366d843cb0cae Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Mon, 13 May 2024 10:34:30 -0700 Subject: [PATCH 50/63] add a script for appending stream (or any) targets to the MTL ledgers --- bin/add_to_mtl_ledgers | 47 ++++++++++++++++++++++++++++++++++++++++++ py/desitarget/cuts.py | 8 +++---- py/desitarget/mtl.py | 8 +++---- 3 files changed, 55 insertions(+), 8 deletions(-) create mode 100755 bin/add_to_mtl_ledgers diff --git a/bin/add_to_mtl_ledgers b/bin/add_to_mtl_ledgers new file mode 100755 index 00000000..9ba75861 --- /dev/null +++ b/bin/add_to_mtl_ledgers @@ -0,0 +1,47 @@ +#!/usr/bin/env python + +from desitarget.mtl import add_to_ledgers, get_mtl_dir, _get_mtl_nside + +# ADM this is the upper-limit for memory for mtl._get_mtl_nside()=32. +nproc = 12 +# ADM this is the upper-limit for memory for mtl._get_mtl_nside()=16. +# nproc = 8 +# ADM retrieve the $MTL_DIR environment variable. +mtldir = get_mtl_dir() +mtlnside = _get_mtl_nside() +# ADM default obsconditions. +obscon = "DARK" + +from argparse import ArgumentParser +ap = ArgumentParser(description='Add new targets to MTL ledger files from a file of targets') +ap.add_argument("targfile", + help="Full path to a file of targets") +ap.add_argument("--dest", + help="Full path to the output directory to host the ledger. The \ + filename and full directory structure are built on-the-fly from \ + file headers in targdirname. [defaults to {})".format(mtldir), + default=mtldir) +ap.add_argument("--obscon", + help="String matching ONE obscondition in the bitmask yaml file \ + (e.g. 'BRIGHT'). Controls priorities when merging targets and \ + where the ledger is written. [defaults to {})".format(obscon), + default=obscon) +ap.add_argument("--numproc", type=int, + help='number of concurrent processes to use [defaults to {}]'. + format(nproc), + default=nproc) +ap.add_argument('--healpixels', + help="HEALPixels corresponding to `nside` (e.g. '6,9,57'). Only \ + write the ledger for targets in these pixels, at the default \ + nside, which is [{}]".format(mtlnside), + default=None) + +ns = ap.parse_args() + +# ADM parse the list of HEALPixels in which to run. +pixlist = ns.healpixels +if pixlist is not None: + pixlist = [int(pix) for pix in pixlist.split(',')] + +add_to_ledgers(ns.targfile, mtldir=ns.dest, pixlist=pixlist, obscon=ns.obscon, + numproc=ns.numproc) diff --git a/py/desitarget/cuts.py b/py/desitarget/cuts.py index ee699b39..e459b9f1 100644 --- a/py/desitarget/cuts.py +++ b/py/desitarget/cuts.py @@ -2392,7 +2392,7 @@ def set_target_bits(photsys_north, photsys_south, obs_rflux, south_cuts = [False, True] if resolvetargs: # ADM if only southern objects were sent this will be [True], if - # ADM only northern it will be [False], else it wil be both. + # ADM only northern it will be [False], else it will be both. south_cuts = list(set(np.atleast_1d(photsys_south))) # ADM default for target classes we WON'T process is all False. @@ -2597,21 +2597,21 @@ def set_target_bits(photsys_north, photsys_south, obs_rflux, mws_faint_red = (mws_faint_red_n & photsys_north) | (mws_faint_red_s & photsys_south) mws_faint_blue = (mws_faint_blue_n & photsys_north) | (mws_faint_blue_s & photsys_south) - # Construct the targetflag bits for DECaLS (i.e. South). + # Construct the target flag bits for DECaLS (i.e. South). desi_target = lrg_south * desi_mask.LRG_SOUTH desi_target |= elg_south * desi_mask.ELG_SOUTH desi_target |= elg_vlo_south * desi_mask.ELG_VLO_SOUTH desi_target |= elg_lop_south * desi_mask.ELG_LOP_SOUTH desi_target |= qso_south * desi_mask.QSO_SOUTH - # Construct the targetflag bits for MzLS and BASS (i.e. North). + # Construct the target flag bits for MzLS and BASS (i.e. North). desi_target |= lrg_north * desi_mask.LRG_NORTH desi_target |= elg_north * desi_mask.ELG_NORTH desi_target |= elg_vlo_north * desi_mask.ELG_VLO_NORTH desi_target |= elg_lop_north * desi_mask.ELG_LOP_NORTH desi_target |= qso_north * desi_mask.QSO_NORTH - # Construct the targetflag bits combining north and south. + # Construct the target flag bits combining north and south. desi_target |= lrg * desi_mask.LRG desi_target |= elg * desi_mask.ELG desi_target |= elg_vlo * desi_mask.ELG_VLO diff --git a/py/desitarget/mtl.py b/py/desitarget/mtl.py index 7e0b15a6..21aed58f 100644 --- a/py/desitarget/mtl.py +++ b/py/desitarget/mtl.py @@ -1274,7 +1274,7 @@ def add_to_ledgers_in_hp(targets, pixlist, mtldir=None, obscon="DARK", def add_to_ledgers(targs, mtldir=None, pixlist=None, obscon="DARK", numproc=1, updatedonefiles=True): """ - Make initial MTL ledger files for HEALPixels, in parallel. + Add new targets to MTL ledger files for HEALPixels, in parallel. Parameters ---------- @@ -1295,14 +1295,14 @@ def add_to_ledgers(targs, mtldir=None, pixlist=None, obscon="DARK", A string matching ONE obscondition in the desitarget bitmask yaml file (i.e. in `desitarget.targetmask.obsconditions`), e.g. "DARK" Governs how priorities are set based on "obsconditions". Also - governs the sub-directory to which the ledger is written. + governs the sub-directory to which the ledgers are added. numproc : :class:`int`, optional, defaults to 1 for serial Number of processes to parallelize across. updatedonefiles: :class:`bool`, optional, defaults to ``True`` If ``False`` then do NOT write a timestamp to the MTL override "done" files indicating an update has occurred. - USE WITH CAUTION. In general, the alt-MTL schema needs to - know when updates happened. + USE WITH CAUTION. In general, the alt-MTL schema NEEDS TO + KNOW WHEN UPDATES HAPPENED. Returns ------- From da4cf597925896499f2eb6fc2ff4ec979f2a581d Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Mon, 13 May 2024 13:58:22 -0700 Subject: [PATCH 51/63] update priorities and numbers of observations for the GD targets --- py/desitarget/data/targetmask.yaml | 10 +++++----- py/desitarget/mtl.py | 5 +++-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/py/desitarget/data/targetmask.yaml b/py/desitarget/data/targetmask.yaml index b77a7e97..3d37d98d 100644 --- a/py/desitarget/data/targetmask.yaml +++ b/py/desitarget/data/targetmask.yaml @@ -418,8 +418,8 @@ priorities: DARK_TOO_LOP: SAME_AS_BRIGHT_TOO_LOP DARK_TOO_HIP: SAME_AS_BRIGHT_TOO_HIP GD1_BRIGHT_PM: {UNOBS: 1520, MORE_ZGOOD: 2, MORE_ZWARN: 2, DONE: 2, OBS: 1, DONOTOBSERVE: 0, MORE_MIDZQSO: 0} - GD1_FAINT_NO_PM: {UNOBS: 1420, MORE_ZGOOD: 2, MORE_ZWARN: 2, DONE: 2, OBS: 1, DONOTOBSERVE: 0, MORE_MIDZQSO: 0} - GD1_FILLER: {UNOBS: 100, MORE_ZGOOD: 2, MORE_ZWARN: 2, DONE: 2, OBS: 1, DONOTOBSERVE: 0, MORE_MIDZQSO: 0} + GD1_FAINT_NO_PM: {UNOBS: 1510, MORE_ZGOOD: 2, MORE_ZWARN: 2, DONE: 2, OBS: 1, DONOTOBSERVE: 0, MORE_MIDZQSO: 0} + GD1_FILLER: {UNOBS: 49, MORE_ZGOOD: 2, MORE_ZWARN: 2, DONE: 2, OBS: 1, DONOTOBSERVE: 0, MORE_MIDZQSO: 0} # ADM INITIAL number of observations (NUMOBS_INIT) for each target bit # ADM SAME_AS_XXX means to use the NUMOBS_INIT for bitname XXX @@ -571,6 +571,6 @@ numobs: DWF_BRIGHT_LO: 10 DWF_DARK_HI: 10 DWF_DARK_LO: 10 - GD1_BRIGHT_PM: 1 - GD1_FAINT_NO_PM: 1 - GD1_FILLER: 1 + GD1_BRIGHT_PM: 4 + GD1_FAINT_NO_PM: 4 + GD1_FILLER: 4 diff --git a/py/desitarget/mtl.py b/py/desitarget/mtl.py index 21aed58f..7775cd2d 100644 --- a/py/desitarget/mtl.py +++ b/py/desitarget/mtl.py @@ -1219,11 +1219,12 @@ def add_to_ledgers_in_hp(targets, pixlist, mtldir=None, obscon="DARK", for col in ["DESI_TARGET", "BGS_TARGET", "MWS_TARGET", "SCND_TARGET"]: oldmatches[col] |= mtlmatches[col] - # ADM set PRIORITY to larger number - # ADM (+ inherit TARGET_STATE, SUBPRIORITY, NUMOBS_MORE). + # ADM set PRIORITY to larger number (plus, inherit + # ADM TARGET_STATE, SUBPRIORITY, NUMOBS_MORE/_INIT). ii = mtlmatches["PRIORITY"] > oldmatches["PRIORITY"] oldmatches["PRIORITY"][ii] = mtlmatches["PRIORITY"][ii] oldmatches["NUMOBS_MORE"][ii] = mtlmatches["NUMOBS_MORE"][ii] + oldmatches["NUMOBS_INIT"][ii] = mtlmatches["NUMOBS_INIT"][ii] oldmatches["SUBPRIORITY"][ii] = mtlmatches["SUBPRIORITY"][ii] oldmatches["TARGET_STATE"][ii] = mtlmatches["TARGET_STATE"][ii] # ADM also need to inherit the new timestamp and code version. From 99bde3167047484434a3631a220b9d9a88538f6c Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Mon, 13 May 2024 16:28:20 -0700 Subject: [PATCH 52/63] update priorities and numbers of observations for the GD targets (again) --- py/desitarget/data/targetmask.yaml | 6 +++--- py/desitarget/test/test_mtl.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/py/desitarget/data/targetmask.yaml b/py/desitarget/data/targetmask.yaml index 3d37d98d..09af28ef 100644 --- a/py/desitarget/data/targetmask.yaml +++ b/py/desitarget/data/targetmask.yaml @@ -417,9 +417,9 @@ priorities: BRIGHT_TOO_HIP: {UNOBS: 9500, MORE_ZGOOD: 2, MORE_ZWARN: 2, DONE: 2, OBS: 1, DONOTOBSERVE: 0, MORE_MIDZQSO: 0} DARK_TOO_LOP: SAME_AS_BRIGHT_TOO_LOP DARK_TOO_HIP: SAME_AS_BRIGHT_TOO_HIP - GD1_BRIGHT_PM: {UNOBS: 1520, MORE_ZGOOD: 2, MORE_ZWARN: 2, DONE: 2, OBS: 1, DONOTOBSERVE: 0, MORE_MIDZQSO: 0} - GD1_FAINT_NO_PM: {UNOBS: 1510, MORE_ZGOOD: 2, MORE_ZWARN: 2, DONE: 2, OBS: 1, DONOTOBSERVE: 0, MORE_MIDZQSO: 0} - GD1_FILLER: {UNOBS: 49, MORE_ZGOOD: 2, MORE_ZWARN: 2, DONE: 2, OBS: 1, DONOTOBSERVE: 0, MORE_MIDZQSO: 0} + GD1_BRIGHT_PM: {UNOBS: 1520, MORE_ZGOOD: 1520, MORE_ZWARN: 1520, DONE: 2, OBS: 1, DONOTOBSERVE: 0, MORE_MIDZQSO: 0} + GD1_FAINT_NO_PM: {UNOBS: 1510, MORE_ZGOOD: 1510, MORE_ZWARN: 1510, DONE: 2, OBS: 1, DONOTOBSERVE: 0, MORE_MIDZQSO: 0} + GD1_FILLER: {UNOBS: 49, MORE_ZGOOD: 49, MORE_ZWARN: 49, DONE: 2, OBS: 1, DONOTOBSERVE: 0, MORE_MIDZQSO: 0} # ADM INITIAL number of observations (NUMOBS_INIT) for each target bit # ADM SAME_AS_XXX means to use the NUMOBS_INIT for bitname XXX diff --git a/py/desitarget/test/test_mtl.py b/py/desitarget/test/test_mtl.py index 8d16922e..9082c1e6 100644 --- a/py/desitarget/test/test_mtl.py +++ b/py/desitarget/test/test_mtl.py @@ -112,7 +112,7 @@ def update_data_model(self, cat): _, _, survey = main_cmx_or_sv(cat) truedm = survey_data_model(cat, survey=survey) addedcols = list(set(truedm.dtype.names) - set(cat.dtype.names)) - # ADM We set Main Survey QN columnsin the SetUp. Add any others. + # ADM We set Main Survey QN columns in the SetUp. Add any others. for col in addedcols: cat[col] = [-1] * len(cat) # ADM Set QN redshifts ('Z_QN') to mimic redrock redshifts ('Z'). From 8181cdef75d1e8d4a5cd2d37b288638f98f93793 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Tue, 14 May 2024 11:06:49 -0700 Subject: [PATCH 53/63] add unit tests for streams MTL (which also tests the priority/numobs progression works as expected --- py/desitarget/test/test_mtl_streams.py | 127 +++++++++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 py/desitarget/test/test_mtl_streams.py diff --git a/py/desitarget/test/test_mtl_streams.py b/py/desitarget/test/test_mtl_streams.py new file mode 100644 index 00000000..2919a9ab --- /dev/null +++ b/py/desitarget/test/test_mtl_streams.py @@ -0,0 +1,127 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +# -*- coding: utf-8 -*- +"""Test desitarget.mtl specifically for secondary/stream programs. +""" + +import os +import unittest +import numpy as np +from astropy.table import Table, join + +from desitarget.targetmask import desi_mask as Mx +from desitarget.targetmask import scnd_mask as sMx + +from desitarget.mtl import make_mtl, mtldatamodel, survey_data_model +from desitarget.targets import initial_priority_numobs, main_cmx_or_sv + + +class TestMTLStreams(unittest.TestCase): + + def setUp(self): + self.targs = Table() + # ADM two copies of each of the GD1-style targets. + self.types = np.array(['GD1_BRIGHT_PM', 'GD1_FAINT_NO_PM', 'GD1_FILLER', + 'GD1_BRIGHT_PM', 'GD1_FAINT_NO_PM', 'GD1_FILLER']) + # ADM the initial values of PRIORITY. + self.priorities = [sMx[t].priorities['UNOBS'] for t in self.types] + # ADM the initial values of NUMOBS_MORE. + self.nom = [sMx[t].numobs for t in self.types] + + nt = len(self.types) + # ADM add some "extra" columns that are needed for observations. + for col in ["RA", "DEC", "PARALLAX", "PMRA", "PMDEC", "REF_EPOCH"]: + self.targs[col] = np.zeros(nt, dtype=mtldatamodel[col].dtype) + self.targs['DESI_TARGET'] = Mx["SCND_ANY"].mask + self.targs['SCND_TARGET'] = [sMx[t].mask for t in self.types] + for col in ['BGS_TARGET', 'MWS_TARGET', 'SUBPRIORITY', "PRIORITY"]: + self.targs[col] = np.zeros(nt, dtype=mtldatamodel[col].dtype) + + n = len(self.targs) + self.targs['TARGETID'] = list(range(n)) + + # ADM determine the initial PRIORITY and NUMOBS. + pinit, ninit = initial_priority_numobs(self.targs, obscon="BRIGHT", + scnd=True) + self.targs["PRIORITY_INIT"] = pinit + self.targs["NUMOBS_INIT"] = ninit + + # ADM set up an ersatz redshift catalog. + self.zcat = Table() + # ADM reverse the TARGETIDs to check joins. + self.zcat['TARGETID'] = np.flip(self.targs['TARGETID']) + + self.zcat['Z'] = [0.001, 0.001, 0.001, 0.001, 0.001, 0.001] + # ADM set ZWARN for half of the objects to test both MORE_ZWARN + # ADM and MORE_ZGOOD. + self.zcat['ZWARN'] = [0, 0, 0, 1, 1, 1] + self.zcat['NUMOBS'] = [1, 1, 1, 1, 1, 1] + self.zcat['ZTILEID'] = [-1, -1, -1, -1, -1, -1] + + # ADM expected progression in priorities and numbers of observations. + # ADM hand-code to some extent to better check for discrepancies. + iigood = self.zcat["ZWARN"] == 0 + zgood = [sMx[t].priorities['MORE_ZGOOD'] for t in self.types[iigood]] + zwarn = [sMx[t].priorities['MORE_ZWARN'] for t in self.types[~iigood]] + # ADM PRIORITY after zero, one, two, three passes through MTL. + self.post_prio = pinit + self.post_prio = np.vstack([self.post_prio, zgood + zwarn]) + self.post_prio = np.vstack([self.post_prio, zgood + zwarn]) + self.post_prio = np.vstack([self.post_prio, zgood + zwarn]) + self.post_prio = np.vstack( + [self.post_prio, [sMx[t].priorities['DONE'] for t in self.types]]) + # ADM NUMOBS after zero, one, two, three passes through MTL. + self.post_nom = ninit + for numobs in np.arange(1, 5): + self.post_nom = np.vstack([self.post_nom, + np.array(self.nom) - numobs]) + + def flesh_out_data_model(self, cat): + """Flesh out columns to produce full Main Survey data model. + """ + truedm = survey_data_model(cat, survey="main") + addedcols = list(set(truedm.dtype.names) - set(cat.dtype.names)) + for col in addedcols: + cat[col] = [-1] * len(cat) + # ADM Set QN redshifts ('Z_QN') to mimic redrock redshifts ('Z'). + if 'Z' in cat.dtype.names: + cat['Z_QN'] = cat['Z'] + cat['IS_QSO_QN'] = 1 + + return cat + + def test_numobs(self): + """Test priorities, numobs, set correctly with no zcat. + """ + t = self.targs.copy() + t = self.flesh_out_data_model(t) + mtl = make_mtl(t, "BRIGHT") + self.assertTrue(np.all(mtl['NUMOBS_MORE'] == self.post_nom[0])) + self.assertTrue(np.all(mtl['PRIORITY'] == self.post_prio[0])) + + def test_zcat(self): + """Test priorities/numobs correct after zcat/multiple passes. + """ + t = self.targs.copy() + t = self.flesh_out_data_model(t) + + zc = self.zcat.copy() + zc = self.flesh_out_data_model(zc) + + for numobs in range(1, 5): + zc["NUMOBS"] = numobs + mtl = make_mtl(t, "BRIGHT", zcat=zc, trim=False) + self.assertTrue(np.all(mtl['PRIORITY'] == self.post_prio[numobs])) + self.assertTrue(np.all(mtl['NUMOBS_MORE'] == self.post_nom[numobs])) + + +if __name__ == '__main__': + unittest.main() + + +def test_suite(): + """Allows testing of only this module with the command: + + python setup.py test -m desitarget.test.test_mtl_streams + """ + return unittest.defaultTestLoader.loadTestsFromName(__name__) + From 1bf7904592f5c04865a5a123fc6e00e5ed230bb7 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Tue, 14 May 2024 12:17:45 -0700 Subject: [PATCH 54/63] code style --- py/desitarget/streams/io.py | 6 +++--- py/desitarget/streams/utilities.py | 4 ++-- py/desitarget/test/test_mtl_streams.py | 1 - 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/py/desitarget/streams/io.py b/py/desitarget/streams/io.py index 63650ce7..a5226d42 100644 --- a/py/desitarget/streams/io.py +++ b/py/desitarget/streams/io.py @@ -111,8 +111,8 @@ def read_data_per_stream_one_file(filename, rapol, decpol, mind, maxd): gaiaobjs = ivars_to_errors( gaiaobjs, colnames=["PARALLAX_IVAR", "PMRA_IVAR", "PMDEC_IVAR"]) - for col in ["PHOT_G_MEAN_MAG", "PHOT_BP_MEAN_MAG", "PHOT_RP_MEAN_MAG" - , "PARALLAX", "PMRA", "PMDEC"]: + for col in ["PHOT_G_MEAN_MAG", "PHOT_BP_MEAN_MAG", "PHOT_RP_MEAN_MAG", + "PARALLAX", "PMRA", "PMDEC"]: ii = gaiaobjs[col] < 1e-16 ii &= gaiaobjs[col] > -1e-16 gaiaobjs[col][ii] = np.nan @@ -266,6 +266,7 @@ def _read_data_per_stream_one_file(filename): nbrick = np.zeros((), dtype='i8') t0 = time() + def _update_status(result): """wrapper for critical reduction operation on main parallel process""" if nbrick % 5 == 0 and nbrick > 0: @@ -288,7 +289,6 @@ def _update_status(result): for fn in infiles: allobjs.append(_update_status(_read_data_per_stream_one_file(fn))) - # ADM assemble all of the relevant objects. allobjs = np.concatenate(allobjs) log.info(f"Found {len(allobjs)} total objects...t={time()-start:.1f}s") diff --git a/py/desitarget/streams/utilities.py b/py/desitarget/streams/utilities.py index df4448e9..139f7395 100644 --- a/py/desitarget/streams/utilities.py +++ b/py/desitarget/streams/utilities.py @@ -73,7 +73,7 @@ def ivars_to_errors(objs, colnames=[]): objs[colname] = error newcolname = colname.replace("IVAR", "ERROR") # ADM rename any IVAR columns. - objs = rfn.rename_fields(objs, {colname:newcolname}) + objs = rfn.rename_fields(objs, {colname: newcolname}) return objs @@ -113,7 +113,7 @@ def errors_to_ivars(objs, colnames=[]): objs[colname] = ivar newcolname = colname.replace("ERROR", "IVAR") # ADM rename any IVAR columns. - objs = rfn.rename_fields(objs, {colname:newcolname}) + objs = rfn.rename_fields(objs, {colname: newcolname}) return objs diff --git a/py/desitarget/test/test_mtl_streams.py b/py/desitarget/test/test_mtl_streams.py index 2919a9ab..4524d3a0 100644 --- a/py/desitarget/test/test_mtl_streams.py +++ b/py/desitarget/test/test_mtl_streams.py @@ -124,4 +124,3 @@ def test_suite(): python setup.py test -m desitarget.test.test_mtl_streams """ return unittest.defaultTestLoader.loadTestsFromName(__name__) - From 7e235e3469ea4c634d8787c13ff47ac071253e15 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Fri, 17 May 2024 09:50:44 -0700 Subject: [PATCH 55/63] updated changes docs --- doc/changes.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/doc/changes.rst b/doc/changes.rst index 796bb319..d17d6b91 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -5,7 +5,12 @@ desitarget Change Log 2.7.1 (unreleased) ------------------ -* No changes yet. +* Add the GD1 dark matter streams program [`PR #814`_]. + * Substantial new code in the `streams` directory. + * Addresses `issue #812`_. + +.. _`issue #812`: https://github.com/desihub/desitarget/issues/812 +.. _`PR #814`: https://github.com/desihub/desitarget/pull/814 2.7.0 (2023-12-05) ------------------ From 60d55933a295ecbaa974b2057f5e57b26049fcd9 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Fri, 17 May 2024 10:14:11 -0700 Subject: [PATCH 56/63] update GD1 priorities and NUMOBS to final values --- py/desitarget/data/targetmask.yaml | 12 ++++++------ py/desitarget/test/test_mtl_streams.py | 19 ++++++++++++++----- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/py/desitarget/data/targetmask.yaml b/py/desitarget/data/targetmask.yaml index 09af28ef..af5fb3a1 100644 --- a/py/desitarget/data/targetmask.yaml +++ b/py/desitarget/data/targetmask.yaml @@ -417,9 +417,9 @@ priorities: BRIGHT_TOO_HIP: {UNOBS: 9500, MORE_ZGOOD: 2, MORE_ZWARN: 2, DONE: 2, OBS: 1, DONOTOBSERVE: 0, MORE_MIDZQSO: 0} DARK_TOO_LOP: SAME_AS_BRIGHT_TOO_LOP DARK_TOO_HIP: SAME_AS_BRIGHT_TOO_HIP - GD1_BRIGHT_PM: {UNOBS: 1520, MORE_ZGOOD: 1520, MORE_ZWARN: 1520, DONE: 2, OBS: 1, DONOTOBSERVE: 0, MORE_MIDZQSO: 0} - GD1_FAINT_NO_PM: {UNOBS: 1510, MORE_ZGOOD: 1510, MORE_ZWARN: 1510, DONE: 2, OBS: 1, DONOTOBSERVE: 0, MORE_MIDZQSO: 0} - GD1_FILLER: {UNOBS: 49, MORE_ZGOOD: 49, MORE_ZWARN: 49, DONE: 2, OBS: 1, DONOTOBSERVE: 0, MORE_MIDZQSO: 0} + GD1_BRIGHT_PM: {UNOBS: 1521, MORE_ZGOOD: 1520, MORE_ZWARN: 1520, DONE: 2, OBS: 1, DONOTOBSERVE: 0, MORE_MIDZQSO: 0} + GD1_FAINT_NO_PM: {UNOBS: 1510, MORE_ZGOOD: 1511, MORE_ZWARN: 1511, DONE: 2, OBS: 1, DONOTOBSERVE: 0, MORE_MIDZQSO: 0} + GD1_FILLER: {UNOBS: 49, MORE_ZGOOD: 50, MORE_ZWARN: 50, DONE: 2, OBS: 1, DONOTOBSERVE: 0, MORE_MIDZQSO: 0} # ADM INITIAL number of observations (NUMOBS_INIT) for each target bit # ADM SAME_AS_XXX means to use the NUMOBS_INIT for bitname XXX @@ -571,6 +571,6 @@ numobs: DWF_BRIGHT_LO: 10 DWF_DARK_HI: 10 DWF_DARK_LO: 10 - GD1_BRIGHT_PM: 4 - GD1_FAINT_NO_PM: 4 - GD1_FILLER: 4 + GD1_BRIGHT_PM: 7 + GD1_FAINT_NO_PM: 7 + GD1_FILLER: 7 diff --git a/py/desitarget/test/test_mtl_streams.py b/py/desitarget/test/test_mtl_streams.py index 4524d3a0..b3428255 100644 --- a/py/desitarget/test/test_mtl_streams.py +++ b/py/desitarget/test/test_mtl_streams.py @@ -14,6 +14,8 @@ from desitarget.mtl import make_mtl, mtldatamodel, survey_data_model from desitarget.targets import initial_priority_numobs, main_cmx_or_sv +from desiutil.log import get_logger +log = get_logger() class TestMTLStreams(unittest.TestCase): @@ -64,14 +66,17 @@ def setUp(self): zwarn = [sMx[t].priorities['MORE_ZWARN'] for t in self.types[~iigood]] # ADM PRIORITY after zero, one, two, three passes through MTL. self.post_prio = pinit - self.post_prio = np.vstack([self.post_prio, zgood + zwarn]) - self.post_prio = np.vstack([self.post_prio, zgood + zwarn]) - self.post_prio = np.vstack([self.post_prio, zgood + zwarn]) + # ADM scalar version of initial numbers of observations. Should + # ADM (deliberately) fail if classes have different NUMOBS_INIT. + self.ninit_int = int(np.unique(ninit)) + # ADM loop through the numbers of observations, retain priority. + for i in range(self.ninit_int - 1): + self.post_prio = np.vstack([self.post_prio, zgood + zwarn]) self.post_prio = np.vstack( [self.post_prio, [sMx[t].priorities['DONE'] for t in self.types]]) # ADM NUMOBS after zero, one, two, three passes through MTL. self.post_nom = ninit - for numobs in np.arange(1, 5): + for numobs in np.arange(1, self.ninit_int + 1): self.post_nom = np.vstack([self.post_nom, np.array(self.nom) - numobs]) @@ -95,6 +100,8 @@ def test_numobs(self): t = self.targs.copy() t = self.flesh_out_data_model(t) mtl = make_mtl(t, "BRIGHT") + log.info(f"Initial: {mtl['PRIORITY']}, {self.post_prio[0]}") + log.info(f"Initial: {mtl['NUMOBS_MORE']}, {self.post_nom[0]}") self.assertTrue(np.all(mtl['NUMOBS_MORE'] == self.post_nom[0])) self.assertTrue(np.all(mtl['PRIORITY'] == self.post_prio[0])) @@ -107,9 +114,11 @@ def test_zcat(self): zc = self.zcat.copy() zc = self.flesh_out_data_model(zc) - for numobs in range(1, 5): + for numobs in range(1, self.ninit_int + 1): zc["NUMOBS"] = numobs mtl = make_mtl(t, "BRIGHT", zcat=zc, trim=False) + log.info(f"{numobs}, {mtl['PRIORITY']}, {self.post_prio[numobs]}") + log.info(f"{numobs}, {mtl['NUMOBS_MORE']}, {self.post_nom[numobs]}") self.assertTrue(np.all(mtl['PRIORITY'] == self.post_prio[numobs])) self.assertTrue(np.all(mtl['NUMOBS_MORE'] == self.post_nom[numobs])) From ea05ff8583172e6623fe65c0dca3926916ad383a Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Fri, 17 May 2024 10:15:17 -0700 Subject: [PATCH 57/63] code style --- py/desitarget/test/test_mtl_streams.py | 1 + 1 file changed, 1 insertion(+) diff --git a/py/desitarget/test/test_mtl_streams.py b/py/desitarget/test/test_mtl_streams.py index b3428255..2c9ce23e 100644 --- a/py/desitarget/test/test_mtl_streams.py +++ b/py/desitarget/test/test_mtl_streams.py @@ -17,6 +17,7 @@ from desiutil.log import get_logger log = get_logger() + class TestMTLStreams(unittest.TestCase): def setUp(self): From b1e1bbec3fe3dbf4a31274c19637e1d39486fe1f Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Fri, 17 May 2024 10:30:54 -0700 Subject: [PATCH 58/63] Try updating numpy to pass unit tests --- .github/workflows/python-package.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index fd991c5e..276a50ed 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -20,7 +20,7 @@ jobs: python-version: [3.9] astropy-version: ['==5.0', '<6'] # fuji version fitsio-version: ['==1.1.6'] # fuji version - numpy-version: ['<1.23'] # to keep asscalar, used by astropy + numpy-version: ['<1.24'] # to keep asscalar, used by astropy env: DESIUTIL_VERSION: 3.2.5 @@ -58,7 +58,7 @@ jobs: os: [ubuntu-latest] python-version: [3.9] fitsio-version: ['==1.1.6'] # fuji version - numpy-version: ['<1.23'] # to keep asscalar, used by astropy + numpy-version: ['<1.24'] # to keep asscalar, used by astropy env: DESIUTIL_VERSION: 3.2.5 DESIMODEL_DATA: branches/test-0.17 From 86f4ab1a6c634c1e684c6e45abe82aa5b4d90735 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Fri, 17 May 2024 10:43:26 -0700 Subject: [PATCH 59/63] try pinning matplotlib instead, to fix unit tests --- .github/workflows/python-package.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 276a50ed..6d28a21c 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -20,7 +20,8 @@ jobs: python-version: [3.9] astropy-version: ['==5.0', '<6'] # fuji version fitsio-version: ['==1.1.6'] # fuji version - numpy-version: ['<1.24'] # to keep asscalar, used by astropy + numpy-version: ['<1.23'] # to keep asscalar, used by astropy + matplotlib-version: ['<3.6.3'] # later versions of matplotlib require later versions of numpy. env: DESIUTIL_VERSION: 3.2.5 @@ -43,6 +44,7 @@ jobs: python -m pip install -r requirements.txt python -m pip install -U 'numpy${{ matrix.numpy-version }}' python -m pip install -U 'astropy${{ matrix.astropy-version }}' + python -m pip install -U 'matplotlib${{ matrix.matplotlib-version }}' python -m pip cache remove fitsio python -m pip install --no-deps --force-reinstall --ignore-installed 'fitsio${{ matrix.fitsio-version }}' svn export https://desi.lbl.gov/svn/code/desimodel/${DESIMODEL_DATA}/data @@ -58,7 +60,8 @@ jobs: os: [ubuntu-latest] python-version: [3.9] fitsio-version: ['==1.1.6'] # fuji version - numpy-version: ['<1.24'] # to keep asscalar, used by astropy + numpy-version: ['<1.23'] # to keep asscalar, used by astropy + matplotlib-version: ['<3.6.3'] # later versions of matplotlib require later versions of numpy. env: DESIUTIL_VERSION: 3.2.5 DESIMODEL_DATA: branches/test-0.17 From 6b7b7df1606e3df262909967866c47e5f63d856e Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Fri, 17 May 2024 10:53:15 -0700 Subject: [PATCH 60/63] also pin matplotlib for coverage tests --- .github/workflows/python-package.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 6d28a21c..03bd902d 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -82,6 +82,7 @@ jobs: python -m pip install git+https://github.com/desihub/desiutil.git@${DESIUTIL_VERSION}#egg=desiutil python -m pip install -r requirements.txt python -m pip install -U 'numpy${{ matrix.numpy-version }}' + python -m pip install -U 'matplotlib${{ matrix.matplotlib-version }}' python -m pip cache remove fitsio python -m pip install --no-deps --force-reinstall --ignore-installed 'fitsio${{ matrix.fitsio-version }}' svn export https://desi.lbl.gov/svn/code/desimodel/${DESIMODEL_DATA}/data From af6b63a85d16872b409c4c78f068f29a130dbfae Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Tue, 28 May 2024 13:33:53 -0700 Subject: [PATCH 61/63] handle case of single Booleans being passed instead of arrays (fixes a unit test failure introduced in PR #823) --- py/desitarget/cuts.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/py/desitarget/cuts.py b/py/desitarget/cuts.py index 417346f6..8fea0b49 100644 --- a/py/desitarget/cuts.py +++ b/py/desitarget/cuts.py @@ -2599,13 +2599,21 @@ def set_target_bits(photsys_north, photsys_south, obs_rflux, # ADM when resolving, strictly only set bits for targets that pass # ADM the northern (southern) cuts in northern (southern) imaging. - res_north = np.ones_like(photsys_north) - res_south = np.ones_like(photsys_south) - if resolvetargs: - # ADM this construction guards against mutability but is far - # ADM quicker than making a copy. - res_north[:] = photsys_north[:] - res_south[:] = photsys_south[:] + # ADM first, guard against photsys quantities passed as scalars. + if isinstance(photsys_north, bool): + res_north = True + res_south = True + if resolvetargs: + res_north = photsys_north + res_south = photsys_south + else: + res_north = np.ones_like(photsys_north) + res_south = np.ones_like(photsys_south) + if resolvetargs: + # ADM this construction guards against mutability but is far + # ADM quicker than making a copy. + res_north[:] = photsys_north[:] + res_south[:] = photsys_south[:] # Construct the targetflag bits for DECaLS (i.e. South). desi_target = (lrg_south & res_south) * desi_mask.LRG_SOUTH From 0915d41b85388ac2a3584b577e9b7593780ac261 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Tue, 28 May 2024 13:35:26 -0700 Subject: [PATCH 62/63] update changes docs --- doc/changes.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/changes.rst b/doc/changes.rst index 486cbd3c..32e01e49 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -16,6 +16,7 @@ desitarget Change Log * Add the GD1 dark matter streams program [`PR #814`_]. * Substantial new code in the `streams` directory. * Addresses `issue #812`_. + * General unit tests fixes, including one introduced in `PR #823`_ .. _`issue #812`: https://github.com/desihub/desitarget/issues/812 .. _`PR #814`: https://github.com/desihub/desitarget/pull/814 From 0ba89028189a92ccccc6405e457850efb79c7cb6 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Tue, 28 May 2024 13:49:29 -0700 Subject: [PATCH 63/63] trivial change to re-trigger unit tests --- py/desitarget/cuts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py/desitarget/cuts.py b/py/desitarget/cuts.py index 8fea0b49..034cb5bf 100644 --- a/py/desitarget/cuts.py +++ b/py/desitarget/cuts.py @@ -2599,7 +2599,7 @@ def set_target_bits(photsys_north, photsys_south, obs_rflux, # ADM when resolving, strictly only set bits for targets that pass # ADM the northern (southern) cuts in northern (southern) imaging. - # ADM first, guard against photsys quantities passed as scalars. + # ADM first guard against photsys quantities bring passed as scalars. if isinstance(photsys_north, bool): res_north = True res_south = True