pyPI conforming changes (#4)

* synthetic injection pipeline frame work, scatter stars to come * synthetic injection pipeline complete, begin bug check * injextnamelist rename * verbose increase for troubleshoot * verbose * is injextnames empty? * injextnames * eliminate injextnames * filter is just a letter but fits header is soooo long... * merge extname, name, and key to key * supress warnings from load nonstd keywords * apparently and ordered dictionary has to be defined... python nonsense * zero safe inversion * redefine cat * make sure to add I before the injected run loop * help with I handling * export outmodel to big dig and pass bigdict tothe save fxn * errant colon * just edit big dict rather than recopy params * update big dict so we can just pass everything to save fxn * change mock cat header name * match Eddie's xy convention, I think psfmod.render will need this * FIX ME number of inject limit * different seed for fits compressin test * update leda to remove by hand list AKS * verbose * more verbose * lift assertion threshold * fix path from scratch to home * recompute coordinates on reduced list * verbose * verbose diam_mod * verbose idx_mod and idx * verbose idx_bad * idx and idx_bad orders seem to have been messed up during the last PR * remove verbose * point to new galmask path * wrap injection write files to clean up deacm_proc * injectfrac fix * psf_shift now just a simple call * seed rng with date of file for reproducible but different inject tests * save decam_proc * import os * add overwrite * injectfrac consistent naming * silent fits files! * indent * indent * force injected sources to be positive * verbose * isnan not is_nan... * verbose gain? * seems to be in the psf shift * print min to confirm it is small and I can set to zero * more psf stats * print key * update size to account for negative stars stripped out * sample star positions with the date seeded rng * save the mock catalogue with the Eddie normalization * psfmod define * scattering returns 6 values * need to have length x now * rename main folder, add pypi required setup.py * remove intermediate files
schlafly · Oct 16, 2021 · 1b0cc73 · 1b0cc73
1 parent 6662e79
commit 1b0cc73
Show file tree

Hide file tree

Showing 20 changed files with 51 additions and 20 deletions.
diff --git a/python/__init__.py → crowdsource/__init__.py b/python/__init__.py → crowdsource/__init__.py
diff --git a/python/basisspline.py → crowdsource/basisspline.py b/python/basisspline.py → crowdsource/basisspline.py
diff --git a/python/crowdsource.py → crowdsource/crowdsource.py b/python/crowdsource.py → crowdsource/crowdsource.py
@@ -16,7 +16,7 @@
 
 import numpy
 import pdb
-import psf as psfmod
+import crowdsource.psf as psfmod
 import scipy.ndimage.filters as filters
 from collections import OrderedDict
 import sys

diff --git a/python/decam_inject.py → crowdsource/decam_inject.py b/python/decam_inject.py → crowdsource/decam_inject.py
@@ -3,8 +3,8 @@
 import copy
 import numpy as np
 from functools import partial
-import psf as psfmod
-import decam_proc
+import crowdsource.psf as psfmod
+import crowdsource.decam_proc as decam_proc
 from collections import OrderedDict
 import os
 

diff --git a/python/decam_proc.py → crowdsource/decam_proc.py b/python/decam_proc.py → crowdsource/decam_proc.py
@@ -6,7 +6,7 @@
 import argparse
 import numpy
 import numpy as np
-import psf as psfmod
+import crowdsource.psf as psfmod
 from astropy.io import fits
 from astropy import wcs
 from functools import partial

diff --git a/python/deconv_psf.py → crowdsource/deconv_psf.py b/python/deconv_psf.py → crowdsource/deconv_psf.py
@@ -3,7 +3,7 @@
 import numpy
 from skimage import restoration
 from astropy.io import fits
-import psf
+import crowdsource.psf as psf
 
 filt = 'ugrizY'
 deconv = {'u': 0.8, 'g': 0.75, 'r': 0.7, 'i': 0.6, 'z': 0.65, 'Y': 0.65}

diff --git a/python/examples/decam_cmds.txt → crowdsource/examples/decam_cmds.txt b/python/examples/decam_cmds.txt → crowdsource/examples/decam_cmds.txt
@@ -120,4 +120,21 @@ python /n/holylfs05/LABS/finkbeiner_lab/Lab/saydjari/ExtSoftware/crowdsource_new
 
 ###
 
-cat decaps_1per_run.txt | parallel --eta --delay 2 -j 45 --colsep ' ' --joblog log/runtask.log --resume "python /n/holylfs05/LABS/finkbeiner_lab/Lab/saydjari/ExtSoftware/crowdsource_new/python/decam_proc.py --nthreads 15 -v -r -c -q -g -m -d cat -e mod --inject 1 {1} {2} {3} {4} >>log/{2/.}.log 2>&1"
+cat decaps_1per_run.txt | parallel --eta --delay 2 -j 45 --colsep ' ' --joblog log/runtask.log --resume "python /n/holylfs05/LABS/finkbeiner_lab/Lab/saydjari/ExtSoftware/crowdsource_new/python/decam_proc.py --nthreads 15 -v -r -c -q -g -m -d cat -e mod --inject 1 {1} {2} {3} {4} >>log/{2/.}.log 2>&1"
+
+###
+
+python /n/holylfs05/LABS/finkbeiner_lab/Lab/saydjari/ExtSoftware/crowdsource_new/python/decam_proc.py -v -r -c -q -g -m -d cat -e mod --ccdlist N12 --injccdlist N12 --inject -1 /n/fink2/decaps/c4d_ 170504_045213 Y v1
+
+srun --exclusive -N1 -n1 --mem-per-cpu=4000 python /n/holylfs05/LABS/finkbeiner_lab/Lab/saydjari/ExtSoftware/crowdsource_new/python/decam_proc.py -v -r -c -q -g -m -d cat -e mod --ccdlist N2 --injccdlist N2 --inject -1 /n/fink2/decaps/c4d_ 170504_045213 Y v1
+
+### 
+
+cgcreate -g memory:holyfink01
+cgset -r memory.limit_in_bytes=7G holyfink01
+
+###
+
+cat decaps_1per_run.txt | parallel --eta --workdir . --sshloginfile nodefile --delay 2 --colsep ' ' --joblog log/runtask.log --resume "python /n/holylfs05/LABS/finkbeiner_lab/Lab/saydjari/Working/2021_09_07/test.py {1} {2} {3} {4} >>log/{2/.}.log 2>&1"
+
+cat decaps_1per_run.txt | parallel --eta --workdir . --filter-hosts --sshloginfile nodefile --delay 2 --colsep ' ' --joblog log/runtask.log --resume "python /n/holylfs05/LABS/finkbeiner_lab/Lab/saydjari/Working/2021_09_07/test.py {1} {2} {3} {4} >>log/{2/.}.log 2>&1"
diff --git a/python/examples/makecat.py → crowdsource/examples/makecat.py b/python/examples/makecat.py → crowdsource/examples/makecat.py
diff --git a/python/examples/wise_proc_example.sh → crowdsource/examples/wise_proc_example.sh b/python/examples/wise_proc_example.sh → crowdsource/examples/wise_proc_example.sh
diff --git a/python/galaxy_mask.py → crowdsource/galaxy_mask.py b/python/galaxy_mask.py → crowdsource/galaxy_mask.py
diff --git a/python/galconv.py → crowdsource/galconv.py b/python/galconv.py → crowdsource/galconv.py
diff --git a/python/mosaic.py → crowdsource/mosaic.py b/python/mosaic.py → crowdsource/mosaic.py
diff --git a/python/nebulosity_mask.py → crowdsource/nebulosity_mask.py b/python/nebulosity_mask.py → crowdsource/nebulosity_mask.py
diff --git a/python/psf.py → crowdsource/psf.py b/python/psf.py → crowdsource/psf.py
diff --git a/python/simple_proc.py → crowdsource/simple_proc.py b/python/simple_proc.py → crowdsource/simple_proc.py
diff --git a/python/unwise_primary.py → crowdsource/unwise_primary.py b/python/unwise_primary.py → crowdsource/unwise_primary.py
diff --git a/python/wise_proc.py → crowdsource/wise_proc.py b/python/wise_proc.py → crowdsource/wise_proc.py
@@ -10,7 +10,9 @@
 from astropy.io import fits
 import crowdsource
 from unwise_psf import unwise_psf
-import unwise_primary
+# implicit dependency for WISE runs only
+# https://github.com/legacysurvey/unwise_psf
+import crowdsource.unwise_primary as unwise_primary
 from astropy import wcs
 from collections import OrderedDict
 from pkg_resources import resource_filename

diff --git a/python/requirements.txt b/python/requirements.txt
diff --git a/setup.cfg b/setup.cfg
@@ -0,0 +1,2 @@
+[metadata]
+description-file = README.md
diff --git a/setup.py b/setup.py
@@ -0,0 +1,23 @@
+from distutils.core import setup
+
+setup(
+    name = 'crowdsource',
+    packages = ['crowdsource'],
+    version = '0.5.1',
+    description = 'Crowded field photometry pipeline',
+    author = 'Andrew Saydjari',
+    author_email = '[email protected]',
+    url = 'https://github.com/schlafly/crowdsource',
+    download_url = 'https://github.com/schlafly/crowdsource/archive/refs/tags/v0.5.1.tar.gz',
+    license = 'MIT',
+    install_requires=[
+        'astropy',
+        'numpy',
+        'scipy',
+        'scikit-image',
+        'guppy3',
+        'matplotlib',
+        'keras',
+        'tensorflow >= 2'
+    ]
+)