diff --git a/code.json b/code.json index fab3978..5c17bfd 100644 --- a/code.json +++ b/code.json @@ -1,8 +1,4 @@ -{ - "version": "2.0.0", - "measurementType": {}, - "agency": "U.S. Department of Interior", - "releases": [ + [ { "name": "libcomcat", "description": "Python wrapper around ComCat web API", @@ -41,4 +37,3 @@ } } ] -} diff --git a/environment.yml b/environment.yml index abc5bbd..a975624 100644 --- a/environment.yml +++ b/environment.yml @@ -5,25 +5,15 @@ channels: - digitalglobe - ioos dependencies: -- beautifulsoup4 -- cartopy -- decorator -- fiona -- h5py - ipython - jupyter -- matplotlib<2.0 - numpy - obspy - pandas -- paramiko -- pycrypto - pytest - pytest-cov - pytest-mpl - python=3.5 -- seaborn -- shapely - xlrd - xlwt - openpyxl diff --git a/environment_linux.yml b/environment_linux.yml index c8fb358..07631c5 100644 --- a/environment_linux.yml +++ b/environment_linux.yml @@ -4,171 +4,132 @@ channels: - defaults - ioos dependencies: -- asn1crypto=0.22.0 -- attrs=17.3.0 -- beautifulsoup4=4.6.0 +- asn1crypto=0.24.0 +- attrs=17.4.0 +- backports=1.0 +- backports.functools_lru_cache=1.5 - blas=1.1 - bleach=2.0.0 -- ca-certificates=2017.11.5 -- cairo=1.14.6 -- cartopy=0.15.1 -- certifi=2017.11.5 -- cffi=1.11.2 +- ca-certificates=2018.1.18 +- certifi=2018.1.18 +- cffi=1.11.5 - chardet=3.0.4 -- click=6.7 -- click-plugins=1.0.3 -- cligj=0.4.0 -- coverage=4.4.2 +- coverage=4.5.1 - cryptography=2.1.4 -- curl=7.55.1 - cycler=0.10.0 -- decorator=4.1.2 +- dbus=1.10.22 +- decorator=4.2.1 - entrypoints=0.2.3 - et_xmlfile=1.0.1 -- expat=2.1.0 -- fiona=1.7.11 -- fontconfig=2.12.1 -- freetype=2.7 -- freexl=1.0.4 +- expat=2.2.5 +- fontconfig=2.12.6 +- freetype=2.8.1 - future=0.16.0 -- gdal=2.1.3 -- geos=3.6.2 -- gettext=0.19.7 -- giflib=5.1.4 -- glib=2.51.4 +- gettext=0.19.8.1 +- glib=2.55.0 - gmp=6.1.2 -- graphite2=1.3.10 -- h5py=2.7.1 -- harfbuzz=1.4.3 -- hdf4=4.2.13 -- hdf5=1.10.1 +- gst-plugins-base=1.8.0 +- gstreamer=1.8.0 - html5lib=1.0.1 - icu=58.2 - idna=2.6 -- ipykernel=4.7.0 +- ipykernel=4.8.2 - ipython=6.2.1 - ipython_genutils=0.2.0 -- ipywidgets=7.0.5 +- ipywidgets=7.1.2 - jdcal=1.3 -- jedi=0.10.2 +- jedi=0.11.1 - jinja2=2.10 - jpeg=9b -- json-c=0.12.1 - jsonschema=2.6.0 - jupyter=1.0.0 -- jupyter_client=5.2.0 +- jupyter_client=5.2.2 - jupyter_console=5.2.0 - jupyter_core=4.4.0 -- kealib=1.4.7 -- krb5=1.14.2 -- libdap4=3.18.3 +- kiwisolver=1.0.1 - libffi=3.2.1 - libiconv=1.15 -- libnetcdf=4.4.1.1 -- libpng=1.6.28 -- libpq=9.6.3 +- libpng=1.6.34 - libsodium=1.0.15 -- libspatialite=4.3.0a -- libssh2=1.8.0 -- libtiff=4.0.9 -- libxml2=2.9.5 +- libxcb=1.12 +- libxml2=2.9.7 - libxslt=1.1.32 - lxml=4.1.1 - markupsafe=1.0 -- matplotlib=1.5.3 +- matplotlib=2.2.0 - mistune=0.8.3 -- munch=2.2.0 - nbconvert=5.3.1 - nbformat=4.4.0 - ncurses=5.9 - nose=1.3.7 -- notebook=5.2.2 -- numpy=1.13.3 +- notebook=5.4.0 +- numpy=1.14.1 - obspy=1.1.0 -- olefile=0.44 - openblas=0.2.20 -- openjpeg=2.1.2 -- openpyxl=2.5.0b1 +- openpyxl=2.5.0 - openssl=1.0.2n -- owslib=0.15.0 -- pandas=0.21.1 -- pandoc=2.0.5 +- pandas=0.22.0 +- pandoc=2.1.2 - pandocfilters=1.4.1 -- pango=1.40.4 -- paramiko=2.3.1 -- patsy=0.4.1 +- parso=0.1.1 - pcre=8.39 -- pexpect=4.3.1 +- pexpect=4.4.0 - pickleshare=0.7.4 -- pillow=4.2.1 - pip=9.0.1 -- pixman=0.34.0 - pluggy=0.6.0 -- proj4=4.9.3 - prompt_toolkit=1.0.15 - ptyprocess=0.5.2 - py=1.5.2 -- pyasn1=0.3.7 - pycparser=2.18 -- pycrypto=2.6.1 -- pyepsg=0.3.2 - pygments=2.2.0 -- pynacl=1.1.2 -- pyopenssl=17.4.0 +- pyopenssl=17.5.0 - pyparsing=2.2.0 -- pyproj=1.9.5.1 -- pyqt=4.11.4 -- pyshp=1.2.12 -- pysocks=1.6.7 -- pytest=3.3.1 +- pyqt=5.6.0 +- pysocks=1.6.8 +- pytest=3.4.2 - pytest-cov=2.5.1 - pytest-mpl=0.9 -- python=3.5.4 +- python=3.5.5 - python-dateutil=2.6.1 -- pytz=2017.3 -- pyzmq=16.0.2 -- qt=4.8.7 +- pytz=2018.3 +- pyzmq=17.0.0 +- qt=5.6.2 - qtconsole=4.3.1 -- readline=6.2 +- readline=7.0 - requests=2.18.4 - scipy=1.0.0 -- seaborn=0.8.1 -- setuptools=38.2.4 -- shapely=1.6.3 +- send2trash=1.5.0 +- setuptools=38.5.1 - simplegeneric=0.8.1 - sip=4.18 - six=1.11.0 -- sqlalchemy=1.1.13 -- sqlite=3.13.0 -- statsmodels=0.8.0 +- sqlalchemy=1.2.4 +- sqlite=3.20.1 - terminado=0.8.1 - testpath=0.3.1 -- tk=8.5.19 -- tornado=4.5.2 +- tk=8.6.7 +- tornado=4.5.3 - traitlets=4.3.2 - urllib3=1.22 - wcwidth=0.1.7 - webencodings=0.5 - wheel=0.30.0 -- widgetsnbextension=3.0.8 -- xerces-c=3.2.0 +- widgetsnbextension=3.1.4 - xlrd=1.1.0 - xlsxwriter=1.0.2 - xlwt=1.3.0 +- xorg-libxau=1.0.8 +- xorg-libxdmcp=1.1.2 - xz=5.2.3 -- zeromq=4.2.1 +- zeromq=4.2.3 - zlib=1.2.11 -- bcrypt=3.1.4 -- libgcc-ng=7.2.0 - libgfortran=3.0.0 -- util-linux=2.21 - pip: + - backports.functools-lru-cache==1.5 - et-xmlfile==1.0.1 - ipython-genutils==0.2.0 - - jupyter-client==5.2.0 + - jupyter-client==5.2.2 - jupyter-console==5.2.0 - jupyter-core==4.4.0 - prompt-toolkit==1.0.15 - - ./impact-utils.zip -prefix: /home/mhearne/miniconda/envs/comcat - + - ./impact-utils.zip diff --git a/environment_osx.yml b/environment_osx.yml index 9061246..ab48f23 100644 --- a/environment_osx.yml +++ b/environment_osx.yml @@ -6,162 +6,120 @@ channels: - ioos dependencies: - appnope=0.1.0 -- asn1crypto=0.22.0 -- attrs=17.3.0 -- autopep8=1.3.3 -- beautifulsoup4=4.6.0 +- asn1crypto=0.24.0 +- attrs=17.4.0 +- backports=1.0 +- backports.functools_lru_cache=1.5 - blas=1.1 - bleach=2.0.0 -- ca-certificates=2017.11.5 -- cartopy=0.15.1 -- certifi=2017.11.5 -- cffi=1.11.2 +- ca-certificates=2018.1.18 +- certifi=2018.1.18 +- cffi=1.11.5 - chardet=3.0.4 -- click=6.7 -- click-plugins=1.0.3 -- cligj=0.4.0 -- coverage=4.4.2 +- coverage=4.5.1 - cryptography=2.1.4 -- curl=7.55.1 - cycler=0.10.0 -- decorator=4.1.2 +- decorator=4.2.1 - entrypoints=0.2.3 - et_xmlfile=1.0.1 -- expat=2.1.0 -- fiona=1.7.11 -- freetype=2.6.3 -- freexl=1.0.4 +- freetype=2.8.1 - future=0.16.0 -- gdal=2.1.3 -- geos=3.6.2 -- giflib=5.1.4 -- h5py=2.7.1 -- hdf4=4.2.13 -- hdf5=1.10.1 - html5lib=1.0.1 - icu=58.2 - idna=2.6 -- ipykernel=4.7.0 +- ipykernel=4.8.2 - ipython=6.2.1 - ipython_genutils=0.2.0 -- ipywidgets=7.0.5 +- ipywidgets=7.1.2 - jdcal=1.3 -- jedi=0.10.2 +- jedi=0.11.1 - jinja2=2.10 - jpeg=9b -- json-c=0.12.1 - jsonschema=2.6.0 - jupyter=1.0.0 -- jupyter_client=5.1.0 +- jupyter_client=5.2.2 - jupyter_console=5.2.0 - jupyter_core=4.4.0 -- kealib=1.4.7 -- krb5=1.14.2 -- libdap4=3.18.3 +- kiwisolver=1.0.1 - libffi=3.2.1 - libgfortran=3.0.0 - libiconv=1.15 -- libnetcdf=4.4.1.1 -- libpng=1.6.28 -- libpq=9.6.3 +- libpng=1.6.34 - libsodium=1.0.15 -- libspatialite=4.3.0a -- libssh2=1.8.0 -- libtiff=4.0.9 -- libxml2=2.9.5 +- libxml2=2.9.7 - libxslt=1.1.32 - lxml=4.1.1 - markupsafe=1.0 -- matplotlib=2.0.0rc2 +- matplotlib=2.2.0 - mistune=0.8.3 -- munch=2.2.0 - nbconvert=5.3.1 - nbformat=4.4.0 - ncurses=5.9 - nose=1.3.7 -- notebook=5.2.2 -- numpy=1.12.1 +- notebook=5.4.0 +- numpy=1.14.1 - obspy=1.1.0 -- olefile=0.44 - openblas=0.2.20 -- openjpeg=2.1.2 -- openpyxl=2.5.0b1 +- openpyxl=2.5.0 - openssl=1.0.2n -- owslib=0.15.0 -- pandas=0.21.1 -- pandoc=2.0.5 +- pandas=0.22.0 +- pandoc=2.1.2 - pandocfilters=1.4.1 -- paramiko=2.3.1 -- patsy=0.4.1 -- pcre=8.39 -- pexpect=4.3.1 +- parso=0.1.1 +- pexpect=4.4.0 - pickleshare=0.7.4 -- pillow=4.0.0 - pip=9.0.1 - pluggy=0.6.0 -- proj4=4.9.3 - prompt_toolkit=1.0.15 - ptyprocess=0.5.2 - py=1.5.2 -- pyasn1=0.3.7 -- pycodestyle=2.3.1 - pycparser=2.18 -- pycrypto=2.6.1 -- pyepsg=0.3.2 - pygments=2.2.0 -- pynacl=1.1.2 -- pyopenssl=17.4.0 +- pyopenssl=17.5.0 - pyparsing=2.2.0 -- pyproj=1.9.5.1 - pyqt=5.6.0 -- pyshp=1.2.12 -- pysocks=1.6.7 -- pytest=3.3.1 +- pysocks=1.6.8 +- pytest=3.4.2 - pytest-cov=2.5.1 - pytest-mpl=0.9 -- python=3.5.4 +- python=3.5.5 - python-dateutil=2.6.1 -- pytz=2017.3 -- pyzmq=16.0.2 +- pytz=2018.3 +- pyzmq=17.0.0 - qt=5.6.2 - qtconsole=4.3.1 -- readline=6.2 +- readline=7.0 - requests=2.18.4 - scipy=1.0.0 -- seaborn=0.8.1 -- setuptools=38.2.4 -- shapely=1.6.3 +- send2trash=1.5.0 +- setuptools=38.5.1 - simplegeneric=0.8.1 - sip=4.18 - six=1.11.0 -- sqlalchemy=1.1.13 -- sqlite=3.13.0 -- statsmodels=0.8.0 +- sqlalchemy=1.2.1 +- sqlite=3.20.1 - terminado=0.8.1 - testpath=0.3.1 -- tk=8.5.19 -- tornado=4.5.2 +- tk=8.6.7 +- tornado=4.5.3 - traitlets=4.3.2 - urllib3=1.22 - wcwidth=0.1.7 - webencodings=0.5 - wheel=0.30.0 -- widgetsnbextension=3.0.8 -- xerces-c=3.2.0 +- widgetsnbextension=3.1.4 - xlrd=1.1.0 - xlsxwriter=1.0.2 - xlwt=1.3.0 - xz=5.2.3 -- zeromq=4.2.1 +- zeromq=4.2.3 - zlib=1.2.11 -- bcrypt=3.1.4 - pip: + - backports.functools-lru-cache==1.5 - et-xmlfile==1.0.1 - ipython-genutils==0.2.0 - - jupyter-client==5.1.0 + - jupyter-client==5.2.2 - jupyter-console==5.2.0 - jupyter-core==4.4.0 - prompt-toolkit==1.0.15 - - ./impact-utils.zip -prefix: /Users/mhearne/anaconda3/envs/comcat - + - ./impact-utils.zip diff --git a/libcomcat/classes.py b/libcomcat/classes.py index f1d4519..d231778 100644 --- a/libcomcat/classes.py +++ b/libcomcat/classes.py @@ -11,6 +11,7 @@ import re from enum import Enum import sys +from io import StringIO # third party imports from obspy.core.event import read_events @@ -514,27 +515,20 @@ def toDict(self, catalog=None, edict.update(_get_focal_mechanism_info(focal)) if get_all_magnitudes: - handle, tmpfile = tempfile.mkstemp() - os.close(handle) - try: - phase_data = self.getProducts('phase-data')[0] - phase_data.getContent('quakeml.xml', filename=tmpfile) - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - catalog = read_events(tmpfile) - event = catalog.events[0] - imag = 1 - if get_all_magnitudes: - for magnitude in event.magnitudes: - edict['magnitude%i' % imag] = magnitude.mag - edict['magtype%i' % - imag] = magnitude.magnitude_type - imag += 1 - except: - raise Exception( - 'Failed to retrieve quakeml.xml file from phase-data product (%s).' % content_url) - finally: - os.remove(tmpfile) + phase_data = self.getProducts('phase-data')[0] + phase_bytes,url = phase_data.getContentBytes('quakeml.xml') + phase_io = StringIO(phase_bytes.decode('utf-8')) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + catalog = read_events(phase_io) + event = catalog.events[0] + imag = 1 + if get_all_magnitudes: + for magnitude in event.magnitudes: + edict['magnitude%i' % imag] = magnitude.mag + edict['magtype%i' % + imag] = magnitude.magnitude_type + imag += 1 return edict @@ -775,18 +769,39 @@ def getContentURL(self, regexp): else: return None - def getContent(self, regexp, filename=None): + def getContent(self, regexp, filename): """Find and download the shortest file name matching the input regular expression. :param regexp: Regular expression which should match one of the content files in the Product. :param filename: - Filename to which content should be downloaded. + Filename to which content should be downloaded. :returns: The URL from which the content was downloaded. :raises: Exception if content could not be downloaded from ComCat after two tries. """ + data,url = self.getContentBytes(regexp) + + f = open(filename, 'wb') + f.write(data) + f.close() + + return url + + + def getContentBytes(self, regexp): + """Find the shortest file name matching the input regular expression, return bytes of that file. + + :param regexp: + Regular expression which should match one of the content files in the Product. + :returns: + Tuple of array of bytes containing file contents, and the source url. Bytes can be + decoded to UTF-8 by the user if file contents are known to be ASCII. i.e., + product.getContentBytes('info.json').decode('utf-8') + :raises: + Exception if content could not be downloaded from ComCat after two tries. + """ content_name = 'a' * 1000 content_url = None for contentkey, content in self._product['contents'].items(): @@ -806,23 +821,19 @@ def getContent(self, regexp, filename=None): fh = request.urlopen(url, timeout=TIMEOUT) data = fh.read() fh.close() - f = open(filename, 'wb') - f.write(data) - f.close() + except HTTPError as htpe: time.sleep(WAITSECS) try: fh = request.urlopen(url, timeout=TIMEOUT) data = fh.read() fh.close() - f = open(filename, 'wb') - f.write(data) - f.close() except Exception as msg: raise Exception('Could not download %s from %s.' % (content_name, url)) - return url + + return (data,url) def hasProperty(self, key): """Determine if this Product contains a given property. diff --git a/tests/libcomcat/classes_test.py b/tests/libcomcat/classes_test.py index 6d26ca7..81e1fd4 100755 --- a/tests/libcomcat/classes_test.py +++ b/tests/libcomcat/classes_test.py @@ -235,7 +235,7 @@ def test_product(): assert product.getContentURL('foo') is None try: - product.getContent('foo') + product.getContent('foo',filename=None) assert 1==2 except AttributeError as ae: pass @@ -259,6 +259,13 @@ def test_product(): finally: os.remove(tfilename) + # test getting content as a string. + infobytes,url = product.getContentBytes('info.json') + infostring = infobytes.decode('utf-8') + jdict = json.loads(infostring) + eid = jdict['input']['event_information']['event_id'] + assert eid == '19940117123055' + if __name__ == '__main__': test_moment_supplement() test_detail_product_versions()