From fa25579c2a8c713c73c7f80ee5ceae1ab02ebe09 Mon Sep 17 00:00:00 2001 From: James McVey <53623232+jmcvey3@users.noreply.github.com> Date: Wed, 8 May 2024 12:32:55 -0700 Subject: [PATCH] Various Updates (#127) * Add function * Use better data for test * Ensure that noise subtraction is working properly * Correct error * Update test files with noise-subtracted variables * Should be using along-beam data for adcp turbulence test * Restore one line * Fix doppler noise variable coordinate names * Need to handle 'time' or 'time_b5' * Update changelog * Fix window check * Update documentation * Workarounds for dual profiling instrument * dual profile final fixes * Refactor 'create_dataset' function * Reinstate some skipped ping logic * Add ability to read ID 31, clean up altimeter attrs * Handle variable bottom track beams_cy * Update test attributes * Update changelog * Cleanup * Update notebook * Revert test file * Revert environment change * Attempt to fix things * Don't add extra dims * Add test data * Small fixes * Remove unused code * Update dependency * More clarity on tke in notebooks * Git lfs pointer warning * Fix test data * Add requirements to conda env --- changelog.md | 6 + docs/ADCP_Example.ipynb | 5549 ++++++++++++----- docs/ADV_Example.ipynb | 611 +- docs/apidoc/dolfyn.binners.rst | 6 +- dolfyn/adp/turbulence.py | 74 +- dolfyn/adv/turbulence.py | 35 +- dolfyn/binned.py | 2 +- dolfyn/example_data/dual_profile.ad2cp | 3 + dolfyn/io/api.py | 8 +- dolfyn/io/base.py | 186 +- dolfyn/io/nortek2.py | 224 +- dolfyn/io/nortek2_lib.py | 91 +- dolfyn/tests/data/BenchFile01.nc | 4 +- dolfyn/tests/data/BenchFile01_avg.nc | 4 +- .../data/BenchFile01_rotate_beam2inst.nc | 4 +- .../BenchFile01_rotate_earth2principal.nc | 4 +- .../data/BenchFile01_rotate_inst2earth.nc | 4 +- dolfyn/tests/data/Sig1000_IMU_bin.nc | 3 - dolfyn/tests/data/Sig1000_tidal_bin.nc | 3 + dolfyn/tests/data/dual_profile.nc | 3 + dolfyn/tests/data/vector_data01_bin.nc | 4 +- dolfyn/tests/make_data.py | 1 + dolfyn/tests/test_analysis.py | 57 +- dolfyn/tests/test_read_adp.py | 6 + dolfyn/tools/psd.py | 21 +- dolfyn/velocity.py | 82 +- environment.yml | 8 +- requirements.txt | 2 +- 28 files changed, 4545 insertions(+), 2460 deletions(-) create mode 100644 dolfyn/example_data/dual_profile.ad2cp delete mode 100644 dolfyn/tests/data/Sig1000_IMU_bin.nc create mode 100644 dolfyn/tests/data/Sig1000_tidal_bin.nc create mode 100644 dolfyn/tests/data/dual_profile.nc diff --git a/changelog.md b/changelog.md index 8f2cce09..5616595c 100644 --- a/changelog.md +++ b/changelog.md @@ -13,10 +13,16 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Fix netCDF4 compression encoding - Retain prior netCDF4 variable encoding - Fix bug in reading raw Nortek Signature altimeter data + - Fix bug where noise input wasn't being subtracted from auto-spectra + - Fix bug that would error out when entering custom FFT window - API/Useability - Updates to support python 3.10 and 3.11 - Added ability to read Nortek AWAC waves data + - Added ability to subtract Doppler noise in TKE dissipation rate functions + - Added function to calculate turbulence intensity and remove noise + - Add ability to read Nortek dual profiling instruments + - Add ability to read ID 31 (initial altimeter scan for averaged altimeter measurements) - Nortek Vectrino (.vno) - Add support for Nortek Vectrino (.vno) files. diff --git a/docs/ADCP_Example.ipynb b/docs/ADCP_Example.ipynb index 8728c02f..55013099 100644 --- a/docs/ADCP_Example.ipynb +++ b/docs/ADCP_Example.ipynb @@ -1,1584 +1,3969 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# ADCP Example" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The following example shows a typical workflow for analyzing ADCP data using DOLfYN's tools.\n", - "\n", - "A typical ADCP data workflow is broken down into\n", - " 1. Review the raw data\n", - " - Check timestamps\n", - " - Calculate/check that the depth bin locations are correct\n", - " - Look at velocity, beam amplitude and/or beam correlation data quality\n", - " 2. Remove data located above the water surface or below the seafloor\n", - " 3. Check for spurious datapoints and remove if necessary\n", - " 4. If not already done within the instrument, average the data into bins of a set time length (normally 5 to 10 min)\n", - " 5. Conduct further analysis as required\n", - "\n", - "Start by importing the necessary DOLfYN tools through MHKiT:" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "# Import core DOLfYN functions\n", - "import dolfyn\n", - "# Import ADCP-specific API tools\n", - "from dolfyn.adp import api" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Read Raw Instrument Data" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The core benefit of DOLfYN is that it can read in raw data directly after transferring it off of the ADCP. The ADCP used here is a Nortek Signature 1000, with the file extension '.ad2cp'. This specific dataset contains several hours worth of velocity data collected at 1 Hz from the ADCP mounted on a bottom lander in a tidal inlet. \n", - "The instruments that DOLfYN supports are listed in the [docs](https://dolfyn.readthedocs.io/en/latest/about.html).\n", - "\n", - "Start by reading in the raw datafile downloaded from the instrument. The `read` function reads the raw file and dumps the information into an xarray Dataset, which contains a few groups of variables:\n", - "\n", - "1. Velocity in the instrument-saved coordinate system (beam, XYZ, ENU)\n", - "2. Beam amplitude and correlation data\n", - "3. Measurements of the instrument's bearing and environment\n", - "4. Orientation matrices DOLfYN uses for rotating through coordinate frames." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Reading file ../dolfyn/example_data/Sig1000_tidal.ad2cp ...\n" - ] - } - ], - "source": [ - "ds = dolfyn.read('../dolfyn/example_data/Sig1000_tidal.ad2cp')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "There are two ways to see what's in a DOLfYN Dataset. The first is to simply type the dataset's name to see the standard xarray output. To access a particular variable in a dataset, use dict-style (`ds['vel']`) or attribute-style syntax (`ds.vel`). See the [xarray docs](http://xarray.pydata.org/en/stable/getting-started-guide/quick-overview.html) for more details on how to use the xarray format." - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
<xarray.Dataset>\n", - "Dimensions: (time: 55000, dirIMU: 3, dir: 4, range: 28, beam: 4,\n", - " earth: 3, inst: 3, q: 4, time_b5: 55000,\n", - " range_b5: 28, x: 4, x*: 4)\n", - "Coordinates:\n", - " * time (time) datetime64[ns] 2020-08-15T00:20:00.500999927 ...\n", - " * dirIMU (dirIMU) <U1 'E' 'N' 'U'\n", - " * dir (dir) <U2 'E' 'N' 'U1' 'U2'\n", - " * range (range) float64 0.6 1.1 1.6 2.1 ... 12.6 13.1 13.6 14.1\n", - " * beam (beam) int32 1 2 3 4\n", - " * earth (earth) <U1 'E' 'N' 'U'\n", - " * inst (inst) <U1 'X' 'Y' 'Z'\n", - " * q (q) <U1 'w' 'x' 'y' 'z'\n", - " * time_b5 (time_b5) datetime64[ns] 2020-08-15T00:20:00.4384999...\n", - " * range_b5 (range_b5) float64 0.6 1.1 1.6 2.1 ... 13.1 13.6 14.1\n", - " * x (x) int32 1 2 3 4\n", - " * x* (x*) int32 1 2 3 4\n", - "Data variables: (12/38)\n", - " c_sound (time) float32 1.502e+03 1.502e+03 ... 1.498e+03\n", - " temp (time) float32 14.55 14.55 14.55 ... 13.47 13.47 13.47\n", - " pressure (time) float32 9.713 9.718 9.718 ... 9.596 9.594 9.596\n", - " mag (dirIMU, time) float32 72.5 72.7 72.6 ... -197.2 -195.7\n", - " accel (dirIMU, time) float32 -0.00479 -0.01437 ... 9.729\n", - " batt (time) float32 16.6 16.6 16.6 16.6 ... 16.4 16.4 15.2\n", - " ... ...\n", - " telemetry_data (time) uint8 0 0 0 0 0 0 0 0 0 0 ... 0 0 0 0 0 0 0 0 0\n", - " boost_running (time) uint8 0 0 0 0 0 0 0 0 1 0 ... 0 1 0 0 0 0 0 0 1\n", - " heading (time) float32 -12.52 -12.51 -12.51 ... -12.52 -12.5\n", - " pitch (time) float32 -0.065 -0.06 -0.06 ... -0.06 -0.05 -0.05\n", - " roll (time) float32 -7.425 -7.42 -7.42 ... -6.45 -6.45 -6.45\n", - " beam2inst_orientmat (x, x*) float32 1.183 0.0 -1.183 ... 0.5518 0.0 0.5518\n", - "Attributes: (12/33)\n", - " filehead_config: {'CLOCKSTR': {'TIME': '"2020-08-13 13:56:21"'}, 'I...\n", - " inst_model: Signature1000\n", - " inst_make: Nortek\n", - " inst_type: ADCP\n", - " rotate_vars: ['vel', 'accel', 'accel_b5', 'angrt', 'angrt_b5', ...\n", - " burst_config: {'press_valid': True, 'temp_valid': True, 'compass...\n", - " ... ...\n", - " proc_idle_less_3pct: 0\n", - " proc_idle_less_6pct: 0\n", - " proc_idle_less_12pct: 0\n", - " coord_sys: earth\n", - " has_imu: 1\n", - " fs: 1
<xarray.DataArray 'range' (range: 28)>\n", - "array([ 1.2, 1.7, 2.2, 2.7, 3.2, 3.7, 4.2, 4.7, 5.2, 5.7, 6.2, 6.7,\n", - " 7.2, 7.7, 8.2, 8.7, 9.2, 9.7, 10.2, 10.7, 11.2, 11.7, 12.2, 12.7,\n", - " 13.2, 13.7, 14.2, 14.7])\n", - "Coordinates:\n", - " * range (range) float64 1.2 1.7 2.2 2.7 3.2 ... 12.7 13.2 13.7 14.2 14.7\n", - "Attributes:\n", - " units: m