diff --git a/docs/available_parameters.rst b/docs/available_parameters.rst index 8baa21c..73f8d66 100644 --- a/docs/available_parameters.rst +++ b/docs/available_parameters.rst @@ -25,6 +25,7 @@ You can check which parameters are available with: SW_OPER_TECxTMS_2F SW_OPER_FACxTMS_2F SW_OPER_EEFxTMS_2F + SW_OPER_IPDxIRR_2F For Alpha-Charlie FAC: ``collection="SW_OPER_FAC_TMS_2F"``. @@ -61,6 +62,10 @@ For EEF:: EEF,RelErr,flags +For IPD:: + + Ne,Te,Background_Ne,Foreground_Ne,PCP_flag,Grad_Ne_at_100km,Grad_Ne_at_50km,Grad_Ne_at_20km,Grad_Ne_at_PCP_edge,ROD,RODI10s,RODI20s,delta_Ne10s,delta_Ne20s,delta_Ne40s,Num_GPS_satellites,mVTEC,mROT,mROTI10s,mROTI20s,IBI_flag,Ionosphere_region_flag,IPIR_index,Ne_quality_flag,TEC_STD + ---- ``models`` @@ -90,7 +95,7 @@ Flexible evaluation of models and defining new derived models is possible with t :: - SyncStatus, Kp10, Kp, Dst, IMF_BY_GSM, IMF_BZ_GSM, IMF_V, F10_INDEX, + SyncStatus, Kp10, Kp, Dst, IMF_BY_GSM, IMF_BZ_GSM, IMF_V, F107, F10_INDEX, OrbitDirection, QDOrbitDirection, OrbitSource, OrbitNumber, AscendingNodeTime, AscendingNodeLongitude, QDLat, QDLon, QDBasis, MLT, SunDeclination, @@ -98,10 +103,17 @@ Flexible evaluation of models and defining new derived models is possible with t SunLongitude, SunVector, DipoleAxisVector, NGPLatitude, NGPLongitude, DipoleTiltAngle, - UpwardCurrent, TotalCurrent, - DivergenceFreeCurrentFunction, F_AMPS, B_NEC_AMPS + UpwardCurrent, TotalCurrent, # AMPS + DivergenceFreeCurrentFunction, F_AMPS, B_NEC_AMPS # AMPS + + +.. note:: -NB: the AMPS model is currently accessible as "auxiliaries" instead of a "model". + - The AMPS model is currently accessible as "auxiliaries" instead of a "model" + - ``Kp`` provides the Kp values in fractional form (e.g 2.2), and ``Kp10`` is multiplied by 10 (as integers) + - ``F107`` is the hourly 10.7 cm solar radio flux value, and ``F10_INDEX`` is the daily average + - ``QDLat`` and ``QDLon`` are quasi-dipole coordinates + - ``OrbitDirection`` and ``QDOrbitDirection`` flags indicate if the satellite is moving towards or away from each pole, respectively for geographic and quasi-dipole magnetic poles. +1 for ascending, and -1 for descending (in latitude); 0 for no data. ---- diff --git a/docs/conf.py b/docs/conf.py index a6f2bea..8269189 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -96,7 +96,7 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +# html_static_path = ['_static'] # Custom sidebar templates, must be a dictionary that maps document names # to template names. diff --git a/docs/installation.rst b/docs/installation.rst index a4ae8a0..2f3849e 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -1,12 +1,16 @@ Installation and first usage ============================ -Installation ------------- +1. Installation +--------------- -Linux/Unix and Python ≥ 3.5 is required for full support (since cdflib requires ≥ 3.5). +.. note:: For VRE (Virtual Research Environment) users: -Python 3.4 can also be used, but conversion from CDF to pandas/xarray is not supported - you can still download and save CDF files - :meth:`viresclient.ReturnedData.to_file`, or download as CSV files and convert to pandas - :meth:`viresclient.ReturnedData.as_dataframe`. + viresclient is already installed so skip this step. Log in at https://vre.vires.services/ and refer to documentation at https://swarm-vre.readthedocs.io/ + + You will still need to configure viresclient (see step 2) + +Python ≥ 3.5 is required. Tested primarily on Linux, but macOS and Windows should also work (on v0.4+). It can currently be installed with:: @@ -15,20 +19,49 @@ It can currently be installed with:: Dependencies:: Jinja2 ≥ 2.10.0 - pandas ≥ 0.18.0 - cdflib = 0.3.9 tables ≥ 3.4.4 tqdm ≥ 4.23.0 - xarray ≥ 0.10.0 + cdflib ≥ 0.3.9 + pandas ≥ 0.18.0 + xarray ≥ 0.11.0 + +(pip will fetch these automatically - if you are using conda, it may be better to install these first using conda instead:: + + conda install jinja2 pytables tqdm pandas xarray + pip install viresclient + +Recommended setup if starting without Python already +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +1. Install Miniconda for Python 3.7: https://docs.conda.io/en/latest/miniconda.html +2. Create a new conda environment with some recommended packages:: + + conda create --name py37 scipy matplotlib pandas xarray cartopy jupyter jupyterlab flake8 dask h5py netCDF4 jinja2 pytables tqdm + + Activate the new environment (you do this each time you want to use it):: + + conda activate py37 + +3. Use pip to install viresclient:: -(pip will fetch these automatically - if you are using conda, it may be better to install these first using conda instead) + pip install viresclient -There is an unresolved bug with Windows support - see here_. -.. _here: https://github.com/ESA-VirES/VirES-Python-Client/issues/1 -First usage / Configuration ---------------------------- +2. First usage / Configuration +------------------------------ + +.. note:: For Jupyter notebook users: + + The instructions for first time usage are also provided as a Jupyter notebook which you might find easier to use. Download the notebook to your environment and follow the instructions. + + https://github.com/smithara/viresclient_examples/blob/master/0_first_usage.ipynb + + To download the whole example repository, open a terminal and do:: + + git clone https://github.com/smithara/viresclient_examples.git + + then launch the notebook, ``viresclient_examples/0_first_usage.ipynb`` Access to the service is through the same user account as on the web interface (https://vires.services/) and is enabled through a token. To get a token, log in to the website and click on your name on the top right to access the settings. From here, click on "Manage access tokens" and follow the instructions to create a new token. @@ -124,8 +157,12 @@ When creating the configuration file manually make sure the file is readable by request = SwarmRequest(url="https://staging.viresdisc.vires.services/ows") -Example use ------------ +3. Example use +-------------- + +.. note:: + + A brief introduction is given here. For more possibilities, see :doc:`notebook_intro` Choose which collection to access (see :doc:`available_parameters` for more options): @@ -162,14 +199,12 @@ Set a parameter range filter to apply. You can add multiple filters in sequence request.set_range_filter("Longitude", 0, 90) -Specify the time range from which to retrieve data, make the request to the server (specifying the output file format, currently either csv or cdf): +Specify the time range from which to retrieve data, make the request to the server: .. code-block:: python data = request.get_between(start_time=dt.datetime(2016,1,1), - end_time=dt.datetime(2016,1,2), - filetype="cdf", - asynchronous=True) + end_time=dt.datetime(2016,1,2)) Transfer your data to a pandas.DataFrame_, or a xarray.Dataset_, or just save it as is: @@ -184,9 +219,12 @@ Transfer your data to a pandas.DataFrame_, or a xarray.Dataset_, or just save it data.to_file('outfile.cdf', overwrite=False) The returned data has columns for: + - ``Spacecraft, Timestamp, Latitude, Longitude, Radius`` - those specified by ``measurements`` and ``auxiliaries`` + ... and model values and residuals, named as: + - ``F_`` -- scalar field - ``B_NEC_`` -- vector field - ``F_res_`` -- scalar field residual (``F - F_``) diff --git a/docs/notebook_intro.rst b/docs/notebook_intro.rst index cf06cfd..4f90ab9 100644 --- a/docs/notebook_intro.rst +++ b/docs/notebook_intro.rst @@ -3,7 +3,24 @@ Introduction to notebooks Jupyter notebooks are a convenient tool for interactive data exploration and are used here to demonstrate usage of ``viresclient``. We encourage users to try out Jupyter for themselves and to consider sharing notebooks with others. If you have a comment about one of the notebooks, or would like a particular demonstration, you can `open a GitHub Issue `_ or get in touch otherwise. +Click `here `_ (nbviewer) to view viresclient_examples non-interactively. Sometimes notebooks won't render directly on the GitHub website - try `nbviewer `_ instead. + Here is a list of related notebook repositories where you are welcome to submit changes or additions. Please get in touch if you would like your own repository to be listed here. - https://github.com/smithara/viresclient_examples - https://github.com/pacesm/jupyter_notebooks + +Other repositories with wider scope: + + - https://github.com/smithara/IAGA_SummerSchool2019 + +To clone a repository to your working space:: + + cd ~ + git clone https://github.com/smithara/viresclient_examples.git + +To clear any changes you made and fetch the latest version:: + + cd ~/viresclient_examples/ + git fetch + git reset --hard origin/master diff --git a/docs/notebooks/simple_example.ipynb b/docs/notebooks/simple_example.ipynb index c208178..02b52d8 100644 --- a/docs/notebooks/simple_example.ipynb +++ b/docs/notebooks/simple_example.ipynb @@ -28,7 +28,7 @@ "output_type": "stream", "text": [ "[1/1] Processing: 100%|██████████| [ Elapsed: 00:01, Remaining: 00:00 ]\n", - " Downloading: 100%|██████████| [ Elapsed: 00:01, Remaining: 00:00 ] (0.766MB)\n" + " Downloading: 100%|██████████| [ Elapsed: 00:00, Remaining: 00:00 ] (0.766MB)\n" ] } ], @@ -91,7 +91,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "metadata": {}, "outputs": [ { @@ -130,23 +130,6 @@ " QDLat\n", " QDLon\n", " \n", - " \n", - " Timestamp\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", " \n", " \n", " \n", @@ -240,7 +223,6 @@ ], "text/plain": [ " Spacecraft Latitude Longitude Radius F \\\n", - "Timestamp \n", "2016-01-01 00:28:00 A 0.197610 -102.681841 6828390.60 24630.6312 \n", "2016-01-01 00:28:10 A 0.838158 -102.693946 6828288.87 24808.1073 \n", "2016-01-01 00:28:20 A 1.478724 -102.706041 6828186.32 24993.3709 \n", @@ -248,7 +230,6 @@ "2016-01-01 00:28:40 A 2.759911 -102.730169 6827978.83 25386.4755 \n", "\n", " F_MCO_SHA_2C F_MMA_SHA_2C-Primary \\\n", - "Timestamp \n", "2016-01-01 00:28:00 24795.742996 127.134287 \n", "2016-01-01 00:28:10 24971.853392 127.197315 \n", "2016-01-01 00:28:20 25155.643730 127.257710 \n", @@ -256,7 +237,6 @@ "2016-01-01 00:28:40 25545.573072 127.370566 \n", "\n", " F_MMA_SHA_2C-Secondary \\\n", - "Timestamp \n", "2016-01-01 00:28:00 33.517576 \n", "2016-01-01 00:28:10 33.718966 \n", "2016-01-01 00:28:20 33.932493 \n", @@ -264,7 +244,6 @@ "2016-01-01 00:28:40 34.394821 \n", "\n", " B_NEC \\\n", - "Timestamp \n", "2016-01-01 00:28:00 [23432.3208, 2928.6274000000003, 7000.6948] \n", "2016-01-01 00:28:10 [23451.1801, 2899.534, 7554.4035] \n", "2016-01-01 00:28:20 [23465.812100000003, 2871.8025000000002, 8109.... \n", @@ -272,7 +251,6 @@ "2016-01-01 00:28:40 [23482.5073, 2817.0387, 9225.1501] \n", "\n", " B_NEC_MCO_SHA_2C \\\n", - "Timestamp \n", "2016-01-01 00:28:00 [23604.765693069177, 2930.2688321265728, 7004.... \n", "2016-01-01 00:28:10 [23623.327008268625, 2901.392633804825, 7557.3... \n", "2016-01-01 00:28:20 [23637.6657342565, 2873.1207904780367, 8112.48... \n", @@ -280,7 +258,6 @@ "2016-01-01 00:28:40 [23653.3949273992, 2818.3663458182045, 9227.67... \n", "\n", " B_NEC_MMA_SHA_2C-Primary \\\n", - "Timestamp \n", "2016-01-01 00:28:00 [-125.78890509740265, -5.302191425378982, 17.6... \n", "2016-01-01 00:28:10 [-125.60166979171441, -5.252627485291866, 19.3... \n", "2016-01-01 00:28:20 [-125.38835978948227, -5.203014637815361, 21.1... \n", @@ -288,7 +265,6 @@ "2016-01-01 00:28:40 [-124.88363654458195, -5.103660714780139, 24.5... \n", "\n", " B_NEC_MMA_SHA_2C-Secondary \\\n", - "Timestamp \n", "2016-01-01 00:28:00 [-32.5031562033331, -1.386212193226803, -8.065... \n", "2016-01-01 00:28:10 [-32.494418615582255, -1.3566476193699097, -8.... \n", "2016-01-01 00:28:20 [-32.47777846302375, -1.3273711242288924, -9.7... \n", @@ -296,7 +272,6 @@ "2016-01-01 00:28:40 [-32.42057710492788, -1.2696744882122784, -11.... \n", "\n", " QDLat QDLon \n", - "Timestamp \n", "2016-01-01 00:28:00 8.425159 -30.812750 \n", "2016-01-01 00:28:10 9.068963 -30.902256 \n", "2016-01-01 00:28:20 9.713511 -30.990784 \n", @@ -304,7 +279,7 @@ "2016-01-01 00:28:40 11.004701 -31.165012 " ] }, - "execution_count": 4, + "execution_count": 3, "metadata": {}, "output_type": "execute_result" } @@ -318,9 +293,261 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "#### Extract numpy arrays\n", + "NB: Some of the columns contain vectors (e.g. B_NEC). Because of the way pandas works, this is a bad way of organising the data - `df[\"B_NEC\"].values` is actually an array of arrays. It is better to \"expand\" the columns above using the `expand` keyword argument:" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
SpacecraftLatitudeLongitudeRadiusFF_MCO_SHA_2CF_MMA_SHA_2C-PrimaryF_MMA_SHA_2C-SecondaryB_NEC_NB_NEC_E...B_NEC_MCO_SHA_2C_EB_NEC_MCO_SHA_2C_CB_NEC_MMA_SHA_2C-Primary_NB_NEC_MMA_SHA_2C-Primary_EB_NEC_MMA_SHA_2C-Primary_CB_NEC_MMA_SHA_2C-Secondary_NB_NEC_MMA_SHA_2C-Secondary_EB_NEC_MMA_SHA_2C-Secondary_CQDLatQDLon
2016-01-01 00:28:00A0.197610-102.6818416828390.6024630.631224795.742996127.13428733.51757623432.32082928.6274...2930.2688327004.101076-125.788905-5.30219117.668192-32.503156-1.386212-8.0654288.425159-30.812750
2016-01-01 00:28:10A0.838158-102.6939466828288.8724808.107324971.853392127.19731533.71896623451.18012899.5340...2901.3926347557.367509-125.601670-5.25262719.385239-32.494419-1.356648-8.9017379.068963-30.902256
2016-01-01 00:28:20A1.478724-102.7060416828186.3224993.370925155.643730127.25771033.93249323465.81212871.8025...2873.1207908112.480942-125.388360-5.20301521.100059-32.477778-1.327371-9.7388969.713511-30.990784
2016-01-01 00:28:30A2.119308-102.7181186828082.9725186.220725346.943757127.31546234.15787823476.35962844.1066...2845.4470788669.299689-125.149003-5.15335822.812207-32.453182-1.298381-10.57666610.358768-31.078363
2016-01-01 00:28:40A2.759911-102.7301696827978.8325386.475525545.573072127.37056634.39482123482.50732817.0387...2818.3663469227.677018-124.883637-5.10366124.521235-32.420577-1.269674-11.41480811.004701-31.165012
\n", + "

5 rows × 22 columns

\n", + "
" + ], + "text/plain": [ + " Spacecraft Latitude Longitude Radius F \\\n", + "2016-01-01 00:28:00 A 0.197610 -102.681841 6828390.60 24630.6312 \n", + "2016-01-01 00:28:10 A 0.838158 -102.693946 6828288.87 24808.1073 \n", + "2016-01-01 00:28:20 A 1.478724 -102.706041 6828186.32 24993.3709 \n", + "2016-01-01 00:28:30 A 2.119308 -102.718118 6828082.97 25186.2207 \n", + "2016-01-01 00:28:40 A 2.759911 -102.730169 6827978.83 25386.4755 \n", + "\n", + " F_MCO_SHA_2C F_MMA_SHA_2C-Primary \\\n", + "2016-01-01 00:28:00 24795.742996 127.134287 \n", + "2016-01-01 00:28:10 24971.853392 127.197315 \n", + "2016-01-01 00:28:20 25155.643730 127.257710 \n", + "2016-01-01 00:28:30 25346.943757 127.315462 \n", + "2016-01-01 00:28:40 25545.573072 127.370566 \n", + "\n", + " F_MMA_SHA_2C-Secondary B_NEC_N B_NEC_E ... \\\n", + "2016-01-01 00:28:00 33.517576 23432.3208 2928.6274 ... \n", + "2016-01-01 00:28:10 33.718966 23451.1801 2899.5340 ... \n", + "2016-01-01 00:28:20 33.932493 23465.8121 2871.8025 ... \n", + "2016-01-01 00:28:30 34.157878 23476.3596 2844.1066 ... \n", + "2016-01-01 00:28:40 34.394821 23482.5073 2817.0387 ... \n", + "\n", + " B_NEC_MCO_SHA_2C_E B_NEC_MCO_SHA_2C_C \\\n", + "2016-01-01 00:28:00 2930.268832 7004.101076 \n", + "2016-01-01 00:28:10 2901.392634 7557.367509 \n", + "2016-01-01 00:28:20 2873.120790 8112.480942 \n", + "2016-01-01 00:28:30 2845.447078 8669.299689 \n", + "2016-01-01 00:28:40 2818.366346 9227.677018 \n", + "\n", + " B_NEC_MMA_SHA_2C-Primary_N B_NEC_MMA_SHA_2C-Primary_E \\\n", + "2016-01-01 00:28:00 -125.788905 -5.302191 \n", + "2016-01-01 00:28:10 -125.601670 -5.252627 \n", + "2016-01-01 00:28:20 -125.388360 -5.203015 \n", + "2016-01-01 00:28:30 -125.149003 -5.153358 \n", + "2016-01-01 00:28:40 -124.883637 -5.103661 \n", + "\n", + " B_NEC_MMA_SHA_2C-Primary_C B_NEC_MMA_SHA_2C-Secondary_N \\\n", + "2016-01-01 00:28:00 17.668192 -32.503156 \n", + "2016-01-01 00:28:10 19.385239 -32.494419 \n", + "2016-01-01 00:28:20 21.100059 -32.477778 \n", + "2016-01-01 00:28:30 22.812207 -32.453182 \n", + "2016-01-01 00:28:40 24.521235 -32.420577 \n", + "\n", + " B_NEC_MMA_SHA_2C-Secondary_E \\\n", + "2016-01-01 00:28:00 -1.386212 \n", + "2016-01-01 00:28:10 -1.356648 \n", + "2016-01-01 00:28:20 -1.327371 \n", + "2016-01-01 00:28:30 -1.298381 \n", + "2016-01-01 00:28:40 -1.269674 \n", + "\n", + " B_NEC_MMA_SHA_2C-Secondary_C QDLat QDLon \n", + "2016-01-01 00:28:00 -8.065428 8.425159 -30.812750 \n", + "2016-01-01 00:28:10 -8.901737 9.068963 -30.902256 \n", + "2016-01-01 00:28:20 -9.738896 9.713511 -30.990784 \n", + "2016-01-01 00:28:30 -10.576666 10.358768 -31.078363 \n", + "2016-01-01 00:28:40 -11.414808 11.004701 -31.165012 \n", + "\n", + "[5 rows x 22 columns]" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df = data.as_dataframe(expand=True)\n", + "df.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Extract numpy arrays\n", "\n", - "Individual entries (rows) in multi-dimensional variables (e.g. B_NEC) are currently stored as lists so they need to be as extracted as arrays to do array operations. This behaviour will likely change in the future to provide \"flat\" dataframes instead (i.e. the B_NEC column expanded as three columns: B_N, B_E, B_C, etc)" + "numpy arrays are accessible as properties of the dataframe:" ] }, { @@ -331,12 +558,8 @@ { "data": { "text/plain": [ - "(array([23432.3208, 23451.1801, 23465.8121, ..., 19454.4269, 19271.3803,\n", - " 19084.199 ]),\n", - " array([2928.6274, 2899.534 , 2871.8025, ..., 824.8021, 808.9134,\n", - " 792.9848]),\n", - " array([ 7000.6948, 7554.4035, 8109.7535, ..., 32624.9088, 33094.2485,\n", - " 33559.0953]))" + "array([23432.3208, 23451.1801, 23465.8121, ..., 19454.4269, 19271.3803,\n", + " 19084.199 ])" ] }, "execution_count": 5, @@ -345,11 +568,33 @@ } ], "source": [ - "import numpy as np\n", - "\n", - "B_NEC = np.stack(df['B_NEC'].values)\n", - "X,Y,Z = (B_NEC[:,i] for i in (0,1,2))\n", - "X,Y,Z" + "df['B_NEC_N'].values" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([[23432.3208, 2928.6274, 7000.6948],\n", + " [23451.1801, 2899.534 , 7554.4035],\n", + " [23465.8121, 2871.8025, 8109.7535],\n", + " ...,\n", + " [19454.4269, 824.8021, 32624.9088],\n", + " [19271.3803, 808.9134, 33094.2485],\n", + " [19084.199 , 792.9848, 33559.0953]])" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df[['B_NEC_N', 'B_NEC_E', 'B_NEC_C']].values" ] }, { @@ -370,7 +615,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, "metadata": {}, "outputs": [ { @@ -395,10 +640,14 @@ " B_NEC_MMA_SHA_2C-Primary (Timestamp, dim) float64 -125.8 -5.302 ... 28.31\n", " B_NEC_MMA_SHA_2C-Secondary (Timestamp, dim) float64 -32.5 -1.386 ... -4.19\n", " QDLat (Timestamp) float64 8.425 9.069 ... 40.18 40.81\n", - " QDLon (Timestamp) float64 -30.81 -30.9 ... -24.81" + " QDLon (Timestamp) float64 -30.81 -30.9 ... -24.81\n", + "Attributes:\n", + " Sources: ['SW_OPER_MAGA_LR_1B_20160101T000000_20160101T235959_050...\n", + " MagneticModels: ['MCO_SHA_2C = MCO_SHA_2C(max_degree=18,min_degree=1)', ...\n", + " RangeFilters: ['Latitude:0,90']" ] }, - "execution_count": 6, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } @@ -412,12 +661,75 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Extracting numpy arrays from xarray:" + "`ds` now contains an xarray `Dataset` which stores all the data variables with an associated \"coordinate\" of Timestamp. The dataset itself comprises `DataArray` objects:" ] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "\n", + "array([[23432.3208, 2928.6274, 7000.6948],\n", + " [23451.1801, 2899.534 , 7554.4035],\n", + " [23465.8121, 2871.8025, 8109.7535],\n", + " ...,\n", + " [19454.4269, 824.8021, 32624.9088],\n", + " [19271.3803, 808.9134, 33094.2485],\n", + " [19084.199 , 792.9848, 33559.0953]])\n", + "Coordinates:\n", + " * Timestamp (Timestamp) datetime64[ns] 2016-01-01T00:28:00 ... 2016-01-01T23:59:50\n", + "Dimensions without coordinates: dim" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ds[\"B_NEC\"]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### To extract numpy arrays:" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([[23432.3208, 2928.6274, 7000.6948],\n", + " [23451.1801, 2899.534 , 7554.4035],\n", + " [23465.8121, 2871.8025, 8109.7535],\n", + " ...,\n", + " [19454.4269, 824.8021, 32624.9088],\n", + " [19271.3803, 808.9134, 33094.2485],\n", + " [19084.199 , 792.9848, 33559.0953]])" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ds[\"B_NEC\"].values" + ] + }, + { + "cell_type": "code", + "execution_count": 10, "metadata": {}, "outputs": [ { @@ -431,7 +743,7 @@ " 33559.0953]))" ] }, - "execution_count": 7, + "execution_count": 10, "metadata": {}, "output_type": "execute_result" } @@ -447,12 +759,12 @@ "source": [ "### Work on xarray objects directly\n", "\n", - "Calculate the custom residual $B_{res} = B_{obs} - B_{MCO} - B_{MMA}$ and plot the Z component against time" + "Calculate the custom residual $B_{res} = B_{obs} - B_{MCO} - B_{MMA}$ and plot the Z component against time. NB: It is possible to also calculate this residual on the server directly." ] }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 11, "metadata": {}, "outputs": [ { @@ -471,7 +783,7 @@ "Dimensions without coordinates: dim" ] }, - "execution_count": 8, + "execution_count": 11, "metadata": {}, "output_type": "execute_result" } @@ -494,13 +806,6 @@ "%matplotlib inline\n", "B_res[:,2].plot(x='Timestamp');" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { @@ -519,9 +824,9 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.3" + "version": "3.7.4" } }, "nbformat": 4, - "nbformat_minor": 2 + "nbformat_minor": 4 } diff --git a/docs/readme.rst b/docs/readme.rst index 66611c1..48c1813 100644 --- a/docs/readme.rst +++ b/docs/readme.rst @@ -19,6 +19,12 @@ Some links where you can read more about VirES: .. _`EOX blog posts`: https://eox.at/category/vires/ .. _`Swarm mission`: https://earth.esa.int/web/guest/missions/esa-operational-eo-missions/swarm +``viresclient`` is installed on the "Virtual Research Environment" (VRE), which is a managed Jupyter-based system provided by ESA: + + - `VRE documentation`_ + + .. _`VRE documentation`: https://swarm-vre.readthedocs.io/ + Data can be accessed from the server as CSV or CDF files and saved to disk, or loaded directly into Python objects pandas.DataFrame_, or xarray.Dataset_. .. _pandas.DataFrame: https://pandas.pydata.org/pandas-docs/stable/dsintro.html#dataframe @@ -31,7 +37,7 @@ cdflib_ is used to read CDF files. The project is on GitHub at https://github.com/ESA-VirES/VirES-Python-Client - please feel free to contribute with any code/suggestions/comments. -A repository of example notebooks can be found at https://github.com/smithara/viresclient_examples. We welcome contribution of notebooks to this repository that show some short analyses or generating useful figures. +A repository of example notebooks can be found at https://github.com/smithara/viresclient_examples. We welcome contribution of notebooks to this repository that show some short analyses or generating useful figures. It can be hard to remember or figure how do things (e.g. remembering Swarm product names; understanding the VirES "language" and what is possible; making certain kinds of figures with various libraries) so these notebooks are meant to act as recipes we can follow, to demonstrate code that can be copied and adapted for other purposes. How to acknowledge VirES ------------------------ diff --git a/docs/release_notes.rst b/docs/release_notes.rst index 2b42a21..0c89b76 100644 --- a/docs/release_notes.rst +++ b/docs/release_notes.rst @@ -1,6 +1,42 @@ Release notes ============= +Changes from 0.3.0 to 0.4.0 +--------------------------- + +- Fixed issues with running on Windows + +- Enforcing Python v3.5+ for installation + +- Allowing higher versions of cdflib, pandas, and xarray + +- Added CLI configuration for setting server address and token + +- Metadata for source lineage is now easier to access (names of original ESA data files, details of models used, and filters applied). These are set as properties of :meth:`viresclient.ReturnedData` (i.e. ``data``) and as metadata (``.attrs``) in the ``Dataset`` returned from ``.as_xarray()``:: + + data.sources + data.magnetic_models + data.range_filters + + ds = data.as_xarray() + ds.Sources + ds.MagneticModels + ds.RangeFilters + +- Added access to collections ``SW_OPER_IPDxIRR_2F`` + +- Added auxiliary data ``F107`` which is the hourly F10.7 value. This is in addition to ``F10_INDEX`` which was already present, which is a daily average. + +- Added possibility of accessing multiple collections simultaneously, e.g.:: + + request.set_collection("SW_OPER_MAGA_LR_1B", "SW_OPER_MAGC_LR_1B") + +- Added optional "expansion" of dataframes with:: + + data.as_dataframe(expand=True) + + This expands columns which contain vectors (e.g. ``B_NEC``) into separate columns named like: ``B_NEC_N``, ``B_NEC_E``, ``B_NEC_C``. This is recommended so that numpy operations will work on the columns. The default is ``expand=False`` to preserve the older behaviour. + Changes from v0.2.6 to 0.3.0 ---------------------------- diff --git a/viresclient/_client.py b/viresclient/_client.py index 4552cfa..7040b49 100644 --- a/viresclient/_client.py +++ b/viresclient/_client.py @@ -407,8 +407,8 @@ def _get(self, request=None, asynchronous=None, response_handler=None, ) except WPSError: raise RuntimeError( - "Server error. Perhaps you are requesting a period outside of" - "product availability? or the request is otherwise invalid." + "Server error. Perhaps you are requesting a period outside of " + "product availability? or the request is otherwise invalid. " "Check the output of: print(request) and " "print(request._request.decode())" ) diff --git a/viresclient/_client_swarm.py b/viresclient/_client_swarm.py index 6a7b8b3..4eba649 100644 --- a/viresclient/_client_swarm.py +++ b/viresclient/_client_swarm.py @@ -560,7 +560,7 @@ def set_collection(self, *args): if collection not in self._available["collections"]: raise ValueError( "Invalid collection: {}. " - "Check available with SwarmRequest.available_collections()" + "Check available with SwarmRequest().available_collections()" .format(collection) ) self._collection_list = collections diff --git a/viresclient/_data_handling.py b/viresclient/_data_handling.py index a4b8ea4..71f583e 100644 --- a/viresclient/_data_handling.py +++ b/viresclient/_data_handling.py @@ -486,9 +486,11 @@ def contents(self, value): def as_dataframe(self, expand=False): """Convert the data to a pandas DataFrame. - If expand is True, expand some columns: - e.g. B_NEC -> B_NEC_N, B_NEC_E, B_NEC_C - B_VFM -> B_VFM_i, B_VFM_j, B_VFM_k + If expand is True, expand some columns, e.g.: + + B_NEC -> B_NEC_N, B_NEC_E, B_NEC_C + + B_VFM -> B_VFM_i, B_VFM_j, B_VFM_k Args: expand (bool)