From 510e25a2e010c89d85ad91414287ebfb93014a69 Mon Sep 17 00:00:00 2001 From: Ashley Smith Date: Thu, 8 Feb 2024 16:41:54 +0000 Subject: [PATCH] Maintenance rework (#35) Major change to the pre-commit fixes & checks to align with the current state of https://learn.scientific-python.org/development/guides/style/ The main thing: Switching to ruff, motivated by problems updating the pre-commit tools (black made changes that flake8 then complained about). Using ruff replaces using black and flake8 directly. It has a long list of rules that can be configured. A lot of style changes were made automatically, and I've temporarily(?) disabled rules that caused a problem needing manual changes. Some minor fixes to CI, including switching from the deprecated pre-commit action to the pre-commit.ci application --- .flake8 | 7 - .github/workflows/ci.yml | 19 +-- .gitignore | 6 +- .pre-commit-config.yaml | 160 +++++++++--------- .readthedocs.yml | 14 +- README.md | 4 +- docs/guides/tfa/intro_tfa.ipynb | 66 ++++---- docs/guides/tfa/tfa_AUX_OBS.ipynb | 22 +-- docs/guides/tfa/tfa_Cluster.ipynb | 20 +-- docs/guides/tfa/tfa_EFIx_TCT.ipynb | 20 +-- docs/guides/tfa/tfa_MAGx_HR.ipynb | 26 +-- docs/guides/tfa/tfa_MAGx_LR.ipynb | 36 ++-- docs/guides/tfa/tfa_background.ipynb | 38 ++--- docs/installation.md | 13 +- pyproject.toml | 59 ++++++- src/swarmpal/__init__.py | 4 +- src/swarmpal/io/__init__.py | 2 + src/swarmpal/io/_datafetchers.py | 2 +- src/swarmpal/toolboxes/dsecs/__init__.py | 2 + src/swarmpal/toolboxes/dsecs/aux_tools.py | 4 +- .../toolboxes/dsecs/dsecs_algorithms.py | 11 +- src/swarmpal/toolboxes/dsecs/processes.py | 2 + src/swarmpal/toolboxes/fac/__init__.py | 2 + src/swarmpal/toolboxes/fac/processes.py | 2 + src/swarmpal/toolboxes/tfa/__init__.py | 2 + src/swarmpal/toolboxes/tfa/plotting.py | 2 +- src/swarmpal/toolboxes/tfa/processes.py | 4 +- src/swarmpal/toolboxes/tfa/tfalib.py | 22 +-- src/swarmpal/utils/exceptions.py | 3 + tests/io/test_datafetchers.py | 6 +- tests/io/test_paldata.py | 24 +-- tests/io/test_palprocess.py | 8 +- tests/test_package.py | 2 +- 33 files changed, 337 insertions(+), 277 deletions(-) delete mode 100644 .flake8 diff --git a/.flake8 b/.flake8 deleted file mode 100644 index 47559c0..0000000 --- a/.flake8 +++ /dev/null @@ -1,7 +0,0 @@ -[flake8] -extend-ignore = E203, E501, E722, B950, W503, B907 -select = C,E,F,W,T,B,I -per-file-ignores = - tests/*: T - noxfile.py: T - __init__.py:F401 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 237c478..ae73618 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,25 +4,18 @@ on: workflow_dispatch: pull_request: branches: [ main, staging ] + schedule: + # Weekly on Sunday + - cron: "0 1 * * 0" jobs: - pre-commit: - name: Format - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v1 - - uses: actions/setup-python@v2 - - uses: pre-commit/action@v3.0.0 - with: - extra_args: --hook-stage manual --all-files - checks: name: Check Python ${{ matrix.python-version }} on ${{ matrix.runs-on }} runs-on: ${{ matrix.runs-on }} strategy: fail-fast: false matrix: - python-version: ["3.9", "3.10"] + python-version: ["3.9", "3.10", "3.11"] runs-on: [ubuntu-latest, macos-latest, windows-latest] # include: @@ -36,9 +29,9 @@ jobs: run: brew reinstall gfortran shell: bash - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} diff --git a/.gitignore b/.gitignore index 8369250..1f4cb4e 100644 --- a/.gitignore +++ b/.gitignore @@ -141,4 +141,8 @@ cython_debug/ src/*/_version.py # VSCode -.vscode/* +.vscode + +# Other dev tool junk +.ruff_cache +node_modules diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3884a6c..e2b7ad2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,97 +1,91 @@ exclude: ^(.*/data/.*) repos: -- repo: https://github.com/psf/black - rev: 23.1.0 - hooks: - - id: black-jupyter + # Strip outputs from notebooks + - repo: https://github.com/kynan/nbstripout + rev: "0.7.1" + hooks: + - id: nbstripout + args: + ["--extra-keys", "metadata.kernelspec metadata.language_info.version"] -- repo: https://github.com/kynan/nbstripout - rev: "0.6.1" - hooks: - - id: nbstripout - args: ["--extra-keys", "metadata.kernelspec metadata.language_info.version"] + # The rest of the checks are from: + # https://github.com/scientific-python/cookie/blob/main/%7B%7Bcookiecutter.project_name%7D%7D/.pre-commit-config.yaml -- repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 - hooks: - - id: check-added-large-files - - id: check-case-conflict - - id: check-merge-conflict - - id: check-symlinks - - id: check-yaml - - id: debug-statements - - id: end-of-file-fixer - - id: mixed-line-ending - - id: requirements-txt-fixer - - id: trailing-whitespace + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: "v4.5.0" + hooks: + - id: check-added-large-files + - id: check-case-conflict + - id: check-merge-conflict + - id: check-symlinks + - id: check-yaml + - id: debug-statements + - id: end-of-file-fixer + - id: mixed-line-ending + - id: name-tests-test + args: ["--pytest-test-first"] + - id: requirements-txt-fixer + - id: trailing-whitespace -- repo: https://github.com/pre-commit/pygrep-hooks - rev: v1.10.0 - hooks: - - id: python-check-blanket-noqa - # - id: python-check-blanket-type-ignore - - id: python-no-eval - - id: python-use-type-annotations - - id: rst-backticks - - id: rst-directive-colons - - id: rst-inline-touching-normal + - repo: https://github.com/pre-commit/pygrep-hooks + rev: "v1.10.0" + hooks: + - id: rst-backticks + - id: rst-directive-colons + - id: rst-inline-touching-normal -- repo: https://github.com/PyCQA/isort - rev: 5.12.0 - hooks: - - id: isort - # args: ["-a", "from __future__ import annotations"] # Python 3.7+ + # - repo: https://github.com/pre-commit/mirrors-prettier + # rev: "v4.0.0-alpha.8" + # hooks: + # - id: prettier + # types_or: [yaml, markdown, html, css, scss, javascript, json] + # args: [--prose-wrap=always] -- repo: https://github.com/asottile/pyupgrade - rev: v3.3.1 - hooks: - - id: pyupgrade - args: ["--py37-plus"] + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: "v0.2.1" + hooks: + - id: ruff + args: ["--fix", "--show-fixes"] + - id: ruff-format -# - repo: https://github.com/hadialqattan/pycln -# rev: v1.1.0 -# hooks: -# - id: pycln -# args: [--config=pyproject.toml] + # TODO: Enable type checking + # - repo: https://github.com/pre-commit/mirrors-mypy + # rev: "v1.8.0" + # hooks: + # - id: mypy + # files: src|tests + # args: [] + # additional_dependencies: + # - pytest -- repo: https://github.com/asottile/yesqa - rev: v1.4.0 - hooks: - - id: yesqa - exclude: docs/conf.py - additional_dependencies: &flake8_dependencies - - flake8-bugbear - - flake8-print + - repo: https://github.com/codespell-project/codespell + rev: "v2.2.6" + hooks: + - id: codespell -- repo: https://github.com/pycqa/flake8 - rev: 6.0.0 - hooks: - - id: flake8 - exclude: docs/conf.py - additional_dependencies: *flake8_dependencies + - repo: https://github.com/shellcheck-py/shellcheck-py + rev: "v0.9.0.6" + hooks: + - id: shellcheck -# - repo: https://github.com/pre-commit/mirrors-mypy -# rev: v0.930 -# hooks: -# - id: mypy -# files: src -# args: [--show-error-codes] + - repo: local + hooks: + - id: disallow-caps + name: Disallow improper capitalization + language: pygrep + entry: PyBind|Numpy|Cmake|CCache|Github|PyTest + exclude: .pre-commit-config.yaml -- repo: https://github.com/codespell-project/codespell - rev: v2.2.2 - hooks: - - id: codespell + - repo: https://github.com/abravalheri/validate-pyproject + rev: "v0.16" + hooks: + - id: validate-pyproject + additional_dependencies: ["validate-pyproject-schema-store[all]"] -- repo: https://github.com/shellcheck-py/shellcheck-py - rev: v0.9.0.2 - hooks: - - id: shellcheck - -- repo: local - hooks: - - id: disallow-caps - name: Disallow improper capitalization - language: pygrep - entry: PyBind|Numpy|Cmake|CCache|Github|PyTest - exclude: .pre-commit-config.yaml + - repo: https://github.com/python-jsonschema/check-jsonschema + rev: "0.28.0" + hooks: + - id: check-dependabot + - id: check-github-workflows + - id: check-readthedocs diff --git a/.readthedocs.yml b/.readthedocs.yml index 4a13fbf..dbab55d 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -13,14 +13,14 @@ sphinx: formats: [] build: - os: ubuntu-20.04 + os: ubuntu-22.04 tools: - python: "3.8" + python: "3.11" python: install: - - method: pip - path: . - extra_requirements: - - docs - - dsecs_cp38_manylinux + - method: pip + path: . + extra_requirements: + - docs + - dsecs_cp311_manylinux diff --git a/README.md b/README.md index 072a352..757f009 100644 --- a/README.md +++ b/README.md @@ -9,8 +9,8 @@ [![GitHub Discussion][github-discussions-badge]][github-discussions-link] [![Actions Status][actions-badge]][actions-link] -[![Code style: black][black-badge]][black-link] [![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit&logoColor=white)](https://github.com/pre-commit/pre-commit) +[![Code style: Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/format.json)](https://github.com/astral-sh/ruff) --- @@ -30,7 +30,7 @@ [rtd-badge]: https://img.shields.io/badge/docs-online-success [rtd-link]: https://swarmpal.readthedocs.io/ -![SwarmPAL diagram](https://swarmpal-benchmarks.netlify.app/swarmpal-diagram.png) +![SwarmPAL diagram](https://swarmdisc.org/wp-content/uploads/2024/02/SwarmPAL-diagrams-overview.png) For more information see: - diff --git a/docs/guides/tfa/intro_tfa.ipynb b/docs/guides/tfa/intro_tfa.ipynb index b7d64bb..78514b5 100644 --- a/docs/guides/tfa/intro_tfa.ipynb +++ b/docs/guides/tfa/intro_tfa.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "5c6e9800-e9d0-498b-a4d5-e2fba6e44db7", + "id": "0", "metadata": {}, "source": [ "# TFA: Time-Frequency Analysis" @@ -10,7 +10,7 @@ }, { "cell_type": "markdown", - "id": "733b5e49-b6c5-4cff-b18a-08dfa6b42953", + "id": "1", "metadata": {}, "source": [ "An introduction to the TFA toolbox. This gives a basic demonstration of a wavelet analysis applied to the MAGx_LR data (magnetic field 1Hz). For more details, see the following pages/notebooks." @@ -19,7 +19,7 @@ { "cell_type": "code", "execution_count": null, - "id": "10d200f5-fc1b-4300-940d-91d8a5bb3347", + "id": "2", "metadata": {}, "outputs": [], "source": [ @@ -31,7 +31,7 @@ }, { "cell_type": "markdown", - "id": "976d0be4-a283-4988-8e8f-ded91bdef848", + "id": "3", "metadata": {}, "source": [ "## Fetching data" @@ -39,7 +39,7 @@ }, { "cell_type": "markdown", - "id": "8aab9b74-4b36-4f94-8861-b58f59a995ab", + "id": "4", "metadata": {}, "source": [ "`create_paldata` and `PalDataItem.from_vires` are used together to pull the desired data from VirES.\n", @@ -56,7 +56,7 @@ { "cell_type": "code", "execution_count": null, - "id": "6108230f-0e02-4cc3-bb43-aab1e78a65d9", + "id": "5", "metadata": {}, "outputs": [], "source": [ @@ -78,7 +78,7 @@ }, { "cell_type": "markdown", - "id": "31ee1889-ffac-445f-acd9-9da89f195110", + "id": "6", "metadata": {}, "source": [ "## Applying processes\n", @@ -88,7 +88,7 @@ }, { "cell_type": "markdown", - "id": "5d328bd6-4d50-4487-a9ef-b9ddb115c27c", + "id": "7", "metadata": {}, "source": [ "### TFA: Preprocess\n", @@ -103,7 +103,7 @@ { "cell_type": "code", "execution_count": null, - "id": "fd772304-faaa-4b4f-9aef-36a70480ede0", + "id": "8", "metadata": {}, "outputs": [], "source": [ @@ -129,7 +129,7 @@ }, { "cell_type": "markdown", - "id": "3dfd918f-658d-48bd-b789-c1f6621376bd", + "id": "9", "metadata": {}, "source": [ "Now we have a process `p1`, we can apply that to the data:" @@ -138,7 +138,7 @@ { "cell_type": "code", "execution_count": null, - "id": "01cf9e9c-1033-468a-b70d-c11f4f95bf5f", + "id": "10", "metadata": {}, "outputs": [], "source": [ @@ -147,7 +147,7 @@ }, { "cell_type": "markdown", - "id": "8367c494-f158-4fb7-968d-f236beeaeee7", + "id": "11", "metadata": {}, "source": [ "The preprocessing plays a number of roles but most importantly it sets a new array, `TFA_Variable`, in the data, which will be used in the following steps." @@ -155,7 +155,7 @@ }, { "cell_type": "markdown", - "id": "6b74173e-d3ad-4ba6-9247-5db6cee85f5b", + "id": "12", "metadata": {}, "source": [ "This data can be inspected directly using tools available through xarray:" @@ -164,7 +164,7 @@ { "cell_type": "code", "execution_count": null, - "id": "d6d11b66-cb1d-48bb-ad74-cb0db5f60211", + "id": "13", "metadata": {}, "outputs": [], "source": [ @@ -174,7 +174,7 @@ { "cell_type": "code", "execution_count": null, - "id": "ac4479e0-760e-4f43-afa6-2b4e3870f2cb", + "id": "14", "metadata": {}, "outputs": [], "source": [ @@ -183,7 +183,7 @@ }, { "cell_type": "markdown", - "id": "a92ad613-db84-4cb3-b703-3e044e42983d", + "id": "15", "metadata": {}, "source": [ "The TFA toolbox also provides a function to inspect the data, `tfa.plotting.time_series`:" @@ -192,7 +192,7 @@ { "cell_type": "code", "execution_count": null, - "id": "910195b5-7f28-4521-a03f-af657930cb7b", + "id": "16", "metadata": {}, "outputs": [], "source": [ @@ -201,7 +201,7 @@ }, { "cell_type": "markdown", - "id": "aa6d3c85-a1b6-4edf-ba24-f2f7d34e192e", + "id": "17", "metadata": {}, "source": [ "We can see an unphysical spike in the data. In this case we could have removed this bad data in the data request step (by filtering according to flag values), but here we will demonstrate the functionality of the TFA toolbox in removing this spike." @@ -209,7 +209,7 @@ }, { "cell_type": "markdown", - "id": "090c9713-b744-45de-8183-23dbd8c6acae", + "id": "18", "metadata": {}, "source": [ "### TFA: Clean\n", @@ -220,7 +220,7 @@ { "cell_type": "code", "execution_count": null, - "id": "65f1ac2e-43f0-4440-9577-b23c21f61325", + "id": "19", "metadata": {}, "outputs": [], "source": [ @@ -236,7 +236,7 @@ }, { "cell_type": "markdown", - "id": "da183cc5-22f8-4246-8678-0870964cfea3", + "id": "20", "metadata": {}, "source": [ "### TFA: Filter\n", @@ -247,7 +247,7 @@ { "cell_type": "code", "execution_count": null, - "id": "cf2e3bd5-204a-4f41-abb2-795ac3c7e9ce", + "id": "21", "metadata": {}, "outputs": [], "source": [ @@ -261,7 +261,7 @@ }, { "cell_type": "markdown", - "id": "61e91fd2-f3da-4df8-adf6-c70c2840f6b3", + "id": "22", "metadata": {}, "source": [ "### TFA: Wavelet\n", @@ -272,7 +272,7 @@ { "cell_type": "code", "execution_count": null, - "id": "9c4b855e-2a45-441c-8274-46c564f4978f", + "id": "23", "metadata": {}, "outputs": [], "source": [ @@ -287,7 +287,7 @@ }, { "cell_type": "markdown", - "id": "44f7f78a-65a5-4892-a722-b3f5e2bcffad", + "id": "24", "metadata": {}, "source": [ "The results of the wavelet analysis are stored as a spectrum in a new `wavelet_power` array:" @@ -296,7 +296,7 @@ { "cell_type": "code", "execution_count": null, - "id": "17999e51-fdbb-4dd1-9061-43bba6fec799", + "id": "25", "metadata": {}, "outputs": [], "source": [ @@ -305,7 +305,7 @@ }, { "cell_type": "markdown", - "id": "480d2b7f-42db-4645-b313-267e0776053c", + "id": "26", "metadata": {}, "source": [ "And we can view it with `tfa.plotting.spectrum`:" @@ -314,7 +314,7 @@ { "cell_type": "code", "execution_count": null, - "id": "52cd01ed-4afa-47d6-82aa-6f0aeae8d626", + "id": "27", "metadata": {}, "outputs": [], "source": [ @@ -323,7 +323,7 @@ }, { "cell_type": "markdown", - "id": "20be4ca4-5e09-447d-80a1-9d275d3ee0e8", + "id": "28", "metadata": {}, "source": [ "## Plotting" @@ -331,7 +331,7 @@ }, { "cell_type": "markdown", - "id": "65078a3c-ef7c-4c9d-b2f9-3b3a4011cf45", + "id": "29", "metadata": {}, "source": [ "We showed basic usage of `tfa.plotting.time_series` and `tfa.plotting.spectrum` above. These have some extra options to control their behaviour. For example:\n", @@ -343,7 +343,7 @@ { "cell_type": "code", "execution_count": null, - "id": "f0628630-1e81-4066-8a62-18cfe171dbae", + "id": "30", "metadata": {}, "outputs": [], "source": [ @@ -356,7 +356,7 @@ }, { "cell_type": "markdown", - "id": "8ededed6-a2e1-45c9-8b97-ec58cfd73724", + "id": "31", "metadata": {}, "source": [ "There is also a quicklook plot that combines the time series, spectrum, and wave power (from `tfa.plotting.wave_index`) into one figure:" @@ -365,7 +365,7 @@ { "cell_type": "code", "execution_count": null, - "id": "ab2925a8-4322-4231-a935-626e0b9991a4", + "id": "32", "metadata": {}, "outputs": [], "source": [ diff --git a/docs/guides/tfa/tfa_AUX_OBS.ipynb b/docs/guides/tfa/tfa_AUX_OBS.ipynb index 63d9e2e..30156e4 100644 --- a/docs/guides/tfa/tfa_AUX_OBS.ipynb +++ b/docs/guides/tfa/tfa_AUX_OBS.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "783bd806-a4e6-434f-9494-cdd98cce9b47", + "id": "0", "metadata": {}, "source": [ "# Analysis of Ground Observatory data" @@ -10,7 +10,7 @@ }, { "cell_type": "markdown", - "id": "2c621f95-3850-4250-afb7-c54b0f4dfe63", + "id": "1", "metadata": {}, "source": [ "VirES also has ground observatory data from INTERMAGNET (via the \"AUX_OBS\" collections). We can apply the same TFA tools to this data.\n", @@ -21,7 +21,7 @@ { "cell_type": "code", "execution_count": null, - "id": "9adecf37-9d3d-4219-84cb-5b7110ec9fcc", + "id": "2", "metadata": {}, "outputs": [], "source": [ @@ -33,7 +33,7 @@ }, { "cell_type": "markdown", - "id": "c4492e83-0c1c-47a4-b92d-13c174d057e5", + "id": "3", "metadata": {}, "source": [ "## Fetching data" @@ -41,7 +41,7 @@ }, { "cell_type": "markdown", - "id": "e975d53f-3576-4249-baed-c76d23040351", + "id": "4", "metadata": {}, "source": [ "Here we access the 1-minute data (the \"M\" in `OBSM2_`) from Hornsund (`HRN`). The three-letter code is an INTERMAGNET observatory code (see ).\n", @@ -52,7 +52,7 @@ { "cell_type": "code", "execution_count": null, - "id": "1cf70e30-1fcb-4fd8-930b-c5ee16fa80d8", + "id": "5", "metadata": {}, "outputs": [], "source": [ @@ -72,7 +72,7 @@ }, { "cell_type": "markdown", - "id": "56aa6b9d-8ece-4aa4-8738-a62ffe9eb478", + "id": "6", "metadata": {}, "source": [ "## Processing" @@ -81,7 +81,7 @@ { "cell_type": "code", "execution_count": null, - "id": "70394c05-7dcb-4bc7-933c-7256a7a7b10a", + "id": "7", "metadata": {}, "outputs": [], "source": [ @@ -118,7 +118,7 @@ }, { "cell_type": "markdown", - "id": "528eef47-568e-4ab5-88fd-88d798b0bd38", + "id": "8", "metadata": {}, "source": [ "## Plotting" @@ -126,7 +126,7 @@ }, { "cell_type": "markdown", - "id": "14e758c8-0280-4b9c-8154-94f8e3dc7a16", + "id": "9", "metadata": {}, "source": [ "Note that since the ground station data are provided in a 1-minute sampling time, the frequency range to be studied must be very low, i.e. Pc5 (2 - 7 mHz). Frequencies higher than 8 mHz cannot really be captured with these data, since the Nyquist frequency for a sampling time dt of 60 seconds is 1/(2*60) = 8.33 mHz!" @@ -135,7 +135,7 @@ { "cell_type": "code", "execution_count": null, - "id": "b90715b6-eac3-4712-b61d-71b8cbe834b6", + "id": "10", "metadata": {}, "outputs": [], "source": [ diff --git a/docs/guides/tfa/tfa_Cluster.ipynb b/docs/guides/tfa/tfa_Cluster.ipynb index 257cf34..3560f43 100644 --- a/docs/guides/tfa/tfa_Cluster.ipynb +++ b/docs/guides/tfa/tfa_Cluster.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "d5b95cd8-6f5e-4332-b70a-6716d13b7c9f", + "id": "0", "metadata": {}, "source": [ "# Analysis of Cluster data" @@ -10,7 +10,7 @@ }, { "cell_type": "markdown", - "id": "f041987c-0abc-4ce0-94b2-5986254bd3ad", + "id": "1", "metadata": {}, "source": [ "In this example we will access Cluster data via a HAPI server (not from VirES)\n", @@ -21,7 +21,7 @@ { "cell_type": "code", "execution_count": null, - "id": "350edcfb-60c8-4bff-9685-5846774ea760", + "id": "2", "metadata": {}, "outputs": [], "source": [ @@ -33,7 +33,7 @@ }, { "cell_type": "markdown", - "id": "200bc103-f5bb-4d1e-8cdc-0a6b598dae14", + "id": "3", "metadata": {}, "source": [ "## Fetching data" @@ -41,7 +41,7 @@ }, { "cell_type": "markdown", - "id": "01b00f2d-092c-45c9-8183-25fff1de2993", + "id": "4", "metadata": {}, "source": [ "We can access HAPI data in an almost identical way as from VirES, instead using `PalDataItem.from_hapi`.\n", @@ -56,7 +56,7 @@ { "cell_type": "code", "execution_count": null, - "id": "0384448e-da8c-4b41-8ed1-dd43b611c2b9", + "id": "5", "metadata": {}, "outputs": [], "source": [ @@ -74,7 +74,7 @@ }, { "cell_type": "markdown", - "id": "28fbf535-d448-46cd-bde3-e4e4467cef1d", + "id": "6", "metadata": {}, "source": [ "## Processing" @@ -83,7 +83,7 @@ { "cell_type": "code", "execution_count": null, - "id": "0174f15b-742f-4510-9630-b602a67131cf", + "id": "7", "metadata": {}, "outputs": [], "source": [ @@ -120,7 +120,7 @@ }, { "cell_type": "markdown", - "id": "871ad53a-f00b-466f-aa84-ee3ed320f11a", + "id": "8", "metadata": {}, "source": [ "## Plotting" @@ -129,7 +129,7 @@ { "cell_type": "code", "execution_count": null, - "id": "70598c47-4a29-4c39-b0aa-e6ce34188956", + "id": "9", "metadata": {}, "outputs": [], "source": [ diff --git a/docs/guides/tfa/tfa_EFIx_TCT.ipynb b/docs/guides/tfa/tfa_EFIx_TCT.ipynb index 8f883e8..b426569 100644 --- a/docs/guides/tfa/tfa_EFIx_TCT.ipynb +++ b/docs/guides/tfa/tfa_EFIx_TCT.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "416d0a2f-2891-42b4-945e-4c2757f35ae2", + "id": "0", "metadata": {}, "source": [ "# Analysis of Swarm Electric Field data" @@ -10,7 +10,7 @@ }, { "cell_type": "markdown", - "id": "ac6252e0-5d02-48c7-9f83-f16563cca3c9", + "id": "1", "metadata": {}, "source": [ "In this example we will use the `SW_EXPT_EFIx_TCT02` dataset which includes electric field measurements from the VxB product, in two directions, “horizontal” or “vertical”, both in the instrument XYZ cartesian coordinate frame.\n", @@ -25,7 +25,7 @@ { "cell_type": "code", "execution_count": null, - "id": "d5674e30-fb16-4e7e-88ad-7e943ec94377", + "id": "2", "metadata": {}, "outputs": [], "source": [ @@ -37,7 +37,7 @@ }, { "cell_type": "markdown", - "id": "d247bb6b-f5d4-4940-8ae0-ed5104260c8d", + "id": "3", "metadata": {}, "source": [ "## Fetching data" @@ -46,7 +46,7 @@ { "cell_type": "code", "execution_count": null, - "id": "e5609465-88b5-4da9-8454-b5540322d3dc", + "id": "4", "metadata": {}, "outputs": [], "source": [ @@ -65,7 +65,7 @@ }, { "cell_type": "markdown", - "id": "66d525da-8e86-4998-bfc2-dad5de64f15e", + "id": "5", "metadata": {}, "source": [ "## Processing" @@ -73,7 +73,7 @@ }, { "cell_type": "markdown", - "id": "d8b04719-da62-468f-8653-7369a2790c9b", + "id": "6", "metadata": {}, "source": [ "While the data above contains separate variables for each vector component (\"Ehx\", \"Ehy\", \"Ehz\"), the TFA toolbox can interpret the variable \"Eh_XYZ\" as the combination of these.\n", @@ -84,7 +84,7 @@ { "cell_type": "code", "execution_count": null, - "id": "8bcad701-1975-4f94-b56b-f4d13d2ada42", + "id": "7", "metadata": {}, "outputs": [], "source": [ @@ -119,7 +119,7 @@ }, { "cell_type": "markdown", - "id": "f167c3a5-98e0-4533-9cbf-f138155da4dd", + "id": "8", "metadata": {}, "source": [ "## Plotting" @@ -128,7 +128,7 @@ { "cell_type": "code", "execution_count": null, - "id": "72fc4219-5530-4b46-becf-6469b1db5575", + "id": "9", "metadata": {}, "outputs": [], "source": [ diff --git a/docs/guides/tfa/tfa_MAGx_HR.ipynb b/docs/guides/tfa/tfa_MAGx_HR.ipynb index b101930..6275fcc 100644 --- a/docs/guides/tfa/tfa_MAGx_HR.ipynb +++ b/docs/guides/tfa/tfa_MAGx_HR.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "3a4f6b0d-c2f1-4123-8f40-6cf61588e4d2", + "id": "0", "metadata": { "tags": [] }, @@ -12,7 +12,7 @@ }, { "cell_type": "markdown", - "id": "86de11d1-0750-41db-9069-7e3b7c29f177", + "id": "1", "metadata": {}, "source": [ "For more information about the data and other ways to access it, see:\n", @@ -25,7 +25,7 @@ { "cell_type": "code", "execution_count": null, - "id": "5f8c8b61-3d88-486b-a993-b5944c84e50f", + "id": "2", "metadata": {}, "outputs": [], "source": [ @@ -39,7 +39,7 @@ }, { "cell_type": "markdown", - "id": "ee5d1261-057a-42be-8bd6-aa320c8cc53a", + "id": "3", "metadata": {}, "source": [ "## Fetching data" @@ -47,7 +47,7 @@ }, { "cell_type": "markdown", - "id": "f599465b-753e-4f0d-8f72-1ff29423ac8e", + "id": "4", "metadata": {}, "source": [ "Fetching data is much the same as before, switching \"LR\" for \"HR\". Note that the data volume is 50 times larger so processing will take longer! It's also appropriate to use a shorter time padding." @@ -56,7 +56,7 @@ { "cell_type": "code", "execution_count": null, - "id": "32b9127b-3660-4879-807f-d9b82ad128db", + "id": "5", "metadata": {}, "outputs": [], "source": [ @@ -76,7 +76,7 @@ }, { "cell_type": "markdown", - "id": "f8d41967-0fa0-469d-a317-80d6bbe6331f", + "id": "6", "metadata": {}, "source": [ "## Processing" @@ -84,7 +84,7 @@ }, { "cell_type": "markdown", - "id": "8942d6c1-416a-40c3-a9ed-ff9d4fc9817c", + "id": "7", "metadata": {}, "source": [ "Here we need to identify the different sampling rate `sampling_rate=50`, and we will also choose to instead use the vector magnitude rather than a single component (`use_magnitude=True`)." @@ -93,7 +93,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c7b703cc-2708-4677-99a1-1014740aaf0c", + "id": "8", "metadata": {}, "outputs": [], "source": [ @@ -126,7 +126,7 @@ { "cell_type": "code", "execution_count": null, - "id": "d076d20c-7772-4798-9df7-62e0d26353a2", + "id": "9", "metadata": {}, "outputs": [], "source": [ @@ -138,7 +138,7 @@ }, { "cell_type": "markdown", - "id": "da3d6c0f-55e7-4d16-a307-da5d6dc9ae18", + "id": "10", "metadata": {}, "source": [ "## Plotting" @@ -146,7 +146,7 @@ }, { "cell_type": "markdown", - "id": "6579d336-04f8-4027-912a-a786617bd5b3", + "id": "11", "metadata": {}, "source": [ "A couple of other tricks with the plotting function:\n", @@ -157,7 +157,7 @@ { "cell_type": "code", "execution_count": null, - "id": "497325a6-9c6e-4e60-8239-4d2785eed8c3", + "id": "12", "metadata": {}, "outputs": [], "source": [ diff --git a/docs/guides/tfa/tfa_MAGx_LR.ipynb b/docs/guides/tfa/tfa_MAGx_LR.ipynb index 0f745af..f2003c5 100644 --- a/docs/guides/tfa/tfa_MAGx_LR.ipynb +++ b/docs/guides/tfa/tfa_MAGx_LR.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "68c253c7-1b1c-48d3-82eb-cedf959c7a6d", + "id": "0", "metadata": {}, "source": [ "# Analysis of Swarm MAG LR data (1Hz)\n", @@ -17,7 +17,7 @@ { "cell_type": "code", "execution_count": null, - "id": "92f4673e-491d-4c59-b376-e86a254d2fd9", + "id": "1", "metadata": {}, "outputs": [], "source": [ @@ -30,7 +30,7 @@ }, { "cell_type": "markdown", - "id": "a90ecd00-48ed-4dbd-9b23-931ba654fc27", + "id": "2", "metadata": {}, "source": [ "## Fetching data" @@ -38,7 +38,7 @@ }, { "cell_type": "markdown", - "id": "f8b55f7c-06e3-43be-86b2-c61d5c4ea6e6", + "id": "3", "metadata": {}, "source": [ "As in the introduction example, we will fetch the MAG LR data." @@ -47,7 +47,7 @@ { "cell_type": "code", "execution_count": null, - "id": "44ea9d02-db7f-4a54-b22c-16fd708ef70a", + "id": "4", "metadata": {}, "outputs": [], "source": [ @@ -69,7 +69,7 @@ }, { "cell_type": "markdown", - "id": "bece3525-5191-4c2f-afa4-78cb9bcac85d", + "id": "5", "metadata": {}, "source": [ "## Processing" @@ -77,7 +77,7 @@ }, { "cell_type": "markdown", - "id": "6295e255-87e9-4914-98e0-3b9067ea1db9", + "id": "6", "metadata": {}, "source": [ "This time we will use the `convert_to_mfa` option to rotate the B_NEC vector to the mean-field aligned (MFA) frame. When the MFA frame is used, the `active_component` must be set to one of (0, 1, 2): 0 is the poloidal component, 1 the toroidal and 2 the compressional. Similarly for B_NEC, the numbers correspond to the North (0), East (1) or Center (2) components." @@ -86,7 +86,7 @@ { "cell_type": "code", "execution_count": null, - "id": "e74fe86b-ab51-4160-860a-fff44fd17f6e", + "id": "7", "metadata": {}, "outputs": [], "source": [ @@ -104,7 +104,7 @@ }, { "cell_type": "markdown", - "id": "89c48259-d5b4-425e-8614-939da0622dfb", + "id": "8", "metadata": {}, "source": [ "Even though `B_MFA` isn't available in the original data, this variable becomes available when we select `convert_to_mfa=True`. For more information on the other options, refer to the documentation:" @@ -113,7 +113,7 @@ { "cell_type": "code", "execution_count": null, - "id": "8d65aa75-d058-419c-94bf-0b017b72b219", + "id": "9", "metadata": {}, "outputs": [], "source": [ @@ -122,7 +122,7 @@ }, { "cell_type": "markdown", - "id": "f0e2d565-ab59-4618-a88e-64938a65e6b0", + "id": "10", "metadata": {}, "source": [ "We can inspect the data directly to get an idea about what has happened using `Preprocess`.\n", @@ -133,7 +133,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c383e483-2ff0-438a-b485-4403b6f12639", + "id": "11", "metadata": {}, "outputs": [], "source": [ @@ -147,7 +147,7 @@ }, { "cell_type": "markdown", - "id": "0944c3d2-d8f5-4cc5-9c35-e9cc1abe5185", + "id": "12", "metadata": {}, "source": [ "Let's prepare the other processes..." @@ -156,7 +156,7 @@ { "cell_type": "code", "execution_count": null, - "id": "5c1e2a51-a17d-48e6-a988-519b9d9a70d4", + "id": "13", "metadata": {}, "outputs": [], "source": [ @@ -180,7 +180,7 @@ }, { "cell_type": "markdown", - "id": "949003be-c283-4cfe-9701-3b0f1e04b551", + "id": "14", "metadata": {}, "source": [ "In practice, you might want to prepare and apply each process in turn to make sure things work right. Here however, we will just apply them all together. Make sure you apply them in the right order!" @@ -189,7 +189,7 @@ { "cell_type": "code", "execution_count": null, - "id": "0c518412-513f-4fab-a065-b77347a65609", + "id": "15", "metadata": {}, "outputs": [], "source": [ @@ -200,7 +200,7 @@ }, { "cell_type": "markdown", - "id": "768468ae-2beb-4fe1-aec2-25b3190372f7", + "id": "16", "metadata": {}, "source": [ "## Plotting" @@ -209,7 +209,7 @@ { "cell_type": "code", "execution_count": null, - "id": "0fde2ec2-4366-43de-b8ad-7514754434b5", + "id": "17", "metadata": {}, "outputs": [], "source": [ diff --git a/docs/guides/tfa/tfa_background.ipynb b/docs/guides/tfa/tfa_background.ipynb index e48ac00..af18e9b 100644 --- a/docs/guides/tfa/tfa_background.ipynb +++ b/docs/guides/tfa/tfa_background.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "87f0aae4-34e4-475e-be72-2b6bb63877f2", + "id": "0", "metadata": {}, "source": [ "# TFA and the Wavelet Transform" @@ -10,7 +10,7 @@ }, { "cell_type": "markdown", - "id": "02a9b938-2d3a-4d75-ae7c-5834512048e3", + "id": "1", "metadata": {}, "source": [ "Here we delve a little into the background of the TFA toolbox.\n", @@ -21,7 +21,7 @@ { "cell_type": "code", "execution_count": null, - "id": "0a174a3b-babf-4386-ad46-7bcc361a4e81", + "id": "2", "metadata": {}, "outputs": [], "source": [ @@ -35,7 +35,7 @@ }, { "cell_type": "markdown", - "id": "66a7fe5b-7589-4ead-b72a-3b28c4fd4fb2", + "id": "3", "metadata": {}, "source": [ "Get some data and apply the preprocessing." @@ -44,7 +44,7 @@ { "cell_type": "code", "execution_count": null, - "id": "145c4054-3285-47dd-b858-83f5d5b1d8ce", + "id": "4", "metadata": {}, "outputs": [], "source": [ @@ -63,7 +63,7 @@ { "cell_type": "code", "execution_count": null, - "id": "a2a8ccce-d99c-4ea8-9212-8cec709c0923", + "id": "5", "metadata": {}, "outputs": [], "source": [ @@ -78,7 +78,7 @@ }, { "cell_type": "markdown", - "id": "f702d99f-a102-4ef2-b50f-7874295228e7", + "id": "6", "metadata": {}, "source": [ "The `TFA_Variable` has been set with the content of `F` (the scalar magnetic data)." @@ -87,7 +87,7 @@ { "cell_type": "code", "execution_count": null, - "id": "9b4b9baf-2e38-430a-9db8-e86b00efa8e8", + "id": "7", "metadata": {}, "outputs": [], "source": [ @@ -96,7 +96,7 @@ }, { "cell_type": "markdown", - "id": "89e457c7-bb2e-4e15-9fb1-b90149d45be4", + "id": "8", "metadata": {}, "source": [ "Let's test the analysis with an artificial series, so we'll replace the `TFA_Variable` with a time series of our choice, with a specific frequency of 40 mHz (i.e. 25 sec) and amplitude of 4 nT.\n", @@ -107,7 +107,7 @@ { "cell_type": "code", "execution_count": null, - "id": "480c693b-41f4-4658-8fc4-885334596419", + "id": "9", "metadata": {}, "outputs": [], "source": [ @@ -129,7 +129,7 @@ { "cell_type": "code", "execution_count": null, - "id": "5332d5c3-f13c-43c3-97d1-ad255c398da9", + "id": "10", "metadata": {}, "outputs": [], "source": [ @@ -138,7 +138,7 @@ }, { "cell_type": "markdown", - "id": "103811c5-72dd-4c49-9319-238356a64940", + "id": "11", "metadata": {}, "source": [ "Let's see the effect of the cleaning routine..." @@ -147,7 +147,7 @@ { "cell_type": "code", "execution_count": null, - "id": "1424d095-c5c7-409d-aebc-7dc6c6f931ae", + "id": "12", "metadata": {}, "outputs": [], "source": [ @@ -163,7 +163,7 @@ }, { "cell_type": "markdown", - "id": "1b0e36d6-b25f-4536-802f-320c3b524efa", + "id": "13", "metadata": {}, "source": [ "... and the filtering..." @@ -172,7 +172,7 @@ { "cell_type": "code", "execution_count": null, - "id": "3aadd612-7864-4d5e-a83e-819ec676ae15", + "id": "14", "metadata": {}, "outputs": [], "source": [ @@ -186,7 +186,7 @@ }, { "cell_type": "markdown", - "id": "793e5ef9-577b-47f8-b6ef-2fba29f24710", + "id": "15", "metadata": {}, "source": [ "Next the wavelet transform is applied..." @@ -195,7 +195,7 @@ { "cell_type": "code", "execution_count": null, - "id": "4321c622-ea6c-4b48-992c-5c3de8a05004", + "id": "16", "metadata": {}, "outputs": [], "source": [ @@ -211,7 +211,7 @@ { "cell_type": "code", "execution_count": null, - "id": "a5e09b36-2486-4a1a-b212-d477858ea2f6", + "id": "17", "metadata": {}, "outputs": [], "source": [ @@ -221,7 +221,7 @@ { "cell_type": "code", "execution_count": null, - "id": "93f1af27-1d18-43fa-ac54-e81134b9391d", + "id": "18", "metadata": {}, "outputs": [], "source": [ diff --git a/docs/installation.md b/docs/installation.md index a6c1d12..d4378b1 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -8,14 +8,21 @@ The easiest way to use SwarmPAL is in the Swarm Virtual Research Environment (re The package is available from PyPI: +If you *do not need* the DSECS toolbox: ```bash pip install swarmpal ``` +If you *do need* the DSECS toolbox: +```bash +pip install swarmpal[dsecs] +``` +which includes [apexpy](https://github.com/aburrell/apexpy), which is needed for the DSECS toolbox. There can be some trouble installing this so you might need to manually install apexpy first. + :::{admonition} New to Python? -:class: note To setup Python on your system, check guidance on the [viresclient installation notes](https://viresclient.readthedocs.io/en/latest/installation.html#recommended-setup-if-starting-without-python-already) + ::: ## Install latest development version @@ -26,13 +33,13 @@ Assuming you have a compatible system with git, a fortran compiler, and a Python pip install git+https://github.com/Swarm-DISC/SwarmPAL@staging#egg=swarmpal[dsecs] ``` -:::{admonition} Fotran compiler? -:class: note +:::{admonition} Fortran compiler? If you are using conda, you can get one from: ``` bash conda install conda-forge::fortran-compiler ``` + ::: The fortran compiler is required in order to install the dependency, apexpy. It may be better to try installing apexpy first and debugging that if you run into trouble. diff --git a/pyproject.toml b/pyproject.toml index 9374900..74ff92c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,8 +49,8 @@ dsecs = [ "apexpy>=1.1.0", "pyproj>=3", ] -dsecs_cp39_manylinux = [ - "apexpy @ https://github.com/smithara/apexpy/releases/download/v2.0.1-binaries/apexpy-2.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", +dsecs_cp311_manylinux = [ + "apexpy @ https://github.com/smithara/apexpy/releases/download/v2.0.1-binaries/apexpy-2.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", "pyproj>=3" ] test = [ @@ -61,10 +61,10 @@ dev = [ "nox >=2022", ] docs = [ - "Sphinx==4.5", - "myst-nb==0.17.1", - "sphinx-book-theme==0.3.3", - "sphinx_copybutton==0.5.1", + "Sphinx==7.2", + "myst-nb==1.0", + "sphinx-book-theme==1.1", + "sphinx_copybutton==0.5", "matplotlib==3.6", "ipywidgets==8.0", "ipython!=8.7.0", @@ -86,6 +86,8 @@ filterwarnings = [ # Ignore warnings raised by apexpy (temporarily?) "ignore::RuntimeWarning", "ignore::UserWarning", + # Ignore warning from pandas + "ignore:(\nPyarrow will become a required dependency of pandas):DeprecationWarning", ] testpaths = [ "tests", @@ -122,3 +124,48 @@ check_untyped_defs = true [tool.isort] profile = "black" + +[tool.ruff] +src = ["src"] + +[tool.ruff.lint] +# TODO: Re-enable those commented out +extend-select = [ +# "B", # flake8-bugbear + "I", # isort + "ARG", # flake8-unused-arguments +# "C4", # flake8-comprehensions +# "EM", # flake8-errmsg +# "ICN", # flake8-import-conventions +# "G", # flake8-logging-format + "PGH", # pygrep-hooks + "PIE", # flake8-pie + "PL", # pylint + "PT", # flake8-pytest-style +# "PTH", # flake8-use-pathlib +# "RET", # flake8-return + "RUF", # Ruff-specific +# "SIM", # flake8-simplify + "T20", # flake8-print + "UP", # pyupgrade + "YTT", # flake8-2020 + "EXE", # flake8-executable + "NPY", # NumPy specific rules + "PD", # pandas-vet + "FURB", # refurb + "PYI", # flake8-pyi +] +ignore = [ + "PLR09", # Too many <...> + "PLR2004", # Magic value used in comparison + "ISC001", # Conflicts with formatter + # TODO: Check if we can re-enable these: + "PLW0127", + "PLR1714", +] +typing-modules = ["mypackage._compat.typing"] +isort.required-imports = ["from __future__ import annotations"] + +[tool.ruff.lint.per-file-ignores] +"tests/**" = ["T20"] +"noxfile.py" = ["T20"] diff --git a/src/swarmpal/__init__.py b/src/swarmpal/__init__.py index 4d28d17..62b4d1c 100644 --- a/src/swarmpal/__init__.py +++ b/src/swarmpal/__init__.py @@ -1,5 +1,7 @@ +from __future__ import annotations + from swarmpal import io, toolboxes -__version__ = "0.1.0" +__version__ = "0.2.0-alpha" __all__ = ("__version__", "io", "toolboxes") diff --git a/src/swarmpal/io/__init__.py b/src/swarmpal/io/__init__.py index b504005..daec22e 100644 --- a/src/swarmpal/io/__init__.py +++ b/src/swarmpal/io/__init__.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from swarmpal.io._paldata import ( PalDataItem, PalDataTreeAccessor, diff --git a/src/swarmpal/io/_datafetchers.py b/src/swarmpal/io/_datafetchers.py index 93c3d1b..ddf5a9f 100644 --- a/src/swarmpal/io/_datafetchers.py +++ b/src/swarmpal/io/_datafetchers.py @@ -173,7 +173,7 @@ def _hapi_to_xarray(data: ArrayLike, meta: dict) -> Dataset: ds = Dataset( data_vars={ timevar: (timevar, tdata), - **{_name: (_dim, data[_name]) for _name, _dim, in zip(varnames, dims)}, + **{_name: (_dim, data[_name]) for _name, _dim in zip(varnames, dims)}, } ) # Assign metadata for each data variable diff --git a/src/swarmpal/toolboxes/dsecs/__init__.py b/src/swarmpal/toolboxes/dsecs/__init__.py index b64ca94..567fc09 100644 --- a/src/swarmpal/toolboxes/dsecs/__init__.py +++ b/src/swarmpal/toolboxes/dsecs/__init__.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from swarmpal.toolboxes.dsecs import aux_tools, dsecs_algorithms, processes __all__ = ( diff --git a/src/swarmpal/toolboxes/dsecs/aux_tools.py b/src/swarmpal/toolboxes/dsecs/aux_tools.py index c70162c..63c2095 100644 --- a/src/swarmpal/toolboxes/dsecs/aux_tools.py +++ b/src/swarmpal/toolboxes/dsecs/aux_tools.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import logging import numpy as np @@ -374,7 +376,7 @@ def sub_Swarm_grids(lat1, lon1, lat2, lon2, Dlat2D, LonRatio, ExtLat2D, ExtLon2D ) # Make 2D second gradient matrix in latitude. - # NOTE: it might be more accurate to make the gradient matric in along-track direction, not latitude. + # NOTE: it might be more accurate to make the gradient matrix in along-track direction, not latitude. # Here we assume that poles are listed along latitude at each longitude. # i.e. [(lat1,lon1) (lat2,lon1) ... (latN,lon1) (lat1,lon2) ...] # This corresponds to lat2D(:) according to the above construction diff --git a/src/swarmpal/toolboxes/dsecs/dsecs_algorithms.py b/src/swarmpal/toolboxes/dsecs/dsecs_algorithms.py index fe15034..9377f40 100644 --- a/src/swarmpal/toolboxes/dsecs/dsecs_algorithms.py +++ b/src/swarmpal/toolboxes/dsecs/dsecs_algorithms.py @@ -3,7 +3,7 @@ Adapted from MatLab code by Heikki Vanhamäki. """ - +from __future__ import annotations import logging @@ -276,7 +276,7 @@ def SECS_1D_DivFree_magnetic(latB, latSECS, rb, rsecs, M): t_aux = np.zeros((len(latB), M)) t_aux[:, 0] = 0.2 * np.pi / rb * t * np.where(rb <= rsecs, 1, -rsecs / rb) above = rb > rsecs - for i in range(0, M - 1): + for i in range(M - 1): t_aux[:, i + 1] = t * t_aux[:, i] t_aux[:, i] = t_aux[:, i] / (i + 1 + above) t_aux[:, M - 1] = t_aux[:, M - 1] / (M + above) @@ -1794,9 +1794,7 @@ def fit1D_cf(self): ) @ cf1D tmp = -( np.sin(thetaJ + dtheta) * apuSouth - np.sin(thetaJ - dtheta) * apuNorth - ) / ( - 2 * dtheta * self.grid.Ri * np.sin(thetaJ) - ) # radial current = -div + ) / (2 * dtheta * self.grid.Ri * np.sin(thetaJ)) # radial current = -div tmp = np.reshape( tmp * 1000, self.grid.out.magLat.shape ) # A/km^2 --> nA/m^2 @@ -2063,8 +2061,7 @@ def dump(self): + self.remoteCf2dDipMagJt ) MagJphiCf = ( - self.cf2dDipMagJp - + self.remoteCf2dDipMagJp + self.cf2dDipMagJp + self.remoteCf2dDipMagJp # self.df1DJp + #self.df2dMagJp + self.cf2dDipMagJp + self.remoteCf2dDipMagJp ) diff --git a/src/swarmpal/toolboxes/dsecs/processes.py b/src/swarmpal/toolboxes/dsecs/processes.py index e77109a..746c727 100644 --- a/src/swarmpal/toolboxes/dsecs/processes.py +++ b/src/swarmpal/toolboxes/dsecs/processes.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from apexpy import Apex from datatree import DataTree from numpy import cos, deg2rad, sin diff --git a/src/swarmpal/toolboxes/fac/__init__.py b/src/swarmpal/toolboxes/fac/__init__.py index b709627..1ede0d3 100644 --- a/src/swarmpal/toolboxes/fac/__init__.py +++ b/src/swarmpal/toolboxes/fac/__init__.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from swarmpal.toolboxes.fac import processes __all__ = ("processes",) diff --git a/src/swarmpal/toolboxes/fac/processes.py b/src/swarmpal/toolboxes/fac/processes.py index 467cbc5..f7366a9 100644 --- a/src/swarmpal/toolboxes/fac/processes.py +++ b/src/swarmpal/toolboxes/fac/processes.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import matplotlib.pyplot as plt from datatree import DataTree, register_datatree_accessor from numpy import stack diff --git a/src/swarmpal/toolboxes/tfa/__init__.py b/src/swarmpal/toolboxes/tfa/__init__.py index 9b123ba..f0de8a6 100644 --- a/src/swarmpal/toolboxes/tfa/__init__.py +++ b/src/swarmpal/toolboxes/tfa/__init__.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from swarmpal.toolboxes.tfa import plotting, processes, tfalib __all__ = ( diff --git a/src/swarmpal/toolboxes/tfa/plotting.py b/src/swarmpal/toolboxes/tfa/plotting.py index 629ca76..85ca04f 100644 --- a/src/swarmpal/toolboxes/tfa/plotting.py +++ b/src/swarmpal/toolboxes/tfa/plotting.py @@ -3,10 +3,10 @@ import logging import matplotlib.dates as mdt -import matplotlib.gridspec as gridspec import matplotlib.pyplot as plt import numpy as np import xarray as xr +from matplotlib import gridspec from swarmpal.utils.exceptions import PalError diff --git a/src/swarmpal/toolboxes/tfa/processes.py b/src/swarmpal/toolboxes/tfa/processes.py index 293ee12..4886684 100644 --- a/src/swarmpal/toolboxes/tfa/processes.py +++ b/src/swarmpal/toolboxes/tfa/processes.py @@ -228,7 +228,7 @@ def _flag_cleaning(self, ds): max_val = max_val if max_val else FLAG_THRESHOLDS[varname]["max_val"] # Set flagged values to NaN inds_to_remove = ds[flagname] > max_val - ds[varname][inds_to_remove, ...] = np.NaN + ds[varname][inds_to_remove, ...] = np.nan return ds def _constant_cadence(self, da): @@ -320,7 +320,7 @@ def _clean_variable(self, target_var) -> DataArray: method=self.config.get("method"), multiplier=self.config.get("multiplier"), ) - target_var.data[inds] = np.NaN + target_var.data[inds] = np.nan # Interpolate over gaps s = target_var.data.shape if len(s) == 1: diff --git a/src/swarmpal/toolboxes/tfa/tfalib.py b/src/swarmpal/toolboxes/tfa/tfalib.py index 3df1f08..9cbc5aa 100644 --- a/src/swarmpal/toolboxes/tfa/tfalib.py +++ b/src/swarmpal/toolboxes/tfa/tfalib.py @@ -3,13 +3,13 @@ @author: constantinos@noa.gr """ +from __future__ import annotations import sys import numpy as np import pandas as pd -import scipy.interpolate as interpolate -import scipy.signal as signal +from scipy import interpolate, signal from scipy.fft import fft, ifft R_E = 6371.2 # reference Earth radius in km @@ -129,10 +129,10 @@ def constant_cadence(t_obs, x_obs, sampling_rate, interp=False): t_rec = np.arange(init_t, init_t + time_rec_N * dt + 1e-6, dt) if multiDim: - x_rec = np.full((N, M), np.NaN) + x_rec = np.full((N, M), np.nan) x_rec[inds, :] = x_obs else: - x_rec = np.full(t_rec.shape, np.NaN) + x_rec = np.full(t_rec.shape, np.nan) x_rec[inds] = x_obs nonnan_mask = ~np.isnan(x_rec) @@ -146,7 +146,7 @@ def constant_cadence(t_obs, x_obs, sampling_rate, interp=False): # numpy interpolation, without extrapolation of final value # x_int = np.interp(t_rec, t_obs, x_obs) - x_int[~nonnan_mask] = np.NaN + x_int[~nonnan_mask] = np.nan if multiDim and transp: x_int = x_int.T @@ -217,8 +217,8 @@ def moving_mean_and_stdev(x, window_size, unbiased_std=True): ) # initialize outputs - moving_mean = np.full(x.shape, np.NaN) - moving_stdev = np.full(x.shape, np.NaN) + moving_mean = np.full(x.shape, np.nan) + moving_stdev = np.full(x.shape, np.nan) for i in range(M): # convolve() works with 1-D series so use the appropriate dimensionality @@ -244,7 +244,7 @@ def moving_mean_and_stdev(x, window_size, unbiased_std=True): stdev *= np.sqrt(moving_N / (moving_N - 1)) # replace NaNs that were removed previously - x1[~nonNaNs] = np.NaN + x1[~nonNaNs] = np.nan moving_mean[:, i] = m moving_stdev[:, i] = stdev @@ -330,7 +330,7 @@ def moving_q25_and_q75(x, window_size): return moving_q25, moving_q75 -def outliers(x, window_size, method="iqr", multiplier=np.NaN): +def outliers(x, window_size, method="iqr", multiplier=np.nan): """ Find statistical outliers in data @@ -394,7 +394,7 @@ def outliers(x, window_size, method="iqr", multiplier=np.NaN): >>> plt.show() """ - if not (type(multiplier) == list or type(multiplier) == np.ndarray): + if not isinstance(multiplier, (list, np.ndarray)): multiplier = [multiplier, multiplier] inds = np.full(x.shape, False) @@ -698,7 +698,7 @@ def mfa(B_NEC, B_MEAN_NEC, R_NEC=None): R_NEC = np.zeros(B_NEC.shape) R_NEC[:, 2] = -1 - MFA = np.full(B_NEC.shape, np.NaN) + MFA = np.full(B_NEC.shape, np.nan) # create the unitary vector of the mean field B_MEAN = magn(B_MEAN_NEC) diff --git a/src/swarmpal/utils/exceptions.py b/src/swarmpal/utils/exceptions.py index d2ae25e..f34f241 100644 --- a/src/swarmpal/utils/exceptions.py +++ b/src/swarmpal/utils/exceptions.py @@ -1,2 +1,5 @@ +from __future__ import annotations + + class PalError(Exception): ... diff --git a/tests/io/test_datafetchers.py b/tests/io/test_datafetchers.py index 788ce7c..7733d2e 100644 --- a/tests/io/test_datafetchers.py +++ b/tests/io/test_datafetchers.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import pytest from xarray import Dataset @@ -11,7 +13,7 @@ def test_get_fetcher(): get_fetcher("manual") -@pytest.mark.remote +@pytest.mark.remote() def test_vires_fetcher(): params = dict( collection="SW_OPER_MAGA_LR_1B", @@ -31,7 +33,7 @@ def test_vires_fetcher(): assert isinstance(vires_dataset, Dataset) -@pytest.mark.remote +@pytest.mark.remote() def test_hapi_fetcher(): params = dict( dataset="SW_OPER_MAGA_LR_1B", diff --git a/tests/io/test_paldata.py b/tests/io/test_paldata.py index 518cafe..b0fc5c3 100644 --- a/tests/io/test_paldata.py +++ b/tests/io/test_paldata.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from datetime import datetime, timedelta from os.path import join as join_path @@ -9,7 +11,7 @@ from swarmpal.io._paldata import PalDataItem, create_paldata -@pytest.mark.remote +@pytest.mark.remote() def test_paldataitem_vires(): params = dict( collection="SW_OPER_MAGA_LR_1B", @@ -30,8 +32,8 @@ def test_paldataitem_vires(): return item.xarray -@pytest.mark.remote -@pytest.fixture +@pytest.mark.remote() +@pytest.fixture() def xarray_data_file(tmp_path): ds = test_paldataitem_vires() target_output = join_path(tmp_path, "test_data.nc") @@ -39,7 +41,7 @@ def xarray_data_file(tmp_path): return target_output -@pytest.mark.remote +@pytest.mark.remote() def test_paldataitem_hapi(): params = dict( dataset="SW_OPER_MAGA_LR_1B", @@ -69,7 +71,7 @@ def test_paldataitem_manual(xarray_data_file): assert isinstance(item.xarray, Dataset) -@pytest.mark.remote +@pytest.mark.remote() def test_time_pad_vires(): start_time = datetime(2016, 1, 1, 0, 0, 0) end_time = datetime(2016, 1, 1, 0, 0, 10) @@ -95,7 +97,7 @@ def test_time_pad_vires(): # assert pdi.xarray.attrs["PAL_meta"]["analysis_window"][1] == end_time.isoformat() -@pytest.mark.remote +@pytest.mark.remote() def test_time_pad_hapi(): start_time = datetime(2016, 1, 1, 0, 0, 0) end_time = datetime(2016, 1, 1, 0, 0, 10) @@ -119,8 +121,8 @@ def test_time_pad_hapi(): # assert pdi.xarray.attrs["PAL_meta"]["analysis_window"][1] == end_time.isoformat() -@pytest.mark.remote -@pytest.fixture +@pytest.mark.remote() +@pytest.fixture() def paldata_item_MAGA(): data_params = dict( collection="SW_OPER_MAGA_LR_1B", @@ -136,8 +138,8 @@ def paldata_item_MAGA(): return pdi -@pytest.mark.remote -@pytest.fixture +@pytest.mark.remote() +@pytest.fixture() def paldata_item_MAGB(): data_params = dict( collection="SW_OPER_MAGB_LR_1B", @@ -153,7 +155,7 @@ def paldata_item_MAGB(): return pdi -@pytest.mark.remote +@pytest.mark.remote() def test_create_paldata(paldata_item_MAGA, paldata_item_MAGB): # Test basic paldata data = create_paldata(paldata_item_MAGA) diff --git a/tests/io/test_palprocess.py b/tests/io/test_palprocess.py index a86615c..48424b5 100644 --- a/tests/io/test_palprocess.py +++ b/tests/io/test_palprocess.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import pytest from datatree import DataTree from xarray import Dataset @@ -5,8 +7,8 @@ from swarmpal.io._paldata import PalDataItem, PalProcess, create_paldata -@pytest.mark.remote -@pytest.fixture +@pytest.mark.remote() +@pytest.fixture() def paldata_MAGA(): data_params = dict( collection="SW_OPER_MAGA_LR_1B", @@ -21,7 +23,7 @@ def paldata_MAGA(): return data -@pytest.mark.remote +@pytest.mark.remote() def test_palprocess(paldata_MAGA): """Test the creation and use of a basic PalProcess""" data = paldata_MAGA diff --git a/tests/test_package.py b/tests/test_package.py index 03021b8..88f5c36 100644 --- a/tests/test_package.py +++ b/tests/test_package.py @@ -17,7 +17,7 @@ def test_imports(): print(module.__doc__) -@pytest.mark.dsecs +@pytest.mark.dsecs() def test_imports_dsecs(): from swarmpal.toolboxes import dsecs