From bb37f294e04c08bf669c8635f728c58db35f6cea Mon Sep 17 00:00:00 2001 From: eli knaap Date: Thu, 15 Feb 2024 14:26:56 -0800 Subject: [PATCH 1/9] update tests --- .github/workflows/build_docs.yml | 2 +- .github/workflows/unittests.yml | 94 ++- {.ci => ci}/310.yml | 2 - {.ci => ci}/311.yml | 11 +- .ci/39.yml => ci/312.yml | 4 +- codecov.yml | 20 + environment.yml | 1 - tobler/_version.py | 623 ------------------- tobler/dasymetric/masked_area_interpolate.py | 3 +- tobler/dasymetric/raster_tools.py | 10 +- tobler/tests/test_area_interpolators.py | 6 +- tobler/tests/test_dasymetric.py | 46 +- tobler/tests/test_model.py | 43 +- 13 files changed, 135 insertions(+), 730 deletions(-) rename {.ci => ci}/310.yml (92%) rename {.ci => ci}/311.yml (69%) rename .ci/39.yml => ci/312.yml (91%) create mode 100644 codecov.yml delete mode 100644 tobler/_version.py diff --git a/.github/workflows/build_docs.yml b/.github/workflows/build_docs.yml index af0611a..1289a43 100644 --- a/.github/workflows/build_docs.yml +++ b/.github/workflows/build_docs.yml @@ -13,7 +13,7 @@ strategy: matrix: os: ['ubuntu-latest'] - environment-file: [.ci/39.yml] + environment-file: [ci/311.yml] experimental: [false] defaults: run: diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml index e9d23e5..427dac0 100644 --- a/.github/workflows/unittests.yml +++ b/.github/workflows/unittests.yml @@ -1,4 +1,5 @@ - name: Unit Tests + name: Continuous Integration + on: push: branches: @@ -8,52 +9,91 @@ - '*' schedule: - cron: '59 23 * * *' + workflow_dispatch: + inputs: + version: + description: Manual CI Run + default: test + required: false jobs: - unittests: - name: CI (${{ matrix.os }}-${{ matrix.environment-file }}) + tests: + name: ${{ matrix.os }}, ${{ matrix.environment-file }} runs-on: ${{ matrix.os }} - continue-on-error: false - timeout-minutes: 45 + timeout-minutes: 30 strategy: matrix: - os: [ubuntu-latest, macos-latest, windows-latest] - environment-file: [ .ci/39.yml, .ci/310.yml, .ci/311.yml] + os: [ubuntu-latest] + environment-file: [ + ci/310.yml, + ci/311.yml, + ci/312.yml, + ] + include: + - environment-file: ci/312.yml + os: macos-latest + - environment-file: ci/312.yml + os: macos-14 # Apple Silicon + - environment-file: ci/312.yml + os: windows-latest fail-fast: false + defaults: + run: + shell: bash -l {0} + steps: - name: checkout repo - uses: actions/checkout@v2 + uses: actions/checkout@v4 + with: + fetch-depth: 0 # Fetch all history for all branches and tags. - name: setup micromamba - uses: mamba-org/provision-with-micromamba@main + uses: mamba-org/setup-micromamba@v1 with: environment-file: ${{ matrix.environment-file }} micromamba-version: 'latest' - - name: reinstall tobler - bash - shell: bash -l {0} - run: pip install -e . --no-deps --force-reinstall - if: matrix.os != 'windows-latest' - - - name: reinstall tobler - powershell - shell: powershell - run: pip install -e . --no-deps --force-reinstall - if: matrix.os == 'windows-latest' + ############################################################################## + #- name: install pulp via pip + # run: | + # pip install pulp + # pulptest + ############################################################################## - - name: run pytest - bash - shell: bash -l {0} - run: pytest -v tobler --cov=tobler --cov-report=xml - if: matrix.os != 'windows-latest' + - name: environment info + run: | + micromamba info + micromamba list + + - name: spatial versions + run: 'python -c "import geopandas; geopandas.show_versions();"' - - name: run pytest - powershell - shell: powershell - run: pytest -v tobler --cov=tobler --cov-report=xml - if: matrix.os == 'windows-latest' + - name: run tests + run: | + pytest tobler \ + -v \ + -r a \ + -n auto \ + --color yes \ + --cov tobler \ + --cov-append \ + --cov-report term-missing \ + --cov-report xml - name: codecov - uses: codecov/codecov-action@v1 + uses: codecov/codecov-action@v3 with: token: ${{ secrets.CODECOV_TOKEN }} file: ./coverage.xml name: tobler-codecov + + - name: Generate and publish the report + if: | + failure() + && steps.status.outcome == 'failure' + && github.event_name == 'schedule' + && github.repository_owner == 'pysal' + uses: xarray-contrib/issue-from-pytest-log@v1 + with: + log-path: pytest-log.jsonl \ No newline at end of file diff --git a/.ci/310.yml b/ci/310.yml similarity index 92% rename from .ci/310.yml rename to ci/310.yml index e666b7a..098016a 100644 --- a/.ci/310.yml +++ b/ci/310.yml @@ -5,7 +5,6 @@ dependencies: - python=3.10 - dask - dask-geopandas - - distributed - jupyterlab - numpy - geopandas @@ -19,7 +18,6 @@ dependencies: - libpysal - tqdm - codecov - - quilt3 - pytest - pytest-mpl - pytest-cov diff --git a/.ci/311.yml b/ci/311.yml similarity index 69% rename from .ci/311.yml rename to ci/311.yml index a533e59..7060008 100644 --- a/.ci/311.yml +++ b/ci/311.yml @@ -6,7 +6,6 @@ dependencies: - jupyterlab - dask - dask-geopandas - - distributed - numpy - geopandas - pandas @@ -20,7 +19,6 @@ dependencies: - tqdm - codecov - pytest - - quilt3 - pytest-mpl - pytest-cov - twine @@ -28,3 +26,12 @@ dependencies: - h3-py - joblib - astropy + - mapclassify + - descartes + - sphinx>=1.4.3 + - sphinxcontrib-bibtex==1.0.0 + - sphinx_bootstrap_theme + - numpydoc + - nbsphinx + - joblib + - astropy \ No newline at end of file diff --git a/.ci/39.yml b/ci/312.yml similarity index 91% rename from .ci/39.yml rename to ci/312.yml index d029839..fa7f409 100644 --- a/.ci/39.yml +++ b/ci/312.yml @@ -2,10 +2,9 @@ name: test channels: - conda-forge dependencies: - - python=3.9 + - python=3.12 - dask - dask-geopandas - - distributed - numpy - geopandas - pandas @@ -18,7 +17,6 @@ dependencies: - tqdm - codecov - pytest - - quilt3 - pytest-mpl - pytest-cov - twine diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 0000000..82613e0 --- /dev/null +++ b/codecov.yml @@ -0,0 +1,20 @@ +codecov: + notify: + after_n_builds: 7 +coverage: + range: 50..95 + round: nearest + precision: 1 + status: + project: + default: + threshold: 15% + patch: + default: + threshold: 20% + target: 60% +comment: + layout: "reach, diff, files" + behavior: once + after_n_builds: 7 + require_changes: true diff --git a/environment.yml b/environment.yml index 7d4c6e2..9930b2f 100644 --- a/environment.yml +++ b/environment.yml @@ -4,7 +4,6 @@ channels: dependencies: - dask-geopandas - dask - - distributed - jupyterlab - numpy - geopandas >=0.13 diff --git a/tobler/_version.py b/tobler/_version.py deleted file mode 100644 index 5b34bec..0000000 --- a/tobler/_version.py +++ /dev/null @@ -1,623 +0,0 @@ - -# This file helps to compute a version number in source trees obtained from -# git-archive tarball (such as those provided by githubs download-from-tag -# feature). Distribution tarballs (built by setup.py sdist) and build -# directories (produced by setup.py build) will contain a much shorter file -# that just contains the computed version number. - -# This file is released into the public domain. Generated by -# versioneer-0.20 (https://github.com/python-versioneer/python-versioneer) - -"""Git implementation of _version.py.""" - -import errno -import os -import re -import subprocess -import sys - - -def get_keywords(): - """Get the keywords needed to look up the version information.""" - # these strings will be replaced by git during git-archive. - # setup.py/versioneer.py will grep for the variable names, so they must - # each be defined on a line of their own. _version.py will just call - # get_keywords(). - git_refnames = "$Format:%d$" - git_full = "$Format:%H$" - git_date = "$Format:%ci$" - keywords = {"refnames": git_refnames, "full": git_full, "date": git_date} - return keywords - - -class VersioneerConfig: # pylint: disable=too-few-public-methods - """Container for Versioneer configuration parameters.""" - - -def get_config(): - """Create, populate and return the VersioneerConfig() object.""" - # these strings are filled in when 'setup.py versioneer' creates - # _version.py - cfg = VersioneerConfig() - cfg.VCS = "git" - cfg.style = "pep440" - cfg.tag_prefix = "v" - cfg.parentdir_prefix = "tobler-" - cfg.versionfile_source = "tobler/_version.py" - cfg.verbose = False - return cfg - - -class NotThisMethod(Exception): - """Exception raised if a method is not valid for the current scenario.""" - - -LONG_VERSION_PY = {} -HANDLERS = {} - - -def register_vcs_handler(vcs, method): # decorator - """Create decorator to mark a method as the handler of a VCS.""" - def decorate(f): - """Store f in HANDLERS[vcs][method].""" - if vcs not in HANDLERS: - HANDLERS[vcs] = {} - HANDLERS[vcs][method] = f - return f - return decorate - - -# pylint:disable=too-many-arguments,consider-using-with # noqa -def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, - env=None): - """Call the given command(s).""" - assert isinstance(commands, list) - process = None - for command in commands: - try: - dispcmd = str([command] + args) - # remember shell=False, so use git.cmd on windows, not just git - process = subprocess.Popen([command] + args, cwd=cwd, env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr - else None)) - break - except EnvironmentError: - e = sys.exc_info()[1] - if e.errno == errno.ENOENT: - continue - if verbose: - print("unable to run %s" % dispcmd) - print(e) - return None, None - else: - if verbose: - print("unable to find command, tried %s" % (commands,)) - return None, None - stdout = process.communicate()[0].strip().decode() - if process.returncode != 0: - if verbose: - print("unable to run %s (error)" % dispcmd) - print("stdout was %s" % stdout) - return None, process.returncode - return stdout, process.returncode - - -def versions_from_parentdir(parentdir_prefix, root, verbose): - """Try to determine the version from the parent directory name. - - Source tarballs conventionally unpack into a directory that includes both - the project name and a version string. We will also support searching up - two directory levels for an appropriately named parent directory - """ - rootdirs = [] - - for _ in range(3): - dirname = os.path.basename(root) - if dirname.startswith(parentdir_prefix): - return {"version": dirname[len(parentdir_prefix):], - "full-revisionid": None, - "dirty": False, "error": None, "date": None} - rootdirs.append(root) - root = os.path.dirname(root) # up a level - - if verbose: - print("Tried directories %s but none started with prefix %s" % - (str(rootdirs), parentdir_prefix)) - raise NotThisMethod("rootdir doesn't start with parentdir_prefix") - - -@register_vcs_handler("git", "get_keywords") -def git_get_keywords(versionfile_abs): - """Extract version information from the given file.""" - # the code embedded in _version.py can just fetch the value of these - # keywords. When used from setup.py, we don't want to import _version.py, - # so we do it with a regexp instead. This function is not used from - # _version.py. - keywords = {} - try: - with open(versionfile_abs, "r") as fobj: - for line in fobj: - if line.strip().startswith("git_refnames ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["refnames"] = mo.group(1) - if line.strip().startswith("git_full ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["full"] = mo.group(1) - if line.strip().startswith("git_date ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["date"] = mo.group(1) - except EnvironmentError: - pass - return keywords - - -@register_vcs_handler("git", "keywords") -def git_versions_from_keywords(keywords, tag_prefix, verbose): - """Get version information from git keywords.""" - if "refnames" not in keywords: - raise NotThisMethod("Short version file found") - date = keywords.get("date") - if date is not None: - # Use only the last line. Previous lines may contain GPG signature - # information. - date = date.splitlines()[-1] - - # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant - # datestamp. However we prefer "%ci" (which expands to an "ISO-8601 - # -like" string, which we must then edit to make compliant), because - # it's been around since git-1.5.3, and it's too difficult to - # discover which version we're using, or to work around using an - # older one. - date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - refnames = keywords["refnames"].strip() - if refnames.startswith("$Format"): - if verbose: - print("keywords are unexpanded, not using") - raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = {r.strip() for r in refnames.strip("()").split(",")} - # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of - # just "foo-1.0". If we see a "tag: " prefix, prefer those. - TAG = "tag: " - tags = {r[len(TAG):] for r in refs if r.startswith(TAG)} - if not tags: - # Either we're using git < 1.8.3, or there really are no tags. We use - # a heuristic: assume all version tags have a digit. The old git %d - # expansion behaves like git log --decorate=short and strips out the - # refs/heads/ and refs/tags/ prefixes that would let us distinguish - # between branches and tags. By ignoring refnames without digits, we - # filter out many common branch names like "release" and - # "stabilization", as well as "HEAD" and "master". - tags = {r for r in refs if re.search(r'\d', r)} - if verbose: - print("discarding '%s', no digits" % ",".join(refs - tags)) - if verbose: - print("likely tags: %s" % ",".join(sorted(tags))) - for ref in sorted(tags): - # sorting will prefer e.g. "2.0" over "2.0rc1" - if ref.startswith(tag_prefix): - r = ref[len(tag_prefix):] - # Filter out refs that exactly match prefix or that don't start - # with a number once the prefix is stripped (mostly a concern - # when prefix is '') - if not re.match(r'\d', r): - continue - if verbose: - print("picking %s" % r) - return {"version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": None, - "date": date} - # no suitable tags, so version is "0+unknown", but full hex is still there - if verbose: - print("no suitable tags, using unknown + full revision id") - return {"version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": "no suitable tags", "date": None} - - -@register_vcs_handler("git", "pieces_from_vcs") -def git_pieces_from_vcs(tag_prefix, root, verbose, runner=run_command): - """Get version from 'git describe' in the root of the source tree. - - This only gets called if the git-archive 'subst' keywords were *not* - expanded, and _version.py hasn't already been rewritten with a short - version string, meaning we're inside a checked out source tree. - """ - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - - _, rc = runner(GITS, ["rev-parse", "--git-dir"], cwd=root, - hide_stderr=True) - if rc != 0: - if verbose: - print("Directory %s not under git control" % root) - raise NotThisMethod("'git rev-parse --git-dir' returned error") - - # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] - # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = runner(GITS, ["describe", "--tags", "--dirty", - "--always", "--long", - "--match", "%s*" % tag_prefix], - cwd=root) - # --long was added in git-1.5.5 - if describe_out is None: - raise NotThisMethod("'git describe' failed") - describe_out = describe_out.strip() - full_out, rc = runner(GITS, ["rev-parse", "HEAD"], cwd=root) - if full_out is None: - raise NotThisMethod("'git rev-parse' failed") - full_out = full_out.strip() - - pieces = {} - pieces["long"] = full_out - pieces["short"] = full_out[:7] # maybe improved later - pieces["error"] = None - - branch_name, rc = runner(GITS, ["rev-parse", "--abbrev-ref", "HEAD"], - cwd=root) - # --abbrev-ref was added in git-1.6.3 - if rc != 0 or branch_name is None: - raise NotThisMethod("'git rev-parse --abbrev-ref' returned error") - branch_name = branch_name.strip() - - if branch_name == "HEAD": - # If we aren't exactly on a branch, pick a branch which represents - # the current commit. If all else fails, we are on a branchless - # commit. - branches, rc = runner(GITS, ["branch", "--contains"], cwd=root) - # --contains was added in git-1.5.4 - if rc != 0 or branches is None: - raise NotThisMethod("'git branch --contains' returned error") - branches = branches.split("\n") - - # Remove the first line if we're running detached - if "(" in branches[0]: - branches.pop(0) - - # Strip off the leading "* " from the list of branches. - branches = [branch[2:] for branch in branches] - if "master" in branches: - branch_name = "master" - elif not branches: - branch_name = None - else: - # Pick the first branch that is returned. Good or bad. - branch_name = branches[0] - - pieces["branch"] = branch_name - - # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] - # TAG might have hyphens. - git_describe = describe_out - - # look for -dirty suffix - dirty = git_describe.endswith("-dirty") - pieces["dirty"] = dirty - if dirty: - git_describe = git_describe[:git_describe.rindex("-dirty")] - - # now we have TAG-NUM-gHEX or HEX - - if "-" in git_describe: - # TAG-NUM-gHEX - mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) - if not mo: - # unparseable. Maybe git-describe is misbehaving? - pieces["error"] = ("unable to parse git-describe output: '%s'" - % describe_out) - return pieces - - # tag - full_tag = mo.group(1) - if not full_tag.startswith(tag_prefix): - if verbose: - fmt = "tag '%s' doesn't start with prefix '%s'" - print(fmt % (full_tag, tag_prefix)) - pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" - % (full_tag, tag_prefix)) - return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix):] - - # distance: number of commits since tag - pieces["distance"] = int(mo.group(2)) - - # commit: short hex revision ID - pieces["short"] = mo.group(3) - - else: - # HEX: no tags - pieces["closest-tag"] = None - count_out, rc = runner(GITS, ["rev-list", "HEAD", "--count"], cwd=root) - pieces["distance"] = int(count_out) # total number of commits - - # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = runner(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[0].strip() - # Use only the last line. Previous lines may contain GPG signature - # information. - date = date.splitlines()[-1] - pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - - return pieces - - -def plus_or_dot(pieces): - """Return a + if we don't already have one, else return a .""" - if "+" in pieces.get("closest-tag", ""): - return "." - return "+" - - -def render_pep440(pieces): - """Build up version string, with post-release "local version identifier". - - Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you - get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty - - Exceptions: - 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += plus_or_dot(pieces) - rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0+untagged.%d.g%s" % (pieces["distance"], - pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_branch(pieces): - """TAG[[.dev0]+DISTANCE.gHEX[.dirty]] . - - The ".dev0" means not master branch. Note that .dev0 sorts backwards - (a feature branch will appear "older" than the master branch). - - Exceptions: - 1: no tags. 0[.dev0]+untagged.DISTANCE.gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - if pieces["branch"] != "master": - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0" - if pieces["branch"] != "master": - rendered += ".dev0" - rendered += "+untagged.%d.g%s" % (pieces["distance"], - pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_pre(pieces): - """TAG[.post0.devDISTANCE] -- No -dirty. - - Exceptions: - 1: no tags. 0.post0.devDISTANCE - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += ".post0.dev%d" % pieces["distance"] - else: - # exception #1 - rendered = "0.post0.dev%d" % pieces["distance"] - return rendered - - -def render_pep440_post(pieces): - """TAG[.postDISTANCE[.dev0]+gHEX] . - - The ".dev0" means dirty. Note that .dev0 sorts backwards - (a dirty tree will appear "older" than the corresponding clean one), - but you shouldn't be releasing software with -dirty anyways. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "g%s" % pieces["short"] - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += "+g%s" % pieces["short"] - return rendered - - -def render_pep440_post_branch(pieces): - """TAG[.postDISTANCE[.dev0]+gHEX[.dirty]] . - - The ".dev0" means not master branch. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0]+gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["branch"] != "master": - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "g%s" % pieces["short"] - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["branch"] != "master": - rendered += ".dev0" - rendered += "+g%s" % pieces["short"] - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_old(pieces): - """TAG[.postDISTANCE[.dev0]] . - - The ".dev0" means dirty. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - return rendered - - -def render_git_describe(pieces): - """TAG[-DISTANCE-gHEX][-dirty]. - - Like 'git describe --tags --dirty --always'. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render_git_describe_long(pieces): - """TAG-DISTANCE-gHEX[-dirty]. - - Like 'git describe --tags --dirty --always -long'. - The distance/hash is unconditional. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render(pieces, style): - """Render the given version pieces into the requested style.""" - if pieces["error"]: - return {"version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None} - - if not style or style == "default": - style = "pep440" # the default - - if style == "pep440": - rendered = render_pep440(pieces) - elif style == "pep440-branch": - rendered = render_pep440_branch(pieces) - elif style == "pep440-pre": - rendered = render_pep440_pre(pieces) - elif style == "pep440-post": - rendered = render_pep440_post(pieces) - elif style == "pep440-post-branch": - rendered = render_pep440_post_branch(pieces) - elif style == "pep440-old": - rendered = render_pep440_old(pieces) - elif style == "git-describe": - rendered = render_git_describe(pieces) - elif style == "git-describe-long": - rendered = render_git_describe_long(pieces) - else: - raise ValueError("unknown style '%s'" % style) - - return {"version": rendered, "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], "error": None, - "date": pieces.get("date")} - - -def get_versions(): - """Get version information or return default if unable to do so.""" - # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have - # __file__, we can work backwards from there to the root. Some - # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which - # case we can only use expanded keywords. - - cfg = get_config() - verbose = cfg.verbose - - try: - return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, - verbose) - except NotThisMethod: - pass - - try: - root = os.path.realpath(__file__) - # versionfile_source is the relative path from the top of the source - # tree (where the .git directory might live) to this file. Invert - # this to find the root from __file__. - for _ in cfg.versionfile_source.split('/'): - root = os.path.dirname(root) - except NameError: - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, - "error": "unable to find root of source tree", - "date": None} - - try: - pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) - return render(pieces, cfg.style) - except NotThisMethod: - pass - - try: - if cfg.parentdir_prefix: - return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) - except NotThisMethod: - pass - - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, - "error": "unable to compute version", "date": None} diff --git a/tobler/dasymetric/masked_area_interpolate.py b/tobler/dasymetric/masked_area_interpolate.py index fe6c53b..74f10ff 100644 --- a/tobler/dasymetric/masked_area_interpolate.py +++ b/tobler/dasymetric/masked_area_interpolate.py @@ -1,8 +1,9 @@ +from warnings import warn + import geopandas as gpd from ..area_weighted import area_interpolate from .raster_tools import extract_raster_features -from warnings import warn def masked_area_interpolate( diff --git a/tobler/dasymetric/raster_tools.py b/tobler/dasymetric/raster_tools.py index b2ce596..253d81a 100644 --- a/tobler/dasymetric/raster_tools.py +++ b/tobler/dasymetric/raster_tools.py @@ -2,18 +2,19 @@ import ast import multiprocessing +import warnings import geopandas as gpd import numpy as np import pandas as pd import rasterio as rio +import rasterstats as rs from joblib import Parallel, delayed from rasterio import features -from shapely.geometry import shape from rasterio.mask import mask -import warnings +from shapely.geometry import shape + from ..util.util import _check_presence_of_crs -import rasterstats as rs def _chunk_dfs(geoms_to_chunk, n_jobs): @@ -104,8 +105,7 @@ def extract_raster_features( if n_jobs == -1: n_jobs = multiprocessing.cpu_count() with rio.open(raster_path) as src: - - raster_crs = src.crs.data + raster_crs = src.crs.to_dict() gdf = gdf.to_crs(raster_crs) geomask = [gdf.unary_union.__geo_interface__] diff --git a/tobler/tests/test_area_interpolators.py b/tobler/tests/test_area_interpolators.py index bc8791b..da0d642 100644 --- a/tobler/tests/test_area_interpolators.py +++ b/tobler/tests/test_area_interpolators.py @@ -15,7 +15,9 @@ def datasets(): sac1 = load_example("Sacramento1") sac2 = load_example("Sacramento2") sac1 = geopandas.read_file(sac1.get_path("sacramentot2.shp")) + sac1 = sac1.to_crs(sac1.estimate_utm_crs()) sac2 = geopandas.read_file(sac2.get_path("SacramentoMSA2.shp")) + sac2 = sac2.to_crs(sac1.crs) sac1["pct_poverty"] = sac1.POV_POP / sac1.POV_TOT categories = ["cat", "dog", "donkey", "wombat", "capybara"] sac1["animal"] = (categories * ((len(sac1) // len(categories)) + 1))[: len(sac1)] @@ -45,8 +47,8 @@ def test_area_interpolate_singlecore(): def test_area_interpolate_extensive(): sac1, sac2 = datasets() area = area_interpolate( - source_df=sac1, - target_df=sac2, + source_df=sac1.to_crs(4326), # trigger warning once + target_df=sac2.to_crs(4326), extensive_variables=["TOT_POP"], n_jobs=1, ) diff --git a/tobler/tests/test_dasymetric.py b/tobler/tests/test_dasymetric.py index ea4122c..0cacef7 100644 --- a/tobler/tests/test_dasymetric.py +++ b/tobler/tests/test_dasymetric.py @@ -1,45 +1,25 @@ """test interpolation functions.""" -import sys -import pandas as pd import geopandas -import pytest -try: - import quilt3 - - QUILTMISSING = False -except ImportError: - QUILTMISSING = True - -import os from libpysal.examples import load_example from tobler.dasymetric import masked_area_interpolate def datasets(): - if not QUILTMISSING: - - if not os.path.exists("nlcd_2011.tif"): - p = quilt3.Package.browse("rasters/nlcd", "s3://spatial-ucr") - p["nlcd_2011.tif"].fetch() - sac1 = load_example("Sacramento1") - sac2 = load_example("Sacramento2") - sac1 = geopandas.read_file(sac1.get_path("sacramentot2.shp")) - sac1 = sac1.to_crs(sac1.estimate_utm_crs()) - sac2 = geopandas.read_file(sac2.get_path("SacramentoMSA2.shp")) - sac2 = sac2.to_crs(sac2.estimate_utm_crs()) - sac1["pct_poverty"] = sac1.POV_POP / sac1.POV_TOT - categories = ["cat", "dog", "donkey", "wombat", "capybara"] - sac1["animal"] = (categories * ((len(sac1) // len(categories)) + 1))[ - : len(sac1) - ] - - return sac1, sac2 - else: - pass + sac1 = load_example("Sacramento1") + sac2 = load_example("Sacramento2") + sac1 = geopandas.read_file(sac1.get_path("sacramentot2.shp")) + sac1 = sac1.to_crs(sac1.estimate_utm_crs()) + sac2 = geopandas.read_file(sac2.get_path("SacramentoMSA2.shp")) + sac2 = sac2.to_crs(sac2.estimate_utm_crs()) + sac1["pct_poverty"] = sac1.POV_POP / sac1.POV_TOT + categories = ["cat", "dog", "donkey", "wombat", "capybara"] + sac1["animal"] = (categories * ((len(sac1) // len(categories)) + 1))[ + : len(sac1) + ] + return sac1, sac2 -@pytest.mark.skipif(QUILTMISSING, reason="quilt3 not available.") def test_masked_area_interpolate(): sac1, sac2 = datasets() masked = masked_area_interpolate( @@ -47,7 +27,7 @@ def test_masked_area_interpolate(): target_df=sac2, extensive_variables=["TOT_POP"], intensive_variables=["pct_poverty"], - raster="nlcd_2011.tif", + raster="https://spatial-ucr.s3.amazonaws.com/nlcd/landcover/nlcd_landcover_2011.tif", pixel_values=[21, 22, 23, 24], ) assert masked.TOT_POP.sum().round(0) == sac1.TOT_POP.sum() diff --git a/tobler/tests/test_model.py b/tobler/tests/test_model.py index 7946669..c0e0ae3 100644 --- a/tobler/tests/test_model.py +++ b/tobler/tests/test_model.py @@ -1,47 +1,30 @@ """test interpolation functions.""" import geopandas -try: - import quilt3 - - QUILTMISSING = False -except ImportError: - QUILTMISSING = True - -import os - -import pytest from libpysal.examples import load_example from tobler.model import glm def datasets(): - if not QUILTMISSING: + sac1 = load_example("Sacramento1") + sac2 = load_example("Sacramento2") + sac1 = geopandas.read_file(sac1.get_path("sacramentot2.shp")) + sac1 = sac1.to_crs(sac1.estimate_utm_crs()) + sac2 = geopandas.read_file(sac2.get_path("SacramentoMSA2.shp")) + sac2 = sac2.to_crs(sac2.estimate_utm_crs()) + sac1["pct_poverty"] = sac1.POV_POP / sac1.POV_TOT + categories = ["cat", "dog", "donkey", "wombat", "capybara"] + sac1["animal"] = (categories * ((len(sac1) // len(categories)) + 1))[ + : len(sac1) + ] - if not os.path.exists("nlcd_2011.tif"): - p = quilt3.Package.browse("rasters/nlcd", "s3://spatial-ucr") - p["nlcd_2011.tif"].fetch() - sac1 = load_example("Sacramento1") - sac2 = load_example("Sacramento2") - sac1 = geopandas.read_file(sac1.get_path("sacramentot2.shp")) - sac1 = sac1.to_crs(sac1.estimate_utm_crs()) - sac2 = geopandas.read_file(sac2.get_path("SacramentoMSA2.shp")) - sac2 = sac2.to_crs(sac2.estimate_utm_crs()) - sac1["pct_poverty"] = sac1.POV_POP / sac1.POV_TOT - categories = ["cat", "dog", "donkey", "wombat", "capybara"] - sac1["animal"] = (categories * ((len(sac1) // len(categories)) + 1))[ - : len(sac1) - ] + return sac1, sac2 - return sac1, sac2 - else: - pass -@pytest.mark.skipif(QUILTMISSING, reason="quilt3 not available.") def test_glm_poisson(): sac1, sac2 = datasets() glm_poisson = glm( - source_df=sac2, target_df=sac1, variable="POP2001", raster="nlcd_2011.tif", + source_df=sac2, target_df=sac1, variable="POP2001", raster="https://spatial-ucr.s3.amazonaws.com/nlcd/landcover/nlcd_landcover_2011.tif", ) assert glm_poisson.POP2001.sum() > 1469000 From 6bbf942b20a842a50f521ae35e3b02464f8ac919 Mon Sep 17 00:00:00 2001 From: eli knaap Date: Thu, 15 Feb 2024 14:48:27 -0800 Subject: [PATCH 2/9] pytest-xdist --- ci/310.yml | 2 ++ ci/311.yml | 2 ++ ci/312.yml | 2 ++ pyproject.toml | 1 + tobler/tests/test_area_interpolators.py | 19 ++++++++++--------- 5 files changed, 17 insertions(+), 9 deletions(-) diff --git a/ci/310.yml b/ci/310.yml index 098016a..ff7f9a5 100644 --- a/ci/310.yml +++ b/ci/310.yml @@ -21,6 +21,8 @@ dependencies: - pytest - pytest-mpl - pytest-cov + - pytest-xdist + - coverage - twine - pip - h3-py diff --git a/ci/311.yml b/ci/311.yml index 7060008..fa7024f 100644 --- a/ci/311.yml +++ b/ci/311.yml @@ -21,6 +21,8 @@ dependencies: - pytest - pytest-mpl - pytest-cov + - pytest-xdist + - coverage - twine - pip - h3-py diff --git a/ci/312.yml b/ci/312.yml index fa7f409..327ad05 100644 --- a/ci/312.yml +++ b/ci/312.yml @@ -16,6 +16,8 @@ dependencies: - libpysal - tqdm - codecov + - pytest-xdist + - coverage - pytest - pytest-mpl - pytest-cov diff --git a/pyproject.toml b/pyproject.toml index 5164011..2b06c19 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,6 +64,7 @@ tests = [ "pytest", "pytest-mpl", "pytest-cov", + "pytest-xdist", "watermark", "h3", "astropy" diff --git a/tobler/tests/test_area_interpolators.py b/tobler/tests/test_area_interpolators.py index da0d642..89067c1 100644 --- a/tobler/tests/test_area_interpolators.py +++ b/tobler/tests/test_area_interpolators.py @@ -1,4 +1,5 @@ """test interpolation functions.""" + import geopandas import dask_geopandas @@ -85,19 +86,17 @@ def test_area_interpolate_categorical(): def test_area_interpolate_categorical_dask(): sac1, sac2 = datasets() - sac1['animal'] = sac1['animal'].astype('category') - dsac1 = ( - dask_geopandas.from_geopandas(sac1, npartitions=2) - .spatial_shuffle(by='hilbert', shuffle='tasks') + sac1["animal"] = sac1["animal"].astype("category") + dsac1 = dask_geopandas.from_geopandas(sac1, npartitions=2).spatial_shuffle( + by="hilbert", shuffle="tasks" ) - dsac2 = ( - dask_geopandas.from_geopandas(sac2, npartitions=2) - .spatial_shuffle(by='hilbert', shuffle='tasks') + dsac2 = dask_geopandas.from_geopandas(sac2, npartitions=2).spatial_shuffle( + by="hilbert", shuffle="tasks" ) area = area_interpolate_dask( source_dgdf=dsac1, target_dgdf=dsac2, - id_col='ZIP', + id_col="ZIP", categorical_variables=["animal"], ).compute() assert_almost_equal(area.animal_cat.sum(), 32, decimal=0) @@ -180,7 +179,9 @@ def test_area_interpolate_parallel(): def test_area_tables_binning(): sac1, sac2 = datasets() - auto = _area_tables_binning(source_df=sac1, target_df=sac2, spatial_index="auto") + auto = _area_tables_binning( + source_df=sac1, target_df=sac2, spatial_index="auto" + ) source = _area_tables_binning( source_df=sac1, target_df=sac2, spatial_index="source" ) From 238f09e3202ded9136ec16e9d73ac60c510d967f Mon Sep 17 00:00:00 2001 From: eli knaap Date: Thu, 15 Feb 2024 14:52:11 -0800 Subject: [PATCH 3/9] install local package; --- .github/workflows/unittests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml index 427dac0..93d131d 100644 --- a/.github/workflows/unittests.yml +++ b/.github/workflows/unittests.yml @@ -68,6 +68,9 @@ - name: spatial versions run: 'python -c "import geopandas; geopandas.show_versions();"' + + - name: install package + run: 'pip install . --no-deps' - name: run tests run: | From b1ed0984b6a91025cc144deb0249883ed4652dd8 Mon Sep 17 00:00:00 2001 From: eli knaap Date: Thu, 15 Feb 2024 15:23:33 -0800 Subject: [PATCH 4/9] reproject to 4326 --- tobler/tests/test_area_interpolators.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tobler/tests/test_area_interpolators.py b/tobler/tests/test_area_interpolators.py index 89067c1..f7274c3 100644 --- a/tobler/tests/test_area_interpolators.py +++ b/tobler/tests/test_area_interpolators.py @@ -178,6 +178,8 @@ def test_area_interpolate_parallel(): def test_area_tables_binning(): sac1, sac2 = datasets() + sac1 = sac1.to_crs(4326) + sac2 = sac2.to_crs(4326) auto = _area_tables_binning( source_df=sac1, target_df=sac2, spatial_index="auto" From c4b5920c535be9cbc11b120c4d228e415d608624 Mon Sep 17 00:00:00 2001 From: eli knaap Date: Thu, 15 Feb 2024 15:31:27 -0800 Subject: [PATCH 5/9] fix h3 multipoly test --- tobler/tests/test_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tobler/tests/test_utils.py b/tobler/tests/test_utils.py index c7cb202..c75ab87 100644 --- a/tobler/tests/test_utils.py +++ b/tobler/tests/test_utils.py @@ -52,5 +52,6 @@ def test_h3fy_clip(): @pytest.mark.skipif(platform.system() == "Windows", reason='Unknown precision error on Windows. See #174 for details') def test_h3_multipoly(): va = geopandas.read_file(load_example("virginia").get_path("virginia.shp")) + va = va.to_crs(va.estimate_utm_crs()) va = h3fy(va) - assert_almost_equal(va.to_crs(2284).unary_union.area, 1106844905155.1118, decimal=0) + assert_almost_equal(va.area.sum(), 103195696198.57391, decimal=0) From 940ed14ce9895e2a86cfcf4718fc7a7e15a05ca6 Mon Sep 17 00:00:00 2001 From: eli knaap Date: Thu, 15 Feb 2024 15:41:25 -0800 Subject: [PATCH 6/9] update known total for h3 multipoly --- tobler/tests/test_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tobler/tests/test_utils.py b/tobler/tests/test_utils.py index c75ab87..5fb0199 100644 --- a/tobler/tests/test_utils.py +++ b/tobler/tests/test_utils.py @@ -53,5 +53,6 @@ def test_h3fy_clip(): def test_h3_multipoly(): va = geopandas.read_file(load_example("virginia").get_path("virginia.shp")) va = va.to_crs(va.estimate_utm_crs()) + va = h3fy(va) - assert_almost_equal(va.area.sum(), 103195696198.57391, decimal=0) + assert_almost_equal(va.area.sum(), 102888497504.47836, decimal=0) From 0bc932105ecd90cd3e0e18e426b2a18da32286a4 Mon Sep 17 00:00:00 2001 From: eli knaap Date: Thu, 15 Feb 2024 16:20:07 -0800 Subject: [PATCH 7/9] update ci deps --- .github/workflows/unittests.yml | 9 +-------- ci/311.yml | 1 - ci/312.yml | 1 - 3 files changed, 1 insertion(+), 10 deletions(-) diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml index 93d131d..8c24665 100644 --- a/.github/workflows/unittests.yml +++ b/.github/workflows/unittests.yml @@ -53,14 +53,7 @@ with: environment-file: ${{ matrix.environment-file }} micromamba-version: 'latest' - - ############################################################################## - #- name: install pulp via pip - # run: | - # pip install pulp - # pulptest - ############################################################################## - + - name: environment info run: | micromamba info diff --git a/ci/311.yml b/ci/311.yml index fa7024f..53c8704 100644 --- a/ci/311.yml +++ b/ci/311.yml @@ -29,7 +29,6 @@ dependencies: - joblib - astropy - mapclassify - - descartes - sphinx>=1.4.3 - sphinxcontrib-bibtex==1.0.0 - sphinx_bootstrap_theme diff --git a/ci/312.yml b/ci/312.yml index 327ad05..2782f0b 100644 --- a/ci/312.yml +++ b/ci/312.yml @@ -25,7 +25,6 @@ dependencies: - pip - h3-py - mapclassify - - descartes - sphinx>=1.4.3 - sphinxcontrib-bibtex==1.0.0 - sphinx_bootstrap_theme From 425d6c82f5322434cbdcf769e227e5c05874c182 Mon Sep 17 00:00:00 2001 From: eli knaap Date: Thu, 15 Feb 2024 16:33:35 -0800 Subject: [PATCH 8/9] update h3 clip test --- tobler/tests/test_utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tobler/tests/test_utils.py b/tobler/tests/test_utils.py index b44f8cf..528ae55 100644 --- a/tobler/tests/test_utils.py +++ b/tobler/tests/test_utils.py @@ -4,7 +4,6 @@ import geopandas import pytest -import shapely from libpysal.examples import load_example from numpy.testing import assert_almost_equal @@ -44,9 +43,12 @@ def test_h3fy_diff_crs(): def test_h3fy_clip(): + sac1 = load_example("Sacramento1") + sac1 = geopandas.read_file(sac1.get_path("sacramentot2.shp")) sac_hex = h3fy(sac1, clip=True) + sac_hex = sac_hex.to_crs(sac_hex.estimate_utm_crs()) assert_almost_equal( - sac_hex.to_crs(32710).unary_union.area, 13131736346.537416, decimal=4 + sac_hex.area.sum(), 13131736346.537422, decimal=0 ) From b9a7cb535404a40c2c009894da7832a0f8148f0f Mon Sep 17 00:00:00 2001 From: eli knaap Date: Thu, 15 Feb 2024 16:43:17 -0800 Subject: [PATCH 9/9] reload data on each test --- tobler/tests/test_utils.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tobler/tests/test_utils.py b/tobler/tests/test_utils.py index 528ae55..b5dca26 100644 --- a/tobler/tests/test_utils.py +++ b/tobler/tests/test_utils.py @@ -9,16 +9,17 @@ from tobler.util import h3fy -sac1 = load_example("Sacramento1") -sac1 = geopandas.read_file(sac1.get_path("sacramentot2.shp")) - def test_h3fy(): + sac1 = load_example("Sacramento1") + sac1 = geopandas.read_file(sac1.get_path("sacramentot2.shp")) sac_hex = h3fy(sac1, return_geoms=True) assert sac_hex.shape == (364, 1) def test_h3fy_nogeoms(): + sac1 = load_example("Sacramento1") + sac1 = geopandas.read_file(sac1.get_path("sacramentot2.shp")) sac_hex = h3fy(sac1, return_geoms=False) assert len(sac_hex) == 364