Skip to content

Commit

Permalink
0.4
Browse files Browse the repository at this point in the history
  • Loading branch information
cmutel committed Oct 14, 2024
1 parent dcccadf commit 2e74648
Show file tree
Hide file tree
Showing 8 changed files with 4,803 additions and 10 deletions.
5 changes: 5 additions & 0 deletions CHANGES.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

## [Unreleased]

## [0.4] - 2024-10-14

* Fixed bugs with custom registry paths
* Added transformations for Agribalyse and Agrifootprint

## [0.3] - 2024-09-26

* Added lots of new data transformations
Expand Down
17 changes: 9 additions & 8 deletions randonneur_data/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
"Registry",
)

__version__ = "0.3"
__version__ = "0.4"


import json
Expand All @@ -15,13 +15,14 @@
from typing import Any, Optional


DATA_DIR = Path(__file__).parent.resolve() / "data"
DEFAULT_DATA_DIR = Path(__file__).parent.resolve() / "data"
DATA_LABELS = {"create", "replace", "update", "delete", "disaggregate"}


class Registry(MutableMapping):
def __init__(self, filepath: Optional[Path] = None):
self.registry_fp = filepath or DATA_DIR / "registry.json"
self.registry_fp = filepath or DEFAULT_DATA_DIR / "registry.json"
self.data_dir = self.registry_fp.parent

def __load(self) -> dict:
try:
Expand Down Expand Up @@ -70,7 +71,7 @@ def format_file(obj: dict) -> str:
)

def __repr__(self) -> str:
return f"`randonneur_data` registry at {str(self.registry_fp.parent)} with {len(self)} files and id {id(self)}"
return f"`randonneur_data` registry at {str(self.data_dir)} with {len(self)} files and id {id(self)}"

def __delitem__(self, name) -> None:
data = self.__load()
Expand Down Expand Up @@ -109,7 +110,7 @@ def add_file(self, filepath: Path, replace: bool = False) -> Path:

self.validate_file(filepath)

new_path = DATA_DIR / filepath.name
new_path = self.data_dir / filepath.name
if new_path.exists() and not replace:
raise ValueError(f"File {new_path} already exists and `replace` is `False`")

Expand All @@ -121,7 +122,7 @@ def add_file(self, filepath: Path, replace: bool = False) -> Path:
if size > 2e5:
data["filename"] = f"{new_path.stem}.xz"
data["compression"] = "lzma"
new_path = new_path.parent / data["filename"]
new_path = self.data_dir / data["filename"]
with lzma.LZMAFile(
new_path, mode="w", check=lzma.CHECK_SHA256, preset=9
) as lzma_file:
Expand Down Expand Up @@ -163,9 +164,9 @@ def get_file(self, label: str) -> dict:
if metadata.get("compression") == "lzma":
return json.load(
lzma.LZMAFile(
filename=DATA_DIR / metadata["filename"],
filename=self.data_dir / metadata["filename"],
mode="rb",
)
)
else:
return json.load(open(DATA_DIR / metadata["filename"]))
return json.load(open(self.data_dir / metadata["filename"]))
Loading

0 comments on commit 2e74648

Please sign in to comment.