-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
19 changed files
with
1,268 additions
and
152 deletions.
There are no files selected for viewing
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -2,8 +2,13 @@ | |
|
||
# Type checking | ||
mypy | ||
types-pyyaml | ||
|
||
# Linting | ||
ruff | ||
pre-commit | ||
|
||
# Testing | ||
pytest | ||
pytest-asyncio | ||
pytest-cov | ||
|
Large diffs are not rendered by default.
Oops, something went wrong.
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
from .purger import Purger | ||
|
||
__all__ = ["Purger"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,48 +1,129 @@ | ||
"""Command-line interface for Google Filestore tools.""" | ||
import argparse | ||
import asyncio | ||
import os | ||
"""Command-line interface for purger.""" | ||
|
||
from pathlib import Path | ||
|
||
import click | ||
import yaml | ||
from pydantic import ValidationError | ||
from safir.asyncio import run_with_asyncio | ||
from safir.click import display_help | ||
from safir.logging import LogLevel, Profile | ||
|
||
from .constants import CONFIG_FILE, ENV_PREFIX | ||
from .models.config import Config | ||
from .purger import Purger | ||
from .constants import POLICY_FILE, ENV_PREFIX | ||
|
||
|
||
def _add_options() -> argparse.ArgumentParser: | ||
"""Add options applicable to any filestore tool.""" | ||
parser = argparse.ArgumentParser() | ||
parser.add_argument( | ||
"-f", | ||
"--file", | ||
"--policy-file", | ||
help="Policy file for purger", | ||
default=os.environ.get(f"{ENV_PREFIX}FILE", POLICY_FILE), | ||
type=Path, | ||
required=True, | ||
) | ||
parser.add_argument( | ||
"-x", | ||
"--dry-run", | ||
help="Do not perform actions, but print what would be done", | ||
type=bool, | ||
default=bool(os.environ.get(f"{ENV_PREFIX}DRY_RUN", "")), | ||
) | ||
parser.add_argument( | ||
"-d", | ||
"--debug", | ||
"--verbose", | ||
default=bool(os.environ.get(f"{ENV_PREFIX}DEBUG", "")), | ||
type=bool, | ||
help="Verbose debugging output", | ||
|
||
|
||
@click.group(context_settings={"help_option_names": ["-h", "--help"]}) | ||
@click.version_option(message="%(version)s") | ||
def main() -> None: | ||
"""Command-line interface for purger.""" | ||
|
||
|
||
@main.command() | ||
@click.pass_context | ||
def help(ctx: click.Context, topic: str | None) -> None: | ||
"""Show help for any command.""" | ||
display_help(main, ctx, topic) | ||
|
||
|
||
config_option = click.option( | ||
"-c", | ||
"--config-file", | ||
"--config", | ||
envvar=ENV_PREFIX + "CONFIG_FILE", | ||
type=click.Path(path_type=Path), | ||
help="Purger application configuration file", | ||
) | ||
policy_option = click.option( | ||
"-p", | ||
"--policy-file", | ||
"--policy", | ||
envvar=ENV_PREFIX + "POLICY_FILE", | ||
type=click.Path(path_type=Path), | ||
help="Purger policy file", | ||
) | ||
debug_option = click.option( | ||
"-d", | ||
"--debug", | ||
envvar=ENV_PREFIX + "DEBUG", | ||
type=bool, | ||
help="Enable debug logging", | ||
) | ||
dry_run_option = click.option( | ||
"-x", | ||
"--dry-run", | ||
envvar=ENV_PREFIX + "DRY_RUN", | ||
type=bool, | ||
help="Dry run: take no action, just emit what would be done.", | ||
) | ||
|
||
|
||
def _get_config( | ||
config_file: Path | None = None, | ||
policy_file: Path | None = None, | ||
debug: bool | None = None, | ||
dry_run: bool | None = None, | ||
) -> Config: | ||
try: | ||
if config_file is None: | ||
config_file = CONFIG_FILE | ||
c_obj = yaml.safe_load(config_file.read_text()) | ||
config = Config.model_validate(c_obj) | ||
except (FileNotFoundError, ValidationError): | ||
config = Config() | ||
if policy_file is not None: | ||
config.policy_file = policy_file | ||
if debug is not None: | ||
config.logging.log_level = LogLevel.DEBUG | ||
config.logging.profile = Profile.development | ||
if dry_run is not None: | ||
config.dry_run = dry_run | ||
return config | ||
|
||
|
||
@config_option | ||
@policy_option | ||
@debug_option | ||
@dry_run_option | ||
@run_with_asyncio | ||
async def report( | ||
*, | ||
config_file: Path | None, | ||
policy_file: Path | None, | ||
debug: bool | None, | ||
dry_run: bool | None, | ||
) -> None: | ||
"""Report what would be purged.""" | ||
config = _get_config( | ||
config_file=config_file, | ||
policy_file=policy_file, | ||
debug=debug, | ||
dry_run=dry_run, | ||
) | ||
return parser | ||
|
||
def purge() -> None: | ||
"""Purge the target filesystems.""" | ||
args = _get_options().parse_args() | ||
purger = Purger( | ||
policy_file=args.policy, | ||
dry_run=args.dry_run, | ||
debug=args.debug | ||
purger = Purger(config=config) | ||
await purger.plan() | ||
await purger.report() | ||
|
||
|
||
@config_option | ||
@policy_option | ||
@debug_option | ||
@dry_run_option | ||
async def purge( | ||
*, | ||
config_file: Path | None, | ||
policy_file: Path | None, | ||
debug: bool | None, | ||
dry_run: bool | None, | ||
) -> None: | ||
"""Report what would be purged.""" | ||
config = _get_config( | ||
config_file=config_file, | ||
policy_file=policy_file, | ||
debug=debug, | ||
dry_run=dry_run, | ||
) | ||
asyncio.run(purger.purge()) | ||
purger = Purger(config=config) | ||
await purger.plan() | ||
await purger.purge() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,11 @@ | ||
"""Exceptions for the purger.""" | ||
|
||
from safir.slack.blockkit import SlackException | ||
|
||
|
||
class PlanNotReadyError(SlackException): | ||
"""An operation needing a Plan was requested, but no Plan is ready.""" | ||
|
||
|
||
class PolicyNotFoundError(SlackException): | ||
"""No Policy matching the given directory was found.""" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,18 +1,74 @@ | ||
"""Application configuration for the purger.""" | ||
|
||
from pathlib import Path | ||
from typing import Annotated | ||
|
||
from pydantic import Field, HttpUrl | ||
from safir.logging import LogLevel, Profile | ||
from safir.pydantic import CamelCaseModel | ||
|
||
from pydantic import Field | ||
from ..constants import ENV_PREFIX, POLICY_FILE | ||
|
||
|
||
class LoggingConfig(CamelCaseModel): | ||
"""Configuration for the purger's logs.""" | ||
|
||
profile: Annotated[ | ||
Profile, | ||
Field( | ||
title="Logging profile", | ||
validation_alias=ENV_PREFIX + "LOGGING_PROFILE", | ||
), | ||
] = Profile.production | ||
|
||
log_level: Annotated[ | ||
LogLevel, | ||
Field(title="Log level", validation_alias=ENV_PREFIX + "LOG_LEVEL"), | ||
] = LogLevel.INFO | ||
|
||
add_timestamp: Annotated[ | ||
bool, | ||
Field( | ||
title="Add timestamp to log lines", | ||
validation_alias=ENV_PREFIX + "ADD_TIMESTAMP", | ||
), | ||
] = False | ||
|
||
from typing import Annotated | ||
|
||
class Config(CamelCaseModel): | ||
"""Top-level configuration for the purger.""" | ||
|
||
policy_file: Annotated[ | ||
Path, | ||
Field( | ||
title="Policy file location", | ||
validation_alias=ENV_PREFIX + "POLICY_FILE", | ||
), | ||
] = POLICY_FILE | ||
|
||
policy_file: Annotated[Path, Field(title="Policy file location")] | ||
dry_run: Annotated[ | ||
bool, | ||
Field( | ||
title="Report rather than execute plan", | ||
validation_alias=ENV_PREFIX + "DRY_RUN", | ||
), | ||
] = False | ||
|
||
dry_run: Annotated[bool, Field(title="Report rather than execute plan", | ||
default=False)] | ||
logging: Annotated[ | ||
LoggingConfig, | ||
Field( | ||
title="Logging configuration", | ||
), | ||
] = LoggingConfig() | ||
|
||
debug: Annotated[bool, Field(title="Verbose debugging output", | ||
default=False)] | ||
|
||
alert_hook: Annotated[ | ||
HttpUrl | None, | ||
Field( | ||
title="Slack webhook URL used for sending alerts", | ||
description=( | ||
"An https URL, which should be considered secret." | ||
" If not set or set to `None`, this feature will be disabled." | ||
), | ||
validation_alias=ENV_PREFIX + "ALERT_HOOK", | ||
), | ||
] = None |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,46 @@ | ||
"""Object representing files to be purged, and why.""" | ||
|
||
from enum import StrEnum | ||
from pathlib import Path | ||
from typing import Annotated | ||
|
||
from pydantic import Field | ||
from safir.pydantic import CamelCaseModel | ||
|
||
|
||
class FileClass(StrEnum): | ||
"""Whether a file is large or small.""" | ||
|
||
LARGE = "LARGE" | ||
SMALL = "SMALL" | ||
|
||
|
||
class FileReason(StrEnum): | ||
"""Whether a file is to be purged on access, creation, or modification | ||
time grounds. | ||
""" | ||
|
||
ATIME = "ATIME" | ||
CTIME = "CTIME" | ||
MTIME = "MTIME" | ||
|
||
|
||
class FileRecord(CamelCaseModel): | ||
"""A file to be purged, and why.""" | ||
|
||
path: Annotated[Path, Field(..., title="Path for file to purge.")] | ||
|
||
file_class: Annotated[ | ||
FileClass, Field(..., title="Class of file to purge (large or small).") | ||
] | ||
|
||
file_reason: Annotated[ | ||
FileReason, | ||
Field(..., title="Reason to purge file (access or creation time)."), | ||
] | ||
|
||
|
||
class Plan(CamelCaseModel): | ||
"""List of files to be purged, and why.""" | ||
|
||
files: Annotated[list[FileRecord], Field(..., title="Files to purge")] |
Oops, something went wrong.