diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..67b5f16 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,44 @@ +name: Bug report +description: Submit a bug report +title: '[BUG]: ' +labels: ['bug'] +assignees: 'bbtufty' + +body: + - type: textarea + id: description + attributes: + label: Describe the Bug + description: A clear and concise description of the bug. + validations: + required: true + - type: textarea + id: expected + attributes: + label: Expected Behavior + description: A clear and concise description of what you expected to happen. + - type: textarea + id: reproduce + attributes: + label: Steps to reproduce the behavior + description: If not present under all circumstances, give a step-by-step on how to reproduce the bug. + value: | + 1. + 2. + ... + - type: textarea + id: screenshots + attributes: + label: Screenshots + description: Attach any applicable screenshots that illustrate your problem. + - type: textarea + id: preferences + attributes: + label: Preference File + description: Paste your config file (likely config.yml), with any sensitive info redacted + render: yaml + - type: textarea + id: log + attributes: + label: Log + description: Attach the relevant log file(s) diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000..0cdce9d --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,33 @@ +name: Feature +description: Suggest a new feature for this project +title: '[FEAT]: ' +labels: ['feature'] +assignees: 'bbtufty' + +body: + - type: textarea + id: problem + attributes: + label: Problem + description: Is your feature request related to a problem? Please describe + placeholder: A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + - type: textarea + id: solution + attributes: + label: Solution + description: Describe the solution you'd like + placeholder: A clear and concise description of what you want to happen. + - type: textarea + id: alternatives + attributes: + label: Alternatives + description: Describe alternatives you've considered + placeholder: A clear and concise description of any alternative solutions or features you've considered. + - type: textarea + id: context + attributes: + label: Context + description: Additional context + placeholder: Add any other context or screenshots about the feature request here. + + diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 0000000..140abb2 --- /dev/null +++ b/.github/workflows/build.yaml @@ -0,0 +1,35 @@ +name: Build + +on: + push: + branches: + - '*' + pull_request: + branches: + - master + +jobs: + build_sdist_and_wheel: + name: Build source distribution + runs-on: ubuntu-latest + strategy: + matrix: + # Versions listed at https://raw.githubusercontent.com/actions/python-versions/main/versions-manifest.json + python-version: [ + "3.11", + "3.12", + ] + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install build + - name: Build package + run: python -m build diff --git a/.gitignore b/.gitignore index 82f9275..7b6caf3 100644 --- a/.gitignore +++ b/.gitignore @@ -159,4 +159,4 @@ cython_debug/ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ +.idea/ diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..7174da0 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,30 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +build: + os: ubuntu-22.04 + tools: + python: "3.11" + +# Build documentation in the docs/ directory with Sphinx +sphinx: + builder: html + configuration: docs/conf.py + fail_on_warning: true + +# Optionally build your docs in additional formats such as PDF and ePub +formats: + - htmlzip + - pdf + +# Optionally set the version of Python and requirements required to build your docs +python: + install: + - method: pip + path: . + extra_requirements: + - docs diff --git a/CHANGES.rst b/CHANGES.rst new file mode 100644 index 0000000..99efa7b --- /dev/null +++ b/CHANGES.rst @@ -0,0 +1,4 @@ +0.0.1 (Unreleased) +================== + +- Initial release \ No newline at end of file diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..ad6d0b0 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,17 @@ +include README.md +include CHANGES.rst +include LICENSE +include pyproject.toml + +recursive-include *.pyx *.c *.pxd +recursive-include docs * +recursive-include licenses * +recursive-include cextern * +recursive-include scripts * + +prune build +prune docs/_build +prune docs/api +prune */__pycache__ + +global-exclude *.pyc *.o \ No newline at end of file diff --git a/README.md b/README.md index 64046f1..5a09be7 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,37 @@ -# nxbrew-watcher -Watch and alert for updates on nxbrew +# NXBrew-watcher + +[![Actions](https://img.shields.io/github/actions/workflow/status/bbtufty/nxbrew-watcher/build.yaml?branch=main&style=flat-square)](https://github.com/bbtufty/nxbrew-watcher/actions) +[![License](https://img.shields.io/badge/license-GNUv3-blue.svg?label=License&style=flat-square)](LICENSE) + +Intro text + +Installation +------------ + +NXBrew-watcher can be installed by cloning the repository and installing via pip: + +```shell +git clone https://github.com/bbtufty/nxbrew-watcher.git +cd nxbrew-watcher +pip install -e . +``` + +Running NXBrew-watcher +---------------------- + +After installing, you can run simply by: + +```python +import os +os.system(r"python nxbrew-watcher\nxbrew_watcher.py") +``` + +Environment variables +--------------------- + +NXBrew-watcher pulls in a number of environment variables that can be configured. These are: + +* `CONFIG_DIR`: Where to save cache and log files to +* `NXBREW_DISCORD_URL`: Webhook URL for Discord to post updates (see [here](https://support.discord.com/hc/en-us/articles/228383668-Intro-to-Webhooks)) +* `NXBREW_CADENCE`: Cadence to perform search on (in minutes). Defaults to 1. +* `NXBREW_LOG_LEVEL`: Level for log files. Defaults to INFO diff --git a/nxbrew_watcher.py b/nxbrew_watcher.py new file mode 100644 index 0000000..4bf076d --- /dev/null +++ b/nxbrew_watcher.py @@ -0,0 +1,4 @@ +from nxbrew_watcher import NXBrewWatcher + +nxbw = NXBrewWatcher() +nxbw.run() diff --git a/nxbrew_watcher/__init__.py b/nxbrew_watcher/__init__.py new file mode 100644 index 0000000..cc84216 --- /dev/null +++ b/nxbrew_watcher/__init__.py @@ -0,0 +1,10 @@ +from importlib.metadata import version + +# Get the version +__version__ = version(__name__) + +from .watcher import NXBrewWatcher + +__all__ = [ + "NXBrewWatcher", +] diff --git a/nxbrew_watcher/utils/__init__.py b/nxbrew_watcher/utils/__init__.py new file mode 100644 index 0000000..6eb71f0 --- /dev/null +++ b/nxbrew_watcher/utils/__init__.py @@ -0,0 +1,10 @@ +from .discord import discord_push +from .io import load_json, save_json +from .logger import setup_logger + +__all__ = [ + "setup_logger", + "load_json", + "save_json", + "discord_push", +] diff --git a/nxbrew_watcher/utils/discord.py b/nxbrew_watcher/utils/discord.py new file mode 100644 index 0000000..7f4e11e --- /dev/null +++ b/nxbrew_watcher/utils/discord.py @@ -0,0 +1,15 @@ +from discordwebhook import Discord + + +def discord_push( + url, + embeds, +): + """Post a message to Discord""" + + discord = Discord(url=url) + discord.post( + embeds=embeds, + ) + + return True diff --git a/nxbrew_watcher/utils/io.py b/nxbrew_watcher/utils/io.py new file mode 100644 index 0000000..c050677 --- /dev/null +++ b/nxbrew_watcher/utils/io.py @@ -0,0 +1,22 @@ +import json + + +def load_json(file): + """Load json file""" + + with open(file, "r", encoding="utf-8") as f: + j = json.load(f) + + return j + + +def save_json(data, out_file): + """Save json in a pretty way""" + + with open(out_file, "w", encoding="utf-8") as f: + json.dump( + data, + f, + ensure_ascii=False, + indent=4, + ) diff --git a/nxbrew_watcher/utils/logger.py b/nxbrew_watcher/utils/logger.py new file mode 100644 index 0000000..e899e32 --- /dev/null +++ b/nxbrew_watcher/utils/logger.py @@ -0,0 +1,109 @@ +import logging +import os +import sys +from logging.handlers import RotatingFileHandler + +import colorlog +from pathvalidate import sanitize_filename + + +def setup_logger( + log_level, + script_name, + log_dir, + additional_dir="", + max_logs=9, +): + """ + Set up the logger. + + Parameters: + log_level (str): The log level to use + script_name (str): The name of the script + log_dir (str): The directory to save logs to + additional_dir (str): Any additional directories to keep log files tidy + max_logs (int): Maximum number of log files to keep + + Returns: + A logger object for logging messages. + """ + + # Sanitize the directories if we need to + additional_dir = [sanitize_filename(f) for f in additional_dir.split(os.path.sep)] + + if os.environ.get("DOCKER_ENV"): + log_dir = os.path.join(log_dir, script_name, *additional_dir) + else: + log_dir = os.path.join(log_dir, script_name, *additional_dir) + + if log_level not in ["DEBUG", "INFO", "CRITICAL"]: + log_level = "INFO" + print(f"Invalid log level '{log_level}', defaulting to 'INFO'") + + # Create the log directory if it doesn't exist + if not os.path.exists(log_dir): + os.makedirs(log_dir) + + # Define the log file path, and sanitize if needs be + log_file = os.path.join(log_dir, f"{script_name}.log") + + # Check if log file already exists + if os.path.isfile(log_file): + for i in range(max_logs - 1, 0, -1): + old_log = f"{log_dir}/{script_name}.log.{i}" + new_log = f"{log_dir}/{script_name}.log.{i + 1}" + if os.path.exists(old_log): + if os.path.exists(new_log): + os.remove(new_log) + os.rename(old_log, new_log) + os.rename(log_file, os.path.join(log_dir, f"{script_name}.log.1")) + + # Create a logger object with the script name + logger = logging.getLogger(script_name) + logger.propagate = False + + # Set the log level based on the provided parameter + log_level = log_level.upper() + if log_level == "DEBUG": + logger.setLevel(logging.DEBUG) + elif log_level == "INFO": + logger.setLevel(logging.INFO) + elif log_level == "CRITICAL": + logger.setLevel(logging.CRITICAL) + else: + logger.critical(f"Invalid log level '{log_level}', defaulting to 'INFO'") + logger.setLevel(logging.INFO) + + # Define the log message format for the log files + logfile_formatter = logging.Formatter( + fmt="%(asctime)s %(levelname)s: %(message)s", datefmt="%m/%d/%y %I:%M %p" + ) + + # Create a RotatingFileHandler for log files + handler = RotatingFileHandler(log_file, delay=True, mode="w", backupCount=max_logs) + handler.setFormatter(logfile_formatter) + + # Add the file handler to the logger + logger.addHandler(handler) + + # Configure console logging with the specified log level + console_handler = colorlog.StreamHandler(sys.stdout) + if log_level == "DEBUG": + console_handler.setLevel(logging.DEBUG) + elif log_level == "INFO": + console_handler.setLevel(logging.INFO) + elif log_level == "CRITICAL": + console_handler.setLevel(logging.CRITICAL) + + # Add the console handler to the logger + console_handler.setFormatter( + colorlog.ColoredFormatter("%(log_color)s%(levelname)s: %(message)s") + ) + logger.addHandler(console_handler) + + # Overwrite previous logger if exists + logging.getLogger(script_name).handlers.clear() + logging.getLogger(script_name).addHandler(handler) + logging.getLogger(script_name).addHandler(console_handler) + + return logger diff --git a/nxbrew_watcher/watcher.py b/nxbrew_watcher/watcher.py new file mode 100644 index 0000000..92e2505 --- /dev/null +++ b/nxbrew_watcher/watcher.py @@ -0,0 +1,201 @@ +import copy +import os +import time + +import emoji +import requests +from bs4 import BeautifulSoup + +from .utils import load_json, save_json, setup_logger, discord_push + +# NXBrew variables +NXBREW_URL = "https://nxbrew.com/" +LATEST_ADDED_ID = "tab-recent-6" +LATEST_UPDATED_ID = "custom_html-6" + +# Pull in environment variables +CONFIG_DIR = os.getenv("CONFIG_DIR", os.getcwd()) + +DISCORD_URL = os.getenv("NXBREW_DISCORD_URL", None) + +CADENCE = os.getenv("NXBREW_CADENCE", "1") +CADENCE = int(CADENCE) * 60 + +LOG_LEVEL = os.getenv("NXBREW_LOG_LEVEL", "INFO") + + +def get_soup(url): + """Scrape URL, soupify + + Args: + url: URL to scrape + """ + + r = requests.get(url) + soup = BeautifulSoup(r.content, "html.parser") + + return soup + + +def get_emoji_free_text(text): + """Remove emoji from a string + + Args: + text: String to remove emoji from + """ + + allchars = [s for s in text] + emoji_list = [c for c in allchars if c in emoji.EMOJI_DATA] + clean_text = ' '.join([s for s in text.split() if s not in emoji_list]) + + return clean_text + + +class NXBrewWatcher: + + def __init__( + self, + ): + """Scrape NXBrew for latest additions and updates""" + + self.cache_file = os.path.join(CONFIG_DIR, "nxbrew_cache.json") + if os.path.exists(self.cache_file): + self.cache = load_json(self.cache_file) + else: + self.cache = {"added": {}, "updated": {}} + + log_dir = os.path.join(CONFIG_DIR, "logs") + logger = setup_logger( + log_level=LOG_LEVEL, + script_name="nxbrew_watcher", + log_dir=log_dir, + ) + self.logger = logger + + def run(self): + """Run the watcher""" + + while True: + # Scrape the website + soup = get_soup(NXBREW_URL) + + # First, scrape and cache latest added + self.scrape_latest_added(soup) + + # Then, scrape latest updates + self.scrape_latest_updated(soup) + + # Save out the cache + save_json(self.cache, self.cache_file) + + # Wait until the next run + self.logger.info(f"Run complete, next run in {CADENCE}s") + time.sleep(CADENCE) + + def scrape_latest_added(self, + soup, + ): + """Look at latest added, and print out any not in the cache + + Args: + soup: BeautifulSoup object + """ + + self.logger.info("Scraping latest added:") + + results = soup.find(id=LATEST_ADDED_ID) + + # Step backwards so the newest is last + for item in results.find_all('li', + )[::-1]: + + # Get title out of the tab-item-title + item_title = item.find("p", + attrs={'class': 'tab-item-title'}, + ) + + # Get thumbnail from tab-item-thumbnail + item_thumb = item.find("div", + attrs={'class': 'tab-item-thumbnail'}, + ) + + # Get title from text + title = copy.deepcopy(item_title.text) + + # # Get href from the "a" tag + url = item_title.find("a").get("href") + + # Get a thumbnail from img + thumb = item_thumb.find("img").get("src") + + if title not in self.cache["added"]: + + self.logger.info(f"-> Found {title}, adding to cache") + self.cache["added"][title] = url + + # Push to discord, if we're doing that + if DISCORD_URL is not None: + embeds = [ + { + "author": { + "name": "Added", + "url": "https://github.com/bbtufty/nxbrew-watcher", + }, + "title": title, + "description": url, + "thumbnail": {"url": thumb}, + } + ] + + discord_push( + url=DISCORD_URL, + embeds=embeds, + ) + + return True + + def scrape_latest_updated(self, + soup, + ): + """Look at latest updated, and print out any not in the cache + + Args: + soup: BeautifulSoup object + """ + + self.logger.info("Scraping latest updated:") + + results = soup.find(id=LATEST_UPDATED_ID) + + # Step backwards so the newest is last + for item in results.find_all('a', + )[::-1]: + + # Clean out the tick emoji and any extraneous whitespace + title = get_emoji_free_text(item.text).strip() + url = item.get("href") + + if title not in self.cache["updated"]: + + self.logger.info(f"-> Found {title}, adding to cache") + self.cache["updated"][title] = url + + # Push to discord, if we're doing that + if DISCORD_URL is not None: + embeds = [ + { + "author": { + "name": "Updated", + "url": "https://github.com/bbtufty/nxbrew-watcher", + }, + "title": title, + "description": url, + } + ] + + discord_push( + url=DISCORD_URL, + embeds=embeds, + ) + + return True diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..92c4a26 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,54 @@ +[project] + +name = "nxbrew-watcher" +version = "0.0.1" +description = "NXBrew watch tools" +readme = "README.md" +requires-python = ">=3.11" +license = {file = "LICENSE"} + +authors = [ + {name = "bbtufty"}, +] +maintainers = [ + {name = "bbtufty"}, +] + +classifiers = [ + # 3 - Alpha + # 4 - Beta + # 5 - Production/Stable + "Development Status :: 3 - Alpha", + + # License + "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)", + + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3 :: Only", +] + +dependencies = [ + "beautifulsoup4>=4.12.3", + "colorlog>=6.8.2", + "discordwebhook>=1.0.3", + "emoji>=2.12.2", + "pathvalidate>=3.2.0", + "requests>=2.31.0", +] + +[project.urls] +"Homepage" = "https://github.com/bbtufty/nxbrew-watcher" +"Bug Reports" = "https://github.com/bbtufty/nxbrew-watcher/issues" +"Source" = "https://github.com/bbtufty/nxbrew-watcher" + +[build-system] +requires = [ + "setuptools>=43.0.0", + "wheel>=0.43.0", + "setuptools_scm>=8.1.0", +] + +build-backend = "setuptools.build_meta" \ No newline at end of file