Fix docker docker compose, linters

* Fix the docker-compose.yml setup for localdev.
* Make startup more robust, adding timeouts while the SQL database is not
ready.
* Add PyMardownlnt Markdown linter
* Get faulthandler working early
* Rework logging a bit

.gitignore

@@ -3,6 +3,7 @@
 *.pyc
 *.tmp
 .env*
+.gitkeep
 .venv*
 /*.iml
 /build

.pre-commit-config.yaml

@@ -0,0 +1,18 @@
+---
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: check-yaml
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+  - repo: https://github.com/psf/black
+    rev: 24.10.0
+    hooks:
+      - id: black
+  - repo: https://github.com/jackdewinter/pymarkdown
+    rev: v0.9.26
+    hooks:
+      - id: pymarkdown
+        args:
+          - "-d MD013 scan ."

CODE_OF_CONDUCT.md

@@ -68,9 +68,9 @@ members of the project's leadership.
 ## Attribution
 
 This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
-available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
+available at [https://www.contributor-covenant.org/version/1/4/code-of-conduct.html](https://www.contributor-covenant.org/version/1/4/code-of-conduct.html)
 
 [homepage]: https://www.contributor-covenant.org
 
 For answers to common questions about this code of conduct, see
-https://www.contributor-covenant.org/faq
+[https://www.contributor-covenant.org/faq](https://www.contributor-covenant.org/faq)

bin/fmt.sh

@@ -6,4 +6,3 @@ echo "*** isort ***"
 isort freezing
 echo "*** djlint ***"
 djlint --reformat freezing/web/templates
-

bin/lint.sh

@@ -9,4 +9,3 @@ flake8 freezing
 echo "*** mypy ***"
 echo "*** djlint ***"
 djlint --check freezing/web/templates
-

docker-compose.yml

@@ -14,13 +14,13 @@ services:
       - ./data/mysql/data:/var/lib/mysql
       - ./data/mysql/sql:/sql
     environment:
-      MYSQL_DATABASE: freezing
-      MYSQL_PASSWORD: zeer0
-      MYSQL_ROOT_PASSWORD: fr33z3
-      MYSQL_USER: freezing
+      MYSQL_DATABASE: ${MYSQL_DATABASE:-freezing}
+      MYSQL_PASSWORD: ${MYSQL_PASSWORD:-zeer0}
+      MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD:-fr33z3}
+      MYSQL_ROOT_USER: ${MYSQL_ROOT_USER:-root}
+      MYSQL_USER: ${MYSQL_USER:-freezing}
 
   freezing-web:
-    image: freezingsaddles/freezing-web
     build: ./
     container_name: freezing-web-dev
     depends_on:
@@ -48,8 +48,8 @@ services:
       OBSERVER_TEAMS: ${OBSERVER_TEAMS:-5678,9013}
       REGISTRATION_SITE: ${REGISTRATION_SITE:-https://freezingsaddles.info/}
       SECRET_KEY: ${SECRET_KEY:-e6c07402-0307-11e8-b087-000000000000}  # yamllint disable-line
-      SQLALCHEMY_URL: 'mysql+pymysql://freezing:zeer0@freezing-db-dev/freezing?charset=utf8mb4&binary_prefix=true'  # yamllint disable-line
-      SQLALCHEMY_URL_ROOT: 'mysql+pymysql://root:fr33z3@freezing-db-dev/mysql?charset=utf8mb4&binary_prefix=true'  # yamllint disable-line
+      SQLALCHEMY_URL: "mysql+pymysql://${MYSQL_USER:-freezing}:${MYSQL_PASSWORD:-zeer0}@freezing-db-dev/${MYSQL_DATABASE:-freezing}?charset=utf8mb4&binary_prefix=true"  # yamllint disable-line
+      SQLALCHEMY_URL_ROOT: "mysql+pymysql://${MYSQL_ROOT_USER:-freezing}:${MYSQL_ROOT_PASSWORD:-zeer0}@freezing-db-dev/mysql?charset=utf8mb4&binary_prefix=true"  # yamllint disable-line
       START_DATE: ${START_DATE:-2018-01-01T00:00:00-05:00}
       STRAVA_CLIENT_ID: ${STRAVA_CLIENT_ID:-?}
       STRAVA_CLIENT_SECRET: ${STRAVA_CLIENT_SECRET:-?}

example.cfg

@@ -8,19 +8,32 @@ BIND_INTERFACE=127.0.0.1
 
 DEBUG=true
 
+# By default this uses the localdev environment for local development, which enables
+# several features useful for developers, such as user impersonation.
+# See
+#ENVIRONMENT=localdev
+
 # The SECRET_KEY is used by Flask to sign sessions. Set this to something else.
 SECRET_KEY=e6c07402-0307-11e8-b087-000000000000
 
 # The URL to the database.  Note that the pymysql driver must be explicitly specified.
+#
+# It is important to use the utf8mb4 charset to get full support for Unicode characters,
+# including emoji.
+#
 # This URL is suitable for use with the database running in the freezing-compose setup in this repository.
 SQLALCHEMY_URL="mysql+pymysql://freezing:zeer0mfreezing-db-dev/freezing?charset=utf8mb4&binary_prefix=true"
 # Use this one if you are running this on your host vs. the database in the
 # https://github.com/freezingsaddles/freezing-compose project.
+# NOTE: If you are using a MySQL docker via freezing-compose use 127.0.0.1 as the host, NOT localhost
 #SQLALCHEMY_URL="mysql+pymysql://freezing:[email protected]/freezing?charset=utf8mb4&binary_prefix=true"
 # If you keep your MySQL database somewhere else, fix this up to match.
 #SQLALCHEMY_URL="mysql+pymysql://freezing:[email protected]/freezing?charset=utf8mb4&binary_prefix=true""
 
 # Configuration for the Strava client.  These settings come from your App setup.
+# Setting this is only required if you are testing parts of this application that exercise the Strava API,
+# such as user registration. That is an advanced topic and not required to make most changes to
+# the web site. Most of the action with the Strava API happens in freezing-sync, not here.
 STRAVA_CLIENT_ID=?
 STRAVA_CLIENT_SECRET=?
 
@@ -32,7 +45,7 @@ TEAMS=1234,1235
 OBSERVER_TEAMS=5678,9013
 
 # The competition title
-COMPETITION_TITLE='BikeArlington Freezing Saddles 2019'
+COMPETITION_TITLE='BikeArlington Freezing Saddles 2018 - localdev'
 
 # The start date of the competition -- WITH TIME ZONE
 START_DATE=2018-01-01T00:00:00-05:00

freezing/web/__init__.py

@@ -1,50 +1,74 @@
-import faulthandler
-import signal
+"""
+This is the main entry point for the freezing-web application.
+
+It sets up the Flask app and initializes the database and logging.
+It also sets up a fault handler with a signal early to ensure stack traces happen.
+"""
+
+import freezing.web._faulthandler  # noqa isort: skip
+
+from socket import gethostbyname
+from time import sleep
+from urllib.parse import urlparse
 
 from flask import Flask, g, session
 from freezing.model import init_model, meta
 
-faulthandler.register(signal.SIGUSR1)
-
-from .config import config  # noqa
-
-# Thanks https://stackoverflow.com/a/17073583
-app = Flask(
-    __name__,
-    static_folder="static",
-    static_url_path="/",
-    instance_path=config.INSTANCE_PATH,
-)
-app.config.from_object(config)
-
-init_model(config.SQLALCHEMY_URL)
-
-# This needs to be after the app is created, unfortunately.
-from freezing.web.views import (  # noqa
-    alt_scoring,
-    api,
-    chartdata,
-    general,
-    leaderboard,
-    people,
-    photos,
-    pointless,
-    tribes,
-    user,
-)
-
-# Register our blueprints
-
-app.register_blueprint(general.blueprint)
-app.register_blueprint(leaderboard.blueprint, url_prefix="/leaderboard")
-app.register_blueprint(alt_scoring.blueprint, url_prefix="/alt_scoring")
-app.register_blueprint(chartdata.blueprint, url_prefix="/chartdata")
-app.register_blueprint(people.blueprint, url_prefix="/people")
-app.register_blueprint(pointless.blueprint, url_prefix="/pointless")
-app.register_blueprint(photos.blueprint, url_prefix="/photos")
-app.register_blueprint(user.blueprint, url_prefix="/my")
-app.register_blueprint(api.blueprint, url_prefix="/api")
-app.register_blueprint(tribes.blueprint, url_prefix="/tribes")
+from freezing.web.autolog import log
+
+from .config import config, init_logging
+
+init_logging(color=config.DEBUG is False)
+
+
+_BANNER = "*************************"
+
+
+def _get_app():
+    log.info(
+        f"{_BANNER} Configuring Flask app with instance_path={config.INSTANCE_PATH} {_BANNER}"
+    )
+    # Thanks https://stackoverflow.com/a/17073583
+    app = Flask(
+        __name__,
+        static_folder="static",
+        static_url_path="/",
+        instance_path=config.INSTANCE_PATH,
+    )
+    app.config.from_object(config)
+    return app
+
+
+def _register_blueprints(app):
+    # This needs to be after the app is created, unfortunately.
+    from freezing.web.views import (
+        alt_scoring,
+        api,
+        chartdata,
+        general,
+        leaderboard,
+        people,
+        photos,
+        pointless,
+        tribes,
+        user,
+    )
+
+    app.register_blueprint(general.blueprint)
+    app.register_blueprint(leaderboard.blueprint, url_prefix="/leaderboard")
+    app.register_blueprint(alt_scoring.blueprint, url_prefix="/alt_scoring")
+    app.register_blueprint(chartdata.blueprint, url_prefix="/chartdata")
+    app.register_blueprint(people.blueprint, url_prefix="/people")
+    app.register_blueprint(pointless.blueprint, url_prefix="/pointless")
+    app.register_blueprint(photos.blueprint, url_prefix="/photos")
+    app.register_blueprint(user.blueprint, url_prefix="/my")
+    app.register_blueprint(api.blueprint, url_prefix="/api")
+    app.register_blueprint(tribes.blueprint, url_prefix="/tribes")
+
+
+# This has to be done before we define the functions with @app decorators
+app = _get_app()
+_register_blueprints(app)
 
 
 @app.before_request
@@ -74,3 +98,33 @@ def inject_config():
         "forum_site": config.FORUM_SITE,
         "version_string": config.VERSION_STRING,
     }
+
+
+def init_db():
+    """
+    Initialize the database. If the database is not available, keep trying for a bit.
+    """
+    TRIES = 6
+    delay = 2
+    for x in range(1, TRIES + 1):
+        try:
+            # Use urllib to extract the host and test whether it can be resolved
+            url = urlparse(config.SQLALCHEMY_URL)
+            host = gethostbyname(url.hostname)
+            log.debug(f"gethostbyname({url.hostname})=={host}")
+            log.info(f"{_BANNER} Connecting to database on {url.hostname} {_BANNER}")
+            init_model(config.SQLALCHEMY_URL)
+            break
+        except Exception as ex:
+            if x == TRIES:
+                raise ex from None
+            log.warning(
+                f"Failed to connect to database, retrying in {delay}s ({x}) - error was {str(ex)}"
+            )
+            delay = delay * 2
+            sleep(delay)
+
+
+init_db()
+
+log.info(f"{_BANNER} freezing-web initialized in {config.ENVIRONMENT} mode {_BANNER}")

freezing/web/_faulthandler.py

@@ -0,0 +1,6 @@
+import faulthandler
+import signal
+
+# Register this super early so we can get a stack trace if
+# there is a problem in initializing the config module
+faulthandler.register(signal.SIGUSR1)

freezing/web/config.py

@@ -26,26 +26,26 @@ class Config:
     Refactored with the help of GitHub Copilot.
     """
 
-    BEANSTALKD_HOST = env("BEANSTALKD_HOST", default="localhost")
-    BEANSTALKD_PORT = env("BEANSTALKD_PORT", cast=int, default=11300)
+    BEANSTALKD_HOST: str = env("BEANSTALKD_HOST", default="localhost")
+    BEANSTALKD_PORT: str = env("BEANSTALKD_PORT", cast=int, default=11300)
     BIND_INTERFACE: str = env("BIND_INTERFACE", default="127.0.0.1")
     COMPETITION_TEAMS: List[int] = env("TEAMS", cast=list, subcast=int, default=[])
-    COMPETITION_TITLE = env("COMPETITION_TITLE", default="Freezing Saddles")
+    COMPETITION_TITLE: str = env("COMPETITION_TITLE", default="Freezing Saddles")
     DEBUG: bool = env("DEBUG", cast=bool, default=False)
     END_DATE: datetime = env(
         "END_DATE", postprocessor=lambda val: arrow.get(val).datetime
     )
     # Environment (localdev, production, etc.)
-    ENVIRONMENT = env("ENVIRONMENT", default="localdev")
+    ENVIRONMENT: str = env("ENVIRONMENT", default="localdev")
     FORUM_SITE: str = env(
         "FORUM_SITE",
         "https://www.bikearlingtonforum.com/forums/forum/freezing-saddles-winter-riding-competition/",
     )
-    INSTANCE_PATH = env(
+    INSTANCE_PATH: str = env(
         "INSTANCE_PATH", default=os.path.join(_basedir, "data/instance")
     )
     # Directory to store leaderboard data
-    LEADERBOARDS_DIR = env(
+    LEADERBOARDS_DIR: str = env(
         "LEADERBOARDS_DIR", default=os.path.join(_basedir, "leaderboards")
     )
     MAIN_TEAM: int = env("MAIN_TEAM", cast=int)
@@ -54,7 +54,8 @@ class Config:
     )
     REGISTRATION_SITE: str = env("REGISTRATION_SITE", "https://freezingsaddles.info/")
     SECRET_KEY = env("SECRET_KEY")
-    SQLALCHEMY_URL = env("SQLALCHEMY_URL")
+    SQLALCHEMY_URL: str = env("SQLALCHEMY_URL")
+    SQLALCHEMY_ROOT_URL: str = env("SQLALCHEMY_ROOT_URL", None)
     START_DATE: datetime = env(
         "START_DATE", postprocessor=lambda val: arrow.get(val).datetime
     )
@@ -65,8 +66,8 @@ class Config:
         default="America/New_York",
         postprocessor=lambda val: pytz.timezone(val),
     )
-    VERSION_NUM = version("freezing-web")
-    VERSION_STRING = f"{VERSION_NUM}+{branch}.{commit}.{build_date}"
+    VERSION_NUM: str = version("freezing-web")
+    VERSION_STRING: str = f"{VERSION_NUM}+{branch}.{commit}.{build_date}"
 
 
 config = Config()

freezing/web/runserver.py

@@ -1,9 +1,8 @@
 from freezing.web import app
-from freezing.web.config import config, init_logging
+from freezing.web.config import config
 
 
 def main():
-    init_logging(color=True)
     app.run(host=config.BIND_INTERFACE, debug=True)
 
 

requirements-test.txt

@@ -4,3 +4,4 @@ djlint==1.36.3
 flake8==7.1.1
 isort==5.13.2
 pur==7.3.2
+pymarkdownlnt==0.9.26