From 9f0ee790603f50678759e589026d0e92993cf16b Mon Sep 17 00:00:00 2001 From: Richard Bullington-McGuire Date: Mon, 30 Dec 2024 15:39:35 -0500 Subject: [PATCH] Fix docker docker compose, linters * Fix the docker-compose.yml setup for localdev. * Make startup more robust, adding timeouts while the SQL database is not ready. * Add PyMardownlnt Markdown linter * Get faulthandler working early * Rework logging a bit --- .gitignore | 1 + .pre-commit-config.yaml | 18 ++++ CODE_OF_CONDUCT.md | 4 +- README.md | 191 +++++++++++++++++++--------------- bin/fmt.sh | 1 - bin/lint.sh | 1 - docker-compose.yml | 14 +-- example.cfg | 15 ++- freezing/web/__init__.py | 140 +++++++++++++++++-------- freezing/web/_faulthandler.py | 6 ++ freezing/web/config.py | 19 ++-- freezing/web/runserver.py | 3 +- requirements-test.txt | 1 + 13 files changed, 264 insertions(+), 150 deletions(-) create mode 100644 .pre-commit-config.yaml create mode 100644 freezing/web/_faulthandler.py diff --git a/.gitignore b/.gitignore index fc0a696e..56ec75a3 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ *.pyc *.tmp .env* +.gitkeep .venv* /*.iml /build diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..3b337d7c --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,18 @@ +--- +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace + - repo: https://github.com/psf/black + rev: 24.10.0 + hooks: + - id: black + - repo: https://github.com/jackdewinter/pymarkdown + rev: v0.9.26 + hooks: + - id: pymarkdown + args: + - "-d MD013 scan ." diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 54e4c9f1..95db269c 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -68,9 +68,9 @@ members of the project's leadership. ## Attribution This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, -available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html +available at [https://www.contributor-covenant.org/version/1/4/code-of-conduct.html](https://www.contributor-covenant.org/version/1/4/code-of-conduct.html) [homepage]: https://www.contributor-covenant.org For answers to common questions about this code of conduct, see -https://www.contributor-covenant.org/faq +[https://www.contributor-covenant.org/faq](https://www.contributor-covenant.org/faq) diff --git a/README.md b/README.md index 010a8744..f8fb6953 100644 --- a/README.md +++ b/README.md @@ -3,14 +3,15 @@ This is the web component for the Freezing Saddles (aka BikeArlington Freezing Saddles, "BAFS") Strava-based winter cycling competition software. **NOTE:** This application conists of multiple components that work together (designed to run as Docker containers). -1. [freezing-web](https://github.com/freezingsaddles/freezing-web) - The website for viewing leaderboards + +1. [freezing-web](https://github.com/freezingsaddles/freezing-web) - The website for viewing leaderboards 1. [freezing-model](https://github.com/freezingsaddles/freezing-model) - A library of shared database and messaging classes. 1. [freezing-sync](https://github.com/freezingsaddles/freezing-sync) - The component that syncs ride data from Strava. 1. [freezing-nq](https://github.com/freezingsaddles/freezing-nq) - The component that receives webhooks and queues them up for syncing. -# Development Setup +## Development setup -## Dependencies +### Dependencies * [Python 3.10+](https://www.python.org/downloads/release/python-3100/) * [pip](https://pypi.org/project/pip/) @@ -19,27 +20,53 @@ This is the web component for the Freezing Saddles (aka BikeArlington Freezing S We recommend that for ease of development and debugging, that you install Python 3.10 and pip directly on your workstation. This is tested to work on macOS Sonoma 14.1.2 (23B92) & Sequoia 15.1.1 (24B91), on multiple Linux distributions, and on Windows 10 or 11. While this will work on Windows, most of the advice below relates to running this on a UNIX-like operating system, such as macOS or Ubuntu. Pull requests to improve cross-platform documentation are welcome. +### Optional Dependencies + +We *strongly recommend* that you also install [Docker](https://www.docker.com/) to run the database and other services in containers. This will make it easier to set up the database and other services, and to run the application in a production-like environment. + ## Installation -Here are some instructions for setting up a development environment: +Here are some instructions for setting up a development environment, using Python virtual environments: (If you are running in Windows, run `env/Scripts/activate` instead of `source env/bin/activate`.) -```bash -# Clone repo -shell$ mkdir freezingsaddles && cd freezingsaddles -shell$ for part in sync web compose nq model; do git clone https://github.com/freezingsaddles/freezing-$part.git; done -# Create and activate a virtual environment for freezing-web +```bash +shell$ git clone https://github.com/freezingsaddles/freezing-web shell$ cd freezing-web shell$ python3 -m venv env shell$ source env/bin/activate -(env) shell$ pip install -r requirements.txt -(env) shell$ python setup.py develop +(env) shell$ pip install -r requirements.txt -r requirements-dev.txt -e . ``` -We will assume for all subsequent shell examples that you are running in the freezing-web activated virtualenv. (This is denoted by using the "(env) shell$" prefix before shell commands.) +We will assume for all subsequent shell examples that you are running in the freezing-web activated virtualenv. (This is denoted by using the "(env) shell$" prefix before shell commands.) Activate the virtualenv when you open a new shell with `source env/bin/activate`. + +At this point, you should have all the dependencies installed. -### Database Setup +Next, you need to ensure you have a working MySQL database. + +### Quick development setup using Docker + +This project has a built-in Docker Compose file that can be used to set up a MySQL database for development, _and_ a test web application. This is the easiest way to get started. + +```bash +(env) shell$ docker-compose up -d +``` + +It will take about 15-30 seconds to start the first time. You can see the logs with the command `docker-compose logs -f`. + +When it starts up, you can view the Docker version of the Freezing saddles app at [http://127.0.0.1:8000/](http://127.0.0.1:8000/) - it will have an empty database that has been initialized with the schema. + +With this Docker Composed setup, the app does not automatically reload when you make changes to the source code, but you can rebuild and redploy it with this command: + +```bash +(env) shell$ docker-compose up -d --build && docker-compose logs -f +``` + +If you want to stop the containers, you can do so with `docker-compose down`. + +It's possible to use both the containerized `freezing-server` application _and_ the local development server at the same time, they use different ports. + +### Database setup This application requires MySQL, for historical reasons. @hozn wrote: @@ -47,55 +74,62 @@ This application requires MySQL, for historical reasons. @hozn wrote: These days, @obscurerichard hosts the production site on AWS, where we have a choice of many more databases, but since it started as MySQL it will probably stay as MySQL unless there's a really good reason to move. Perhaps PostgreSQL and its geospacial integration would be a better choice in the long run. Also, [Amazon Aurora](https://aws.amazon.com/rds/aurora/) is really slick for MySQL-compatible datbase engines, so we are sticking with MySQL for now. -#### Database Setup using Docker +#### Alternative: using the freezing-compose orchestrated MySQL Database -We have some development support Docker Compose files that can help make database setup simpler, head over to the [freezing-compose](https://github.com/freezingsaddles/freezing-compose) repo for those instructions. +You could _instead_ use the MySQL server defined in [freezing-compose](https://github.com/freezingsaddles/freezing-compose) via `docker-compose-dev` as the MySQL database, but you only need to do that if you are testing out container orchestration in a development environment. -#### Manual Database Setup +#### Alternative: manual database setup -Install MySQL, version 8.0 or newer. The current production server for https://freezingsaddles.org/ runs MySQL 8.0. +Install MySQL, version 8.0 or newer. The current production server for [Freezing Saddles](https://freezingsaddles.org/) runs MySQL 8.0. You should create a database and create a user that can access the database. Something like this might work in the default case: ```bash -shell$ mysql -uroot +(env) shell$ mysql -uroot -p --host 127.0.0.1 +# Enter your MySQL root password mysql> create database freezing; mysql> create user freezing@localhost identified by 'REDACTED'; -mysql> grant all on freezing.* to freezing@localhost; +mysql> grant all on freezing.* to freezing@127.0.0.1; ``` -## Configure and Run Server +### Configuring and running freezing-web for local development + +This is designed to work with configuration files that are shell environment files. You can also use environment variables directly when running the server. -Configuration files are shell environment files (or you can use environment variables dirctly). +There is a sample file (`example.cfg`) that you can reference. You need to set an environment variable called `APP_SETTINGS` to the path to the file you wish to use when you start `freezing-server`. -There is a sample file (`example.cfg`) that you can reference. You need to set an environment variable called `APP_SETTINGS` to the path to the file you wish to use. +Edit the file to change the value of `SECRET_KEY`, and set competition dates. Good date ranges are either the range for a prior year's competition, for testing with an archived database dump, or a 3 month range that includes the current date, for testing in conjunction with fresh data downloaded with [freezing-sync])https://github.com/freezingsaddles/freezing-sync) + +This component is designed to run as a container and should be configured with environment variables for: +* `DEBUG`: Whether to display exception stack traces, etc. +* `SECRET_KEY`: Used to cryptographically sign the Flask session cookies. +* `SQLALCHEMY_URL`: The URL to the database. +* `STRAVA_CLIENT_ID`: The ID of the Strava application. +* `STRAVA_CLIENT_SECRET`: Secret key for the app (available from App settings page in Strava) +* `TEAMS`: A comma-separated list of team (Strava club) IDs for the competition. = env('TEAMS', cast=list, subcast=int, default=[]) +* `OBSERVER_TEAMS`: Comma-separated list of any teams that are just observing, not playing (they can get their overall stats included, but won't be part of leaderboards) +* `START_DATE`: The beginning of the competition. +* `END_DATE`: The end of the competition. + +Changing _all_ these values is not necessary for a basic development setup. However, you should ensure these items are set appropriately: +* The team IDs for the competition, `MAIN_TEAM`, `TEAMS` and any `OBSERVER_TEAMS`, if you are loading an archived database. +* `SQLALCHEMY_URL` database credentials, if you are are using something other than the default Docker Compose setup. +* Strava client API credentials, if you want to test athlete registration and authorization. (You don't have to do this to test user pages, see the [impersonation feature](https://github.com/freezingsaddles/freezing-web/pull/332)) +* The start and end dates for the competition. + +Here is an example of editing the file and starting the webserver using settings from a new `development.cfg` config file: -Here is an example of starting the webserver using settings from a new `development.cfg` config file: ```bash (env) shell$ cp example.cfg development.cfg -# Edit the file +(env) shell$ nano development.cfg # Edit the file to set the SECRET_KEY and other settings (env) shell$ APP_SETTINGS=development.cfg freezing-server ``` -Critical things to set include: -* Database URI -* Strava Client info (ID and secret), if you want to test registration/authorization/login. - -```bash -# The SQLALchemy connection URL for your MySQL database. -# NOTE THE CHARSET! -# NOTE: If you are using docker use 127.0.0.1 as the host, NOT localhost -SQLALCHEMY_URL=mysql+pymysql://freezing@localhost/freezing?charset=utf8mb4&binary_prefix=true - -# These are issued when you create a Strava application. -# These are really only needed if you want to test app authorization or login features. -STRAVA_CLIENT_ID=xxxx1234 -STRAVA_CLIENT_SECRET=5678zzzz -``` +Doing this will start the server on port 5000. You can access the site at [http://localhost:5000/](http://localhost:5000/) and if you make changes to the code and save the file, the server will automatically restart. ### Development setup to work with `freezing-model` -During development, you may find you need to make changes to the database. Because this suite of projects uses SQLAlchemy and Alembic, and multiple projects depend on the model, it is in a [separate git repo](https://github.com/freezingsaddles/freezing-model). +During development, you may find you need to make changes to the database. Because this suite of projects uses SQLAlchemy and Alembic, and multiple projects depend on the model, it is in a [separate git repo](https://github.com/freezingsaddles/freezing-model). This an easy pattern to use to make changes to the project `freezing-model` that this depends on, without having to push tags to the repository. Assuming you have the project checked out in a directory called `workspace` below your home directory, try this: @@ -111,40 +145,29 @@ Now freezing-model is symlinked in, so you can make changes and add migrations t To get `freezing-web` to permanently use the `freezing-model` changes you will have to tag the `freezing-model` repository with a new version number (don't forget to update `setup.py` also) and update the tag in [freezing-web/requirements.txt](requirements.txt) to match the tag number. It's ok to make a pull request in `freezing-model` and bump the version after merging `master` into your branch. - ### Coding standards -The `freezing-web` code is intended to be [PEP-8](https://www.python.org/dev/peps/pep-0008/) compliant. Code formatting is done with [black](https://black.readthedocs.io/en/stable/), [isort](https://pycqa.github.io/isort/) and [djlint}(https://www.djlint.com/) and can be linted with [flake8](http://flake8.pycqa.org/en/latest/). See the [.flake8](.flake8) file and install the test dependencies to get these tools (`pip install -r test-requirements.txt`). +The `freezing-web` code is intended to be [PEP-8](https://www.python.org/dev/peps/pep-0008/) compliant. Code formatting is done with [black](https://black.readthedocs.io/en/stable/), [isort](https://pycqa.github.io/isort/) and [djlint](https://www.djlint.com/) and can be linted with [flake8](http://flake8.pycqa.org/en/latest/). See the [.flake8](.flake8) file and install the test dependencies to get these tools (`pip install -r test-requirements.txt`). To run _all_ the linters and formatters, use the following commands: -``` + +```bash bin/lint.sh bin/fmt.sh ``` -## Docker Deployment +## Production deployment -See [freezing-compose](https://github.com/freezingsaddles/freezing-compose) for guide to deploying this in production along +See [freezing-compose](https://github.com/freezingsaddles/freezing-compose) for a guide to deploying this in production along with the related containers. -This component is designed to run as a container and should be configured with environment variables for: -- `DEBUG`: Whether to display exception stack traces, etc. -- `SECRET_KEY`: Used to cryptographically sign the Flask session cookies. -- `SQLALCHEMY_URL`: The URL to the database. -- `STRAVA_CLIENT_ID`: The ID of the Strava application. -- `STRAVA_CLIENT_SECRET`: Secret key for the app (available from App settings page in Strava) -- `TEAMS`: A comma-separated list of team (Strava club) IDs for the competition. = env('TEAMS', cast=list, subcast=int, default=[]) -- `OBSERVER_TEAMS`: Comma-separated list of any teams that are just observing, not playing (they can get their overall stats included, but won't be part of leaderboards) -- `START_DATE`: The beginning of the competition. -- `END_DATE`: The end of the competition. - -## Beginning of year procedures +### Beginning of year procedures * Ensure that someone creates a new Strava main group. Usually the person running the sign-up process does this. [Search for "Freezing"](https://www.strava.com/clubs/search?utf8=%E2%9C%93&text=freezing&location=&%5Bcountry%5D=&%5Bstate%5D=&%5Bcity%5D=&%5Blat_lng%5D=&sport_type=cycling&club_type=all) and you may be surprised to see it has already been created! * Get the numeric club ID from the URL of the Strava _Recent Activity_ page for the club. * Gain access to the production server via SSH * Ensure you have MySQL client access to the production database, either through SSH port forwarding or by running a MySQL client through docker on the production server, or some other means. -* Make a backup of the database: +* Make a backup of the database: ```bash mkdir -p ~/backups @@ -152,7 +175,7 @@ time mysqldump > $HOME/backups/freezing-$(date +'%Y-%m-%d').sql ``` * Make a backup of the `.env` file from `/opt/compose/.env`: - + ```bash cd /opt/compose cp .env $HOME/backups/.env-$(date +'%Y-%m-%d') @@ -182,15 +205,15 @@ vim /opt/compose/.env insert into teams values (567288, 'Freezing Saddles 2020', 1); -* Restart the services: +* Restart the services: - cd /opt/compose && docker-compose up -d + cd /opt/compose && docker compose up -d * Once the teams are announced (for the original Freezing Saddles competition, typically at the Happy Hour in early January): * Add the team IDs for the competition teams and any observer teams (ringer teams) into the production `.env` file * Restart the services: - cd /opt/compose && docker-compose up -d + cd /opt/compose && docker compose up -d Athletes will get assigned to their correct teams as soon as they join exactly one of the defined competition teams. @@ -204,20 +227,20 @@ It would be a good idea to first drop the database, then recreate it along with You may have to edit the resulting SQL dump to redo the SQL SECURITY DEFINER clauses. The examples below do not have the real production root user name in them, observe the error messages from the production dump restoration to get the user name you will need (or ask @obscurerichard in Slack). -``` +```SQL /*!50013 DEFINER=`mysql-admin-user`@`%` SQL SECURITY DEFINER */ ``` In this case you could edit the SQL dump to fix up the root user expressions: -``` +```bash # Thanks https://stackoverflow.com/a/23584470/424301 LC_ALL=C sed -i.bak 's/mysql-admin-user/root/g' freezing-2023-11-20.sql ``` Here is a lightly redacted transcript of a MySQL interactive session, run on a local dev environment, demonstrating how to prepare for restoring a dump: -``` +```bash $ docker run -it --rm --network=host mysql:5.7 mysql --host=127.0.0.1 --port=3306 --user=root --password=REDACTED mysql: [Warning] Using a password on the command line interface can be insecure. Welcome to the MySQL monitor. Commands end with ; or \g. @@ -256,13 +279,13 @@ $ LC_ALL=C sed -i.bak 's/mysql-admin-user/root/g' freezing-2023-11-20.sql $ time docker run -i --rm --network=host mysql:5.7 mysql --host=127.0.0.1 --port=3306 --user=root --password=REDACTED --database=freezing --default-character-set=utf8mb4 < freezing-2023-11-20.sql mysql: [Warning] Using a password on the command line interface can be insecure. -real 0m43.612s -user 0m0.510s -sys 0m0.994s +real 0m43.612s +user 0m0.510s +sys 0m0.994s $ ``` -# Scoring system +## Scoring system The Freezing Saddles scoring system has evolved over the years to encourage every day riding for those particpating. The scoring system heavily weights the early miles of each ride. You get these points for riding outdoors, no indoor trainer rides count: @@ -270,11 +293,11 @@ The Freezing Saddles scoring system has evolved over the years to encourage ever • Additional mileage points as follows: Mile 1=10 points; Mile 2=9 pts; Mile 3=8 pts, etc. Miles 10 and over = 1 pt each. • There is no weekly point cap or distinction between individual and team points. Ride your hearts out! - -## Scoring Cheat Sheet for the Mathematically Challenged +### Scoring Cheat Sheet for the Mathematically Challenged Here is a cumulative list of the points you get for riding up to 20 miles per day: -``` + +```plaintext Miles = Points 1 = 20 2 = 29 @@ -300,9 +323,9 @@ Miles = Points The scores are rounded to the nearest integer point for display, but the system uses precise floating point calculations of points to determine rank. This can lead to some counterintuitive results at first glance, such as a whole-number points tie with the person in the lead having fewer miles recorded. -In 2024, this happened as of Jan 7 between Paul Wilson and Steve Szibler. Check out this detail - -``` +In 2024, this happened as of Jan 7 between Paul Wilson and Steve Szibler. Check out this detail + +```sql mysql> select a.name, ds.distance, ds.points, ds.ride_date from daily_scores ds inner join athletes a on (ds.athlete_id = a.id) where a.name like 'Steve S%' or name like 'Paul Wilson' order by name, ride_date; +-------------------+--------------------+--------------------+------------+ | name | distance | points | ride_date | @@ -334,16 +357,16 @@ mysql> select a.name, sum(ds.distance), sum(ds.points) from daily_scores ds inne 2 rows in set (0.02 sec) ``` -# Legal +## Legal This software is a community-driven effort, and as such the contributions are owned by the individual contributors: -Copyright 2015 Ian Will
-Copyright 2019 Hans Lillelid
-Copyright 2020 Jon Renaut
-Copyright 2020 Merlin Hughes
-Copyright 2020 Richard Bullington-McGuire
-Copyright 2020 Adrian Porter
-Copyright 2020 Joe Tatsuko
+* Copyright 2015 Ian Will +* Copyright 2019 Hans Lillelid +* Copyright 2020 Jon Renaut +* Copyright 2020 Merlin Hughes +* Copyright 2020 Richard Bullington-McGuire +* Copyright 2020 Adrian Porter +* Copyright 2020 Joe Tatsuko -This software is licensed under the [Apache 2.0 license](LICENSE), with some marked portions available under compatible licenses (such as the [MIT-licensed `test/wget-spider.sh`].) +This software is licensed under the [Apache 2.0 license](LICENSE), with some marked portions available under compatible licenses (such as the [MIT-licensed `test/wget-spider.sh`].) diff --git a/bin/fmt.sh b/bin/fmt.sh index ddf8b3f9..f04a1496 100755 --- a/bin/fmt.sh +++ b/bin/fmt.sh @@ -6,4 +6,3 @@ echo "*** isort ***" isort freezing echo "*** djlint ***" djlint --reformat freezing/web/templates - diff --git a/bin/lint.sh b/bin/lint.sh index 0fa0fb7f..36ce2ef9 100755 --- a/bin/lint.sh +++ b/bin/lint.sh @@ -9,4 +9,3 @@ flake8 freezing echo "*** mypy ***" echo "*** djlint ***" djlint --check freezing/web/templates - diff --git a/docker-compose.yml b/docker-compose.yml index d0be5719..83610f6f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,13 +14,13 @@ services: - ./data/mysql/data:/var/lib/mysql - ./data/mysql/sql:/sql environment: - MYSQL_DATABASE: freezing - MYSQL_PASSWORD: zeer0 - MYSQL_ROOT_PASSWORD: fr33z3 - MYSQL_USER: freezing + MYSQL_DATABASE: ${MYSQL_DATABASE:-freezing} + MYSQL_PASSWORD: ${MYSQL_PASSWORD:-zeer0} + MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD:-fr33z3} + MYSQL_ROOT_USER: ${MYSQL_ROOT_USER:-root} + MYSQL_USER: ${MYSQL_USER:-freezing} freezing-web: - image: freezingsaddles/freezing-web build: ./ container_name: freezing-web-dev depends_on: @@ -48,8 +48,8 @@ services: OBSERVER_TEAMS: ${OBSERVER_TEAMS:-5678,9013} REGISTRATION_SITE: ${REGISTRATION_SITE:-https://freezingsaddles.info/} SECRET_KEY: ${SECRET_KEY:-e6c07402-0307-11e8-b087-000000000000} # yamllint disable-line - SQLALCHEMY_URL: 'mysql+pymysql://freezing:zeer0@freezing-db-dev/freezing?charset=utf8mb4&binary_prefix=true' # yamllint disable-line - SQLALCHEMY_URL_ROOT: 'mysql+pymysql://root:fr33z3@freezing-db-dev/mysql?charset=utf8mb4&binary_prefix=true' # yamllint disable-line + SQLALCHEMY_URL: "mysql+pymysql://${MYSQL_USER:-freezing}:${MYSQL_PASSWORD:-zeer0}@freezing-db-dev/${MYSQL_DATABASE:-freezing}?charset=utf8mb4&binary_prefix=true" # yamllint disable-line + SQLALCHEMY_URL_ROOT: "mysql+pymysql://${MYSQL_ROOT_USER:-freezing}:${MYSQL_ROOT_PASSWORD:-zeer0}@freezing-db-dev/mysql?charset=utf8mb4&binary_prefix=true" # yamllint disable-line START_DATE: ${START_DATE:-2018-01-01T00:00:00-05:00} STRAVA_CLIENT_ID: ${STRAVA_CLIENT_ID:-?} STRAVA_CLIENT_SECRET: ${STRAVA_CLIENT_SECRET:-?} diff --git a/example.cfg b/example.cfg index 7b2bb17c..cf560b97 100644 --- a/example.cfg +++ b/example.cfg @@ -8,19 +8,32 @@ BIND_INTERFACE=127.0.0.1 DEBUG=true +# By default this uses the localdev environment for local development, which enables +# several features useful for developers, such as user impersonation. +# See +#ENVIRONMENT=localdev + # The SECRET_KEY is used by Flask to sign sessions. Set this to something else. SECRET_KEY=e6c07402-0307-11e8-b087-000000000000 # The URL to the database. Note that the pymysql driver must be explicitly specified. +# +# It is important to use the utf8mb4 charset to get full support for Unicode characters, +# including emoji. +# # This URL is suitable for use with the database running in the freezing-compose setup in this repository. SQLALCHEMY_URL="mysql+pymysql://freezing:zeer0mfreezing-db-dev/freezing?charset=utf8mb4&binary_prefix=true" # Use this one if you are running this on your host vs. the database in the # https://github.com/freezingsaddles/freezing-compose project. +# NOTE: If you are using a MySQL docker via freezing-compose use 127.0.0.1 as the host, NOT localhost #SQLALCHEMY_URL="mysql+pymysql://freezing:please-change-me-as-this-is-a-default@127.0.0.1/freezing?charset=utf8mb4&binary_prefix=true" # If you keep your MySQL database somewhere else, fix this up to match. #SQLALCHEMY_URL="mysql+pymysql://freezing:freezing@127.0.0.1/freezing?charset=utf8mb4&binary_prefix=true"" # Configuration for the Strava client. These settings come from your App setup. +# Setting this is only required if you are testing parts of this application that exercise the Strava API, +# such as user registration. That is an advanced topic and not required to make most changes to +# the web site. Most of the action with the Strava API happens in freezing-sync, not here. STRAVA_CLIENT_ID=? STRAVA_CLIENT_SECRET=? @@ -32,7 +45,7 @@ TEAMS=1234,1235 OBSERVER_TEAMS=5678,9013 # The competition title -COMPETITION_TITLE='BikeArlington Freezing Saddles 2019' +COMPETITION_TITLE='BikeArlington Freezing Saddles 2018 - localdev' # The start date of the competition -- WITH TIME ZONE START_DATE=2018-01-01T00:00:00-05:00 diff --git a/freezing/web/__init__.py b/freezing/web/__init__.py index d8e42891..862a9911 100644 --- a/freezing/web/__init__.py +++ b/freezing/web/__init__.py @@ -1,50 +1,74 @@ -import faulthandler -import signal +""" +This is the main entry point for the freezing-web application. + +It sets up the Flask app and initializes the database and logging. +It also sets up a fault handler with a signal early to ensure stack traces happen. +""" + +import freezing.web._faulthandler # noqa isort: skip + +from socket import gethostbyname +from time import sleep +from urllib.parse import urlparse from flask import Flask, g, session from freezing.model import init_model, meta -faulthandler.register(signal.SIGUSR1) - -from .config import config # noqa - -# Thanks https://stackoverflow.com/a/17073583 -app = Flask( - __name__, - static_folder="static", - static_url_path="/", - instance_path=config.INSTANCE_PATH, -) -app.config.from_object(config) - -init_model(config.SQLALCHEMY_URL) - -# This needs to be after the app is created, unfortunately. -from freezing.web.views import ( # noqa - alt_scoring, - api, - chartdata, - general, - leaderboard, - people, - photos, - pointless, - tribes, - user, -) - -# Register our blueprints - -app.register_blueprint(general.blueprint) -app.register_blueprint(leaderboard.blueprint, url_prefix="/leaderboard") -app.register_blueprint(alt_scoring.blueprint, url_prefix="/alt_scoring") -app.register_blueprint(chartdata.blueprint, url_prefix="/chartdata") -app.register_blueprint(people.blueprint, url_prefix="/people") -app.register_blueprint(pointless.blueprint, url_prefix="/pointless") -app.register_blueprint(photos.blueprint, url_prefix="/photos") -app.register_blueprint(user.blueprint, url_prefix="/my") -app.register_blueprint(api.blueprint, url_prefix="/api") -app.register_blueprint(tribes.blueprint, url_prefix="/tribes") +from freezing.web.autolog import log + +from .config import config, init_logging + +init_logging(color=config.DEBUG is False) + + +_BANNER = "*************************" + + +def _get_app(): + log.info( + f"{_BANNER} Configuring Flask app with instance_path={config.INSTANCE_PATH} {_BANNER}" + ) + # Thanks https://stackoverflow.com/a/17073583 + app = Flask( + __name__, + static_folder="static", + static_url_path="/", + instance_path=config.INSTANCE_PATH, + ) + app.config.from_object(config) + return app + + +def _register_blueprints(app): + # This needs to be after the app is created, unfortunately. + from freezing.web.views import ( + alt_scoring, + api, + chartdata, + general, + leaderboard, + people, + photos, + pointless, + tribes, + user, + ) + + app.register_blueprint(general.blueprint) + app.register_blueprint(leaderboard.blueprint, url_prefix="/leaderboard") + app.register_blueprint(alt_scoring.blueprint, url_prefix="/alt_scoring") + app.register_blueprint(chartdata.blueprint, url_prefix="/chartdata") + app.register_blueprint(people.blueprint, url_prefix="/people") + app.register_blueprint(pointless.blueprint, url_prefix="/pointless") + app.register_blueprint(photos.blueprint, url_prefix="/photos") + app.register_blueprint(user.blueprint, url_prefix="/my") + app.register_blueprint(api.blueprint, url_prefix="/api") + app.register_blueprint(tribes.blueprint, url_prefix="/tribes") + + +# This has to be done before we define the functions with @app decorators +app = _get_app() +_register_blueprints(app) @app.before_request @@ -74,3 +98,33 @@ def inject_config(): "forum_site": config.FORUM_SITE, "version_string": config.VERSION_STRING, } + + +def init_db(): + """ + Initialize the database. If the database is not available, keep trying for a bit. + """ + TRIES = 6 + delay = 2 + for x in range(1, TRIES + 1): + try: + # Use urllib to extract the host and test whether it can be resolved + url = urlparse(config.SQLALCHEMY_URL) + host = gethostbyname(url.hostname) + log.debug(f"gethostbyname({url.hostname})=={host}") + log.info(f"{_BANNER} Connecting to database on {url.hostname} {_BANNER}") + init_model(config.SQLALCHEMY_URL) + break + except Exception as ex: + if x == TRIES: + raise ex from None + log.warning( + f"Failed to connect to database, retrying in {delay}s ({x}) - error was {str(ex)}" + ) + delay = delay * 2 + sleep(delay) + + +init_db() + +log.info(f"{_BANNER} freezing-web initialized in {config.ENVIRONMENT} mode {_BANNER}") diff --git a/freezing/web/_faulthandler.py b/freezing/web/_faulthandler.py new file mode 100644 index 00000000..ee756649 --- /dev/null +++ b/freezing/web/_faulthandler.py @@ -0,0 +1,6 @@ +import faulthandler +import signal + +# Register this super early so we can get a stack trace if +# there is a problem in initializing the config module +faulthandler.register(signal.SIGUSR1) diff --git a/freezing/web/config.py b/freezing/web/config.py index 9770c75b..390e9b2b 100644 --- a/freezing/web/config.py +++ b/freezing/web/config.py @@ -26,26 +26,26 @@ class Config: Refactored with the help of GitHub Copilot. """ - BEANSTALKD_HOST = env("BEANSTALKD_HOST", default="localhost") - BEANSTALKD_PORT = env("BEANSTALKD_PORT", cast=int, default=11300) + BEANSTALKD_HOST: str = env("BEANSTALKD_HOST", default="localhost") + BEANSTALKD_PORT: str = env("BEANSTALKD_PORT", cast=int, default=11300) BIND_INTERFACE: str = env("BIND_INTERFACE", default="127.0.0.1") COMPETITION_TEAMS: List[int] = env("TEAMS", cast=list, subcast=int, default=[]) - COMPETITION_TITLE = env("COMPETITION_TITLE", default="Freezing Saddles") + COMPETITION_TITLE: str = env("COMPETITION_TITLE", default="Freezing Saddles") DEBUG: bool = env("DEBUG", cast=bool, default=False) END_DATE: datetime = env( "END_DATE", postprocessor=lambda val: arrow.get(val).datetime ) # Environment (localdev, production, etc.) - ENVIRONMENT = env("ENVIRONMENT", default="localdev") + ENVIRONMENT: str = env("ENVIRONMENT", default="localdev") FORUM_SITE: str = env( "FORUM_SITE", "https://www.bikearlingtonforum.com/forums/forum/freezing-saddles-winter-riding-competition/", ) - INSTANCE_PATH = env( + INSTANCE_PATH: str = env( "INSTANCE_PATH", default=os.path.join(_basedir, "data/instance") ) # Directory to store leaderboard data - LEADERBOARDS_DIR = env( + LEADERBOARDS_DIR: str = env( "LEADERBOARDS_DIR", default=os.path.join(_basedir, "leaderboards") ) MAIN_TEAM: int = env("MAIN_TEAM", cast=int) @@ -54,7 +54,8 @@ class Config: ) REGISTRATION_SITE: str = env("REGISTRATION_SITE", "https://freezingsaddles.info/") SECRET_KEY = env("SECRET_KEY") - SQLALCHEMY_URL = env("SQLALCHEMY_URL") + SQLALCHEMY_URL: str = env("SQLALCHEMY_URL") + SQLALCHEMY_ROOT_URL: str = env("SQLALCHEMY_ROOT_URL", None) START_DATE: datetime = env( "START_DATE", postprocessor=lambda val: arrow.get(val).datetime ) @@ -65,8 +66,8 @@ class Config: default="America/New_York", postprocessor=lambda val: pytz.timezone(val), ) - VERSION_NUM = version("freezing-web") - VERSION_STRING = f"{VERSION_NUM}+{branch}.{commit}.{build_date}" + VERSION_NUM: str = version("freezing-web") + VERSION_STRING: str = f"{VERSION_NUM}+{branch}.{commit}.{build_date}" config = Config() diff --git a/freezing/web/runserver.py b/freezing/web/runserver.py index e18de253..f31d69e2 100644 --- a/freezing/web/runserver.py +++ b/freezing/web/runserver.py @@ -1,9 +1,8 @@ from freezing.web import app -from freezing.web.config import config, init_logging +from freezing.web.config import config def main(): - init_logging(color=True) app.run(host=config.BIND_INTERFACE, debug=True) diff --git a/requirements-test.txt b/requirements-test.txt index 43fcfa6d..4da32b72 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -4,3 +4,4 @@ djlint==1.36.3 flake8==7.1.1 isort==5.13.2 pur==7.3.2 +pymarkdownlnt==0.9.26