From 084fbe0747b0fb30a85a130bce0374719784538c Mon Sep 17 00:00:00 2001 From: Jared Lumpe Date: Sat, 3 Aug 2024 20:46:35 -0700 Subject: [PATCH] Remove database migrations module --- docs/source/api/database.rst | 6 - setup.cfg | 5 +- src/gambit/db/migrate/__init__.py | 137 ------------------ src/gambit/db/migrate/alembic.ini | 89 ------------ src/gambit/db/migrate/alembic/README | 1 - src/gambit/db/migrate/alembic/env.py | 62 -------- src/gambit/db/migrate/alembic/script.py.mako | 24 --- .../versions/c43540b80d50_gambit_0_1_0.py | 98 ------------- tests/db/test_migrate.py | 68 --------- 9 files changed, 2 insertions(+), 488 deletions(-) delete mode 100644 src/gambit/db/migrate/__init__.py delete mode 100644 src/gambit/db/migrate/alembic.ini delete mode 100644 src/gambit/db/migrate/alembic/README delete mode 100644 src/gambit/db/migrate/alembic/env.py delete mode 100644 src/gambit/db/migrate/alembic/script.py.mako delete mode 100644 src/gambit/db/migrate/alembic/versions/c43540b80d50_gambit_0_1_0.py delete mode 100644 tests/db/test_migrate.py diff --git a/docs/source/api/database.rst b/docs/source/api/database.rst index bfcf307..1655432 100644 --- a/docs/source/api/database.rst +++ b/docs/source/api/database.rst @@ -32,9 +32,3 @@ gambit.db.sqla .. autoclass:: ReadOnlySession :exclude-members: __init__, __new__ :no-members: - - -gambit.db.migrate ------------------ - -.. automodule:: gambit.db.migrate diff --git a/setup.cfg b/setup.cfg index 442fa38..2295cc8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -22,7 +22,6 @@ install_requires = sqlalchemy~=1.1 # Seq stores data as bytes biopython~=1.79 - alembic~=1.0 attrs>=20 # Minimum for 3.12, also introduces potentially breaking changes cattrs>=23.2 @@ -55,8 +54,8 @@ test = pytest # Also check docstrings in package testpaths = tests gambit -# Run doctests on all modules (except __main__.py and alembic config directory) -addopts = --doctest-modules --ignore-glob "**/__main__.py" --ignore "gambit/db/migrate/alembic/" +# Run doctests on all modules (except __main__.py) +addopts = --doctest-modules --ignore-glob "**/__main__.py" doctest_optionflags = NORMALIZE_WHITESPACE IGNORE_EXCEPTION_DETAIL # Treat functions/classes prefixed with "benchmark" as tests, for files in tests/benchmarks/. diff --git a/src/gambit/db/migrate/__init__.py b/src/gambit/db/migrate/__init__.py deleted file mode 100644 index c10d164..0000000 --- a/src/gambit/db/migrate/__init__.py +++ /dev/null @@ -1,137 +0,0 @@ -"""Perform genome database migrations with Alembic. - -This package contains all Alembic configuration and data files. Revision files are located in -``./alembic/versions``. - -Note on alembic configuration - seems like normal usage of Alembic involves getting the database URL -from ``alembic.ini``. Since this application has no fixed location for the database we can't use -this method. Instead we are following the -`Sharing a Connection with a Series of Migration Commands and Environments `_ -recipe in Alembic's documentation, where the connectable object is generated programmatically -somehow and then attached to the Alembic configuration object's ``attributes`` dict. The -``run_migrations_offline`` and ``run_migrations_online`` functions in ``alembic/env.py`` are -modified from the version generated by ``alembic init`` to get their connectable object from this -dict instead of creating it based on the contents of ``alembic.ini``. Note that this means we -can't do (online) migration stuff from the standard alembic CLI command, which gets its -connection information only from ``alembic.ini``. - -The way to use this setup is instead to create an :class:`alembic.config.Config` instance with -:func:`.get_alembic_config` and use the functions in :mod:`alembic.command`. - -.. _alembic-recipe: https://alembic.sqlalchemy.org/en/latest/cookbook.html#sharing-a-connection-with-a-series-of-migration-commands-and-environments -""" - -from typing import Optional - -from alembic.config import Config -from alembic import command -from alembic.migration import MigrationContext -from alembic.script import ScriptDirectory -from pkg_resources import resource_filename -from sqlalchemy.engine import Connectable - - -INI_PATH = resource_filename(__name__, 'alembic.ini') - - -def get_alembic_config(connectable: Optional[Connectable] = None, **kwargs) -> Config: - """Get an alembic config object to perform migrations. - - Parameters - ---------- - connectable - SQLAlchemy connectable specifying database connection info (optional). Assigned to - ``'connectable'`` key of :attr:`alembic.config.Config.attributes`. - \\**kwargs - Keyword arguments to pass to :meth:`alembic.config.Config.__init__`. - - Returns - ------- - Alembic config object. - """ - config = Config(INI_PATH, **kwargs) - config.attributes['connectable'] = connectable - - return config - - -def current_head() -> str: - """Get the current head revision number.""" - conf = get_alembic_config() - scriptdir = ScriptDirectory.from_config(conf) - return scriptdir.get_current_head() - - -def current_revision(connectable: Connectable) -> str: - """Get the current revision number of a genome database.""" - with connectable.connect() as conn: - ctx = MigrationContext.configure(conn) - return ctx.get_current_revision() - - -def is_current_revision(connectable: Connectable): - """Check if the current revision of a genome database is the most recent (head) revision.""" - head = current_head() - current = current_revision(connectable) - return current == head - - -def upgrade(connectable: Connectable, revision: str = 'head', tag=None, **kwargs): - """Run the alembic upgrade command. - - See :func:`alembic.command.upgrade` for more information on how this works. - - Parameters - ---------- - connectable - SQLAlchemy connectable specifying genome database connection info. - revision - Revision to upgrade to. Passed to :func:`alembic.command.upgrade`. - tag - Passed to :func:`alembic.command.upgrade`. - \\**kwargs - Passed to :func:`.get_alembic_config`. - """ - config = get_alembic_config(connectable, **kwargs) - command.upgrade(config, revision, tag=tag) - - -def init_db(connectable: Connectable): - """ - Initialize the genome database schema by creating all tables and stamping with the latest - Alembic revision. - - Expects a fresh database that does not already contain any tables for the :mod:`gambit.db.models` - models and has not had any migrations run on it yet. - - Parameters - ---------- - connectable - SQLAlchemy connectable specifying database connection info. - - Raises - ------ - RuntimeError - If the database is already stamped with an Alembic revision. - sqlalchemy.exc.OperationalError - If any of the database tables to be created already exist. - """ - from gambit.db.models import Base - - conf = get_alembic_config() - script = ScriptDirectory.from_config(conf) - - with connectable.connect() as conn: - ctx = MigrationContext.configure(conn) - - # Check there is no current revision stamped - current = ctx.get_current_revision() - if current is not None: - raise RuntimeError(f'Expected uninitialized database, but current alembic revision is {current}') - - # Create tables - # Set checkfirst=false so that we get an SQL error if any tables already exist - Base.metadata.create_all(conn, checkfirst=False) - - # Stamp latest alembic version - ctx.stamp(script, 'head') diff --git a/src/gambit/db/migrate/alembic.ini b/src/gambit/db/migrate/alembic.ini deleted file mode 100644 index e1d4181..0000000 --- a/src/gambit/db/migrate/alembic.ini +++ /dev/null @@ -1,89 +0,0 @@ -# A generic, single database configuration. - -[alembic] -# Database connection set dynamically in migrate.get_alembic_config function. - -# path to migration scripts -script_location = gambit.db.migrate:alembic - -# template used to generate migration files -# file_template = %%(rev)s_%%(slug)s - -# sys.path path, will be prepended to sys.path if present. -# defaults to the current working directory. -prepend_sys_path = . - -# timezone to use when rendering the date -# within the migration file as well as the filename. -# string value is passed to dateutil.tz.gettz() -# leave blank for localtime -# timezone = - -# max length of characters to apply to the -# "slug" field -# truncate_slug_length = 40 - -# set to 'true' to run the environment during -# the 'revision' command, regardless of autogenerate -# revision_environment = false - -# set to 'true' to allow .pyc and .pyo files without -# a source .py file to be detected as revisions in the -# versions/ directory -# sourceless = false - -# version location specification; this defaults -# to ./alembic/versions. When using multiple version -# directories, initial revisions must be specified with --version-path -# version_locations = %(here)s/bar %(here)s/bat ./alembic/versions - -# the output encoding used when revision files -# are written from script.py.mako -# output_encoding = utf-8 - - -[post_write_hooks] -# post_write_hooks defines scripts or Python functions that are run -# on newly generated revision scripts. See the documentation for further -# detail and examples - -# format using "black" - use the console_scripts runner, against the "black" entrypoint -# hooks = black -# black.type = console_scripts -# black.entrypoint = black -# black.options = -l 79 REVISION_SCRIPT_FILENAME - -# Logging configuration -[loggers] -keys = root,sqlalchemy,alembic - -[handlers] -keys = console - -[formatters] -keys = generic - -[logger_root] -level = WARN -handlers = console -qualname = - -[logger_sqlalchemy] -level = WARN -handlers = -qualname = sqlalchemy.engine - -[logger_alembic] -level = INFO -handlers = -qualname = alembic - -[handler_console] -class = StreamHandler -args = (sys.stderr,) -level = NOTSET -formatter = generic - -[formatter_generic] -format = %(levelname)-5.5s [%(name)s] %(message)s -datefmt = %H:%M:%S diff --git a/src/gambit/db/migrate/alembic/README b/src/gambit/db/migrate/alembic/README deleted file mode 100644 index 98e4f9c..0000000 --- a/src/gambit/db/migrate/alembic/README +++ /dev/null @@ -1 +0,0 @@ -Generic single-database configuration. \ No newline at end of file diff --git a/src/gambit/db/migrate/alembic/env.py b/src/gambit/db/migrate/alembic/env.py deleted file mode 100644 index bc15eb7..0000000 --- a/src/gambit/db/migrate/alembic/env.py +++ /dev/null @@ -1,62 +0,0 @@ -from logging.config import fileConfig - -from alembic import context - -# this is the Alembic Config object, which provides -# access to the values within the .ini file in use. -config = context.config - -# Interpret the config file for Python logging. -# This line sets up loggers basically. -fileConfig(config.config_file_name) - -# add your model's MetaData object here -# for 'autogenerate' support -from gambit.db.models import Base -target_metadata = Base.metadata - -# other values from the config, defined by the needs of env.py, -# can be acquired: -# my_important_option = config.get_main_option("my_important_option") -# ... etc. - - -def run_migrations_offline(): - """Run migrations in 'offline' mode. - - Since we don't have a connection URL written into alembic.ini, we need to specify the - "dialect_name" argument. - """ - context.configure( - dialect_name='sqlite', - target_metadata=target_metadata, - literal_binds=True, - dialect_opts={"paramstyle": "named"}, - ) - - with context.begin_transaction(): - context.run_migrations() - - -def run_migrations_online(): - """Run migrations in 'online' mode. - - Expects a value for the "connectable" argument to migrate.get_alembic_config(). - """ - connectable = config.attributes.get('connectable') - if connectable is None: - raise RuntimeError('Connectable object must be passed to gambit.db.migrate.get_alembic_config()') - - with connectable.connect() as connection: - context.configure( - connection=connection, target_metadata=target_metadata - ) - - with context.begin_transaction(): - context.run_migrations() - - -if context.is_offline_mode(): - run_migrations_offline() -else: - run_migrations_online() diff --git a/src/gambit/db/migrate/alembic/script.py.mako b/src/gambit/db/migrate/alembic/script.py.mako deleted file mode 100644 index 2c01563..0000000 --- a/src/gambit/db/migrate/alembic/script.py.mako +++ /dev/null @@ -1,24 +0,0 @@ -"""${message} - -Revision ID: ${up_revision} -Revises: ${down_revision | comma,n} -Create Date: ${create_date} - -""" -from alembic import op -import sqlalchemy as sa -${imports if imports else ""} - -# revision identifiers, used by Alembic. -revision = ${repr(up_revision)} -down_revision = ${repr(down_revision)} -branch_labels = ${repr(branch_labels)} -depends_on = ${repr(depends_on)} - - -def upgrade(): - ${upgrades if upgrades else "pass"} - - -def downgrade(): - ${downgrades if downgrades else "pass"} diff --git a/src/gambit/db/migrate/alembic/versions/c43540b80d50_gambit_0_1_0.py b/src/gambit/db/migrate/alembic/versions/c43540b80d50_gambit_0_1_0.py deleted file mode 100644 index 2548f7b..0000000 --- a/src/gambit/db/migrate/alembic/versions/c43540b80d50_gambit_0_1_0.py +++ /dev/null @@ -1,98 +0,0 @@ -"""GAMBIT 0.1.0 - -Revision ID: c43540b80d50 -Revises: -Create Date: 2021-07-08 13:34:30.131392 - -Creates 0.1.0 database from scratch. -""" -from alembic import op -import sqlalchemy as sa - -from gambit.db.sqla import JsonString - - -# revision identifiers, used by Alembic. -revision = 'c43540b80d50' -down_revision = None -branch_labels = None -depends_on = None - - -def upgrade(): - op.create_table('genome_sets', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('key', sa.String(), nullable=False), - sa.Column('version', sa.String(), nullable=True), - sa.Column('name', sa.String(), nullable=False), - sa.Column('description', sa.String(), nullable=True), - sa.Column('extra', JsonString(), nullable=True), - sa.PrimaryKeyConstraint('id', name=op.f('pk_genome_sets')), - sa.UniqueConstraint('key', 'version', name=op.f('uq_genome_sets_key')) - ) - op.create_index(op.f('ix_genome_sets_key'), 'genome_sets', ['key'], unique=False) - - op.create_table('genomes', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('key', sa.String(), nullable=False), - sa.Column('description', sa.String(), nullable=False), - sa.Column('ncbi_db', sa.String(), nullable=True), - sa.Column('ncbi_id', sa.Integer(), nullable=True), - sa.Column('genbank_acc', sa.String(), nullable=True), - sa.Column('refseq_acc', sa.String(), nullable=True), - sa.Column('extra', JsonString(), nullable=True), - sa.PrimaryKeyConstraint('id', name=op.f('pk_genomes')), - sa.UniqueConstraint('genbank_acc', name=op.f('uq_genomes_genbank_acc')), - sa.UniqueConstraint('key', name=op.f('uq_genomes_key')), - sa.UniqueConstraint('ncbi_db', 'ncbi_id', name=op.f('uq_genomes_ncbi_db')), - sa.UniqueConstraint('refseq_acc', name=op.f('uq_genomes_refseq_acc')) - ) - - op.create_table('taxa', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('key', sa.String(), nullable=False), - sa.Column('name', sa.String(), nullable=False), - sa.Column('rank', sa.String(), nullable=True), - sa.Column('description', sa.String(), nullable=True), - sa.Column('distance_threshold', sa.Float(), nullable=True), - sa.Column('report', sa.Boolean(), server_default=sa.text('1'), nullable=False), - sa.Column('genome_set_id', sa.Integer(), nullable=False), - sa.Column('parent_id', sa.Integer(), nullable=True), - sa.Column('ncbi_id', sa.Integer(), nullable=True), - sa.Column('extra', JsonString(), nullable=True), - sa.ForeignKeyConstraint(['genome_set_id'], ['genome_sets.id'], name=op.f('fk_taxa_genome_set_id_genome_sets'), ondelete='CASCADE'), - sa.ForeignKeyConstraint(['parent_id'], ['taxa.id'], name=op.f('fk_taxa_parent_id_taxa'), ondelete='SET NULL'), - sa.PrimaryKeyConstraint('id', name=op.f('pk_taxa')), - sa.UniqueConstraint('key', name=op.f('uq_taxa_key')) - ) - op.create_index(op.f('ix_taxa_genome_set_id'), 'taxa', ['genome_set_id'], unique=False) - op.create_index(op.f('ix_taxa_name'), 'taxa', ['name'], unique=False) - op.create_index(op.f('ix_taxa_ncbi_id'), 'taxa', ['ncbi_id'], unique=False) - op.create_index(op.f('ix_taxa_parent_id'), 'taxa', ['parent_id'], unique=False) - op.create_index(op.f('ix_taxa_rank'), 'taxa', ['rank'], unique=False) - - op.create_table('genome_annotations', - sa.Column('genome_id', sa.Integer(), nullable=False), - sa.Column('genome_set_id', sa.Integer(), nullable=False), - sa.Column('taxon_id', sa.Integer(), nullable=True), - sa.Column('organism', sa.String(), nullable=True), - sa.ForeignKeyConstraint(['genome_id'], ['genomes.id'], name=op.f('fk_genome_annotations_genome_id_genomes'), ondelete='CASCADE'), - sa.ForeignKeyConstraint(['genome_set_id'], ['genome_sets.id'], name=op.f('fk_genome_annotations_genome_set_id_genome_sets'), ondelete='CASCADE'), - sa.ForeignKeyConstraint(['taxon_id'], ['taxa.id'], name=op.f('fk_genome_annotations_taxon_id_taxa'), ondelete='SET NULL'), - sa.PrimaryKeyConstraint('genome_id', 'genome_set_id', name=op.f('pk_genome_annotations')) - ) - op.create_index(op.f('ix_genome_annotations_taxon_id'), 'genome_annotations', ['taxon_id'], unique=False) - - -def downgrade(): - op.drop_index(op.f('ix_genome_annotations_taxon_id'), table_name='genome_annotations') - op.drop_table('genome_annotations') - op.drop_index(op.f('ix_taxa_rank'), table_name='taxa') - op.drop_index(op.f('ix_taxa_parent_id'), table_name='taxa') - op.drop_index(op.f('ix_taxa_ncbi_id'), table_name='taxa') - op.drop_index(op.f('ix_taxa_name'), table_name='taxa') - op.drop_index(op.f('ix_taxa_genome_set_id'), table_name='taxa') - op.drop_table('taxa') - op.drop_table('genomes') - op.drop_index(op.f('ix_genome_sets_key'), table_name='genome_sets') - op.drop_table('genome_sets') diff --git a/tests/db/test_migrate.py b/tests/db/test_migrate.py deleted file mode 100644 index de8845b..0000000 --- a/tests/db/test_migrate.py +++ /dev/null @@ -1,68 +0,0 @@ -"""Test the gambit.db.migrate module.""" - -import pytest -from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker -from alembic.migration import MigrationContext -from alembic.script import ScriptDirectory - -from gambit.db.migrate import (current_head, current_revision, is_current_revision, init_db, - get_alembic_config) -from gambit.db import models - - -# Expected current head revision -# Need to update this value every time a new revision is introduced -CURRENT_HEAD = 'c43540b80d50' - -# Old revision number to test. Must actually exist in the scripts directory. -# TODO - set this once we have more than one revision file -OLD_REVISION = None - - -def test_current_head(): - assert current_head() == CURRENT_HEAD - - -class TestCurrentRevision: - """Test the current_revision() and is_current_revision() functions.""" - - def test_uninitialized(self): - """Test on uninitialized database (not stamped).""" - engine = create_engine('sqlite:///:memory:') - assert current_revision(engine) is None - assert not is_current_revision(engine) - - def test_empty(self): - """Test on freshly initialized database.""" - engine = create_engine('sqlite:///:memory:') - init_db(engine) - assert current_revision(engine) == CURRENT_HEAD - assert is_current_revision(engine) - - @pytest.mark.skipif(OLD_REVISION is None, reason='No older revisions to test.') - def test_old(self): - """Test on uninitialized database stamped with an old revision no.""" - engine = create_engine('sqlite:///:memory:') - conf = get_alembic_config(engine) - scriptdir = ScriptDirectory.from_config(conf) - - with engine.connect() as conn: - ctx = MigrationContext.configure(conn) - ctx.stamp(scriptdir, OLD_REVISION) - - assert current_revision(engine) == OLD_REVISION - - -def test_init_db(): - """Test the init_db() function.""" - engine = create_engine('sqlite:///:memory:') - init_db(engine) - - # Check current revision matches - assert current_revision(engine) == current_head() - - # Check we can query all models (won't return any results, but would fail if tables didn't exist). - session = sessionmaker(engine)() - for model in [models.Genome, models.ReferenceGenomeSet, models.AnnotatedGenome, models.Taxon]: - session.query(model).all()