From 78651e524b8105a6369f226de7cec6ee3f3e380a Mon Sep 17 00:00:00 2001 From: Alfred Hodler Date: Fri, 23 Feb 2024 01:04:22 +0000 Subject: [PATCH 1/3] Dynamic migration dicovery, bug fix * Fixes a bug in `get_last_applied_migration` due to an incorrect SQL query that partially uses dynamic table names and partially hardcodes the default one. * Adds a new utility function that enables dynamic migration discovery where embedding is not desirable. --- refinery/src/lib.rs | 2 +- refinery_core/src/lib.rs | 2 +- refinery_core/src/traits/mod.rs | 2 +- refinery_core/src/util.rs | 46 ++++++++++++++++++++++++++++++++- 4 files changed, 48 insertions(+), 4 deletions(-) diff --git a/refinery/src/lib.rs b/refinery/src/lib.rs index a3a4d780..93cfaac5 100644 --- a/refinery/src/lib.rs +++ b/refinery/src/lib.rs @@ -32,7 +32,7 @@ for more examples refer to the [examples](https://github.com/rust-db/refinery/tr */ pub use refinery_core::config; -pub use refinery_core::{error, Error, Migration, Report, Runner, Target}; +pub use refinery_core::{error, load_sql_migrations, Error, Migration, Report, Runner, Target}; #[doc(hidden)] pub use refinery_core::{AsyncMigrate, Migrate}; pub use refinery_macros::embed_migrations; diff --git a/refinery_core/src/lib.rs b/refinery_core/src/lib.rs index 35d02a47..cdd426de 100644 --- a/refinery_core/src/lib.rs +++ b/refinery_core/src/lib.rs @@ -9,7 +9,7 @@ pub use crate::error::Error; pub use crate::runner::{Migration, Report, Runner, Target}; pub use crate::traits::r#async::AsyncMigrate; pub use crate::traits::sync::Migrate; -pub use crate::util::{find_migration_files, MigrationType}; +pub use crate::util::{find_migration_files, load_sql_migrations, MigrationType}; #[cfg(feature = "rusqlite")] pub use rusqlite; diff --git a/refinery_core/src/traits/mod.rs b/refinery_core/src/traits/mod.rs index 9d79a5d1..d3eef6d3 100644 --- a/refinery_core/src/traits/mod.rs +++ b/refinery_core/src/traits/mod.rs @@ -115,7 +115,7 @@ pub(crate) const GET_APPLIED_MIGRATIONS_QUERY: &str = "SELECT version, name, app pub(crate) const GET_LAST_APPLIED_MIGRATION_QUERY: &str = "SELECT version, name, applied_on, checksum - FROM %MIGRATION_TABLE_NAME% WHERE version=(SELECT MAX(version) from refinery_schema_history)"; + FROM %MIGRATION_TABLE_NAME% WHERE version=(SELECT MAX(version) from %MIGRATION_TABLE_NAME%)"; pub(crate) const DEFAULT_MIGRATION_TABLE_NAME: &str = "refinery_schema_history"; diff --git a/refinery_core/src/util.rs b/refinery_core/src/util.rs index 5992aab3..86fa42e4 100644 --- a/refinery_core/src/util.rs +++ b/refinery_core/src/util.rs @@ -1,4 +1,5 @@ use crate::error::{Error, Kind}; +use crate::Migration; use regex::Regex; use std::ffi::OsStr; use std::path::{Path, PathBuf}; @@ -58,9 +59,34 @@ pub fn find_migration_files( Ok(file_paths) } +/// Loads SQL migrations from a path. This enables dynamic migration discovery, as opposed to +/// embedding. The resulting collection is ordered by version. +pub fn load_sql_migrations(location: impl AsRef) -> Result, Error> { + let migration_files = find_migration_files(location, MigrationType::Sql)?; + + let mut migrations = vec![]; + + for path in migration_files { + let sql = std::fs::read_to_string(path.as_path()) + .map_err(|e| Error::new(Kind::InvalidMigrationPath(path.to_owned(), e), None))?; + + //safe to call unwrap as find_migration_filenames returns canonical paths + let filename = path + .file_stem() + .and_then(|file| file.to_os_string().into_string().ok()) + .unwrap(); + + let migration = Migration::unapplied(&filename, &sql)?; + migrations.push(migration); + } + + migrations.sort(); + Ok(migrations) +} + #[cfg(test)] mod tests { - use super::{find_migration_files, MigrationType}; + use super::{find_migration_files, load_sql_migrations, MigrationType}; use std::fs; use std::path::PathBuf; use tempfile::TempDir; @@ -146,4 +172,22 @@ mod tests { let mut mods = find_migration_files(migrations_dir, MigrationType::All).unwrap(); assert!(mods.next().is_none()); } + + #[test] + fn loads_migrations_from_path() { + let tmp_dir = TempDir::new().unwrap(); + let migrations_dir = tmp_dir.path().join("migrations"); + fs::create_dir(&migrations_dir).unwrap(); + let sql1 = migrations_dir.join("V1__first.sql"); + fs::File::create(&sql1).unwrap(); + let sql2 = migrations_dir.join("V2__second.sql"); + fs::File::create(&sql2).unwrap(); + let rs3 = migrations_dir.join("V3__third.rs"); + fs::File::create(&rs3).unwrap(); + + let migrations = load_sql_migrations(migrations_dir).unwrap(); + assert_eq!(migrations.len(), 2); + assert_eq!(&migrations[0].to_string(), "V1__first"); + assert_eq!(&migrations[1].to_string(), "V2__second"); + } } From de55e01510a228cd6d5bc51edbb82d5d3ee749c7 Mon Sep 17 00:00:00 2001 From: Alfred Hodler Date: Sat, 2 Mar 2024 13:01:04 +0000 Subject: [PATCH 2/3] Add a new error variant --- refinery_core/src/error.rs | 3 +++ refinery_core/src/util.rs | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/refinery_core/src/error.rs b/refinery_core/src/error.rs index 3e865c1e..79e3a3bf 100644 --- a/refinery_core/src/error.rs +++ b/refinery_core/src/error.rs @@ -66,6 +66,9 @@ pub enum Kind { /// An Error from an underlying database connection Error #[error("`{0}`, `{1}`")] Connection(String, #[source] Box), + /// An Error from an invalid migration file (not UTF-8 etc) + #[error("invalid migration file at path {0}, {1}")] + InvalidMigrationFile(PathBuf, std::io::Error), } // Helper trait for adding custom messages and applied migrations to Connection error's. diff --git a/refinery_core/src/util.rs b/refinery_core/src/util.rs index 86fa42e4..a9f401df 100644 --- a/refinery_core/src/util.rs +++ b/refinery_core/src/util.rs @@ -68,7 +68,7 @@ pub fn load_sql_migrations(location: impl AsRef) -> Result, for path in migration_files { let sql = std::fs::read_to_string(path.as_path()) - .map_err(|e| Error::new(Kind::InvalidMigrationPath(path.to_owned(), e), None))?; + .map_err(|e| Error::new(Kind::InvalidMigrationFile(path.to_owned(), e), None))?; //safe to call unwrap as find_migration_filenames returns canonical paths let filename = path From 189eb6b0ca7f501095153b5d997dfd93a4a32a30 Mon Sep 17 00:00:00 2001 From: Alfred Hodler Date: Sun, 3 Mar 2024 12:28:56 +0000 Subject: [PATCH 3/3] Distinguish between error types when loading migrations --- refinery_core/src/util.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/refinery_core/src/util.rs b/refinery_core/src/util.rs index a9f401df..f36ed140 100644 --- a/refinery_core/src/util.rs +++ b/refinery_core/src/util.rs @@ -67,8 +67,15 @@ pub fn load_sql_migrations(location: impl AsRef) -> Result, let mut migrations = vec![]; for path in migration_files { - let sql = std::fs::read_to_string(path.as_path()) - .map_err(|e| Error::new(Kind::InvalidMigrationFile(path.to_owned(), e), None))?; + let sql = std::fs::read_to_string(path.as_path()).map_err(|e| { + let path = path.to_owned(); + let kind = match e.kind() { + std::io::ErrorKind::NotFound => Kind::InvalidMigrationPath(path, e), + _ => Kind::InvalidMigrationFile(path, e), + }; + + Error::new(kind, None) + })?; //safe to call unwrap as find_migration_filenames returns canonical paths let filename = path