From 24a548422cd49aa56410d423ce7446eb2b7c12be Mon Sep 17 00:00:00 2001 From: Zixuan James Li Date: Fri, 13 Dec 2024 16:40:54 -0500 Subject: [PATCH 1/8] db [nfc]: Remove dead code Signed-off-by: Zixuan James Li --- lib/model/database.dart | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/lib/model/database.dart b/lib/model/database.dart index 6ca2aa3726..6aa3309c6d 100644 --- a/lib/model/database.dart +++ b/lib/model/database.dart @@ -1,10 +1,5 @@ -import 'dart:io'; - import 'package:drift/drift.dart'; -import 'package:drift/native.dart'; import 'package:drift/remote.dart'; -import 'package:path/path.dart' as path; -import 'package:path_provider/path_provider.dart'; import 'package:sqlite3/common.dart'; part 'database.g.dart'; @@ -52,21 +47,10 @@ class UriConverter extends TypeConverter { @override Uri fromSql(String fromDb) => Uri.parse(fromDb); } -LazyDatabase _openConnection() { - return LazyDatabase(() async { - // TODO decide if this path is the right one to use - final dbFolder = await getApplicationDocumentsDirectory(); - final file = File(path.join(dbFolder.path, 'db.sqlite')); - return NativeDatabase.createInBackground(file); - }); -} - @DriftDatabase(tables: [Accounts]) class AppDatabase extends _$AppDatabase { AppDatabase(super.e); - AppDatabase.live() : this(_openConnection()); - // When updating the schema: // * Make the change in the table classes, and bump schemaVersion. // * Export the new schema and generate test migrations: From 1a23a0d0505fe323ba4156189f93a70e1e8fd258 Mon Sep 17 00:00:00 2001 From: Zixuan James Li Date: Tue, 17 Dec 2024 11:45:01 -0500 Subject: [PATCH 2/8] db [nfc]: Mention build_runner for schema changes Signed-off-by: Zixuan James Li --- lib/model/database.dart | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/model/database.dart b/lib/model/database.dart index 6aa3309c6d..6f941c184b 100644 --- a/lib/model/database.dart +++ b/lib/model/database.dart @@ -53,8 +53,11 @@ class AppDatabase extends _$AppDatabase { // When updating the schema: // * Make the change in the table classes, and bump schemaVersion. - // * Export the new schema and generate test migrations: + // * Export the new schema and generate test migrations with drift: // $ tools/check --fix drift + // and generate database code with build_runner. + // See ../../README.md#generated-files for more + // information on using the build_runner. // * Write a migration in `onUpgrade` below. // * Write tests. @override From f85b5fca6cd368d5360ad7ef5f9d5e901465dc59 Mon Sep 17 00:00:00 2001 From: Zixuan James Li Date: Wed, 18 Dec 2024 16:20:19 -0500 Subject: [PATCH 3/8] db test [nfc]: Add and skip test for downgrading Signed-off-by: Zixuan James Li --- test/model/database_test.dart | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/model/database_test.dart b/test/model/database_test.dart index cb3a7d299b..0bfe8cd580 100644 --- a/test/model/database_test.dart +++ b/test/model/database_test.dart @@ -98,6 +98,13 @@ void main() { verifier = SchemaVerifier(GeneratedHelper()); }); + test('downgrading', () async { + final connection = await verifier.startAt(2); + final db = AppDatabase(connection); + await verifier.migrateAndValidate(db, 1); + await db.close(); + }, skip: true); // TODO(#1172): unskip this + test('upgrade to v2, empty', () async { final connection = await verifier.startAt(1); final db = AppDatabase(connection); From 8eb2d5101722b9ee90ef4947df470dfb682a5c38 Mon Sep 17 00:00:00 2001 From: Zixuan James Li Date: Tue, 17 Dec 2024 19:09:47 -0500 Subject: [PATCH 4/8] db test: Add missing `after.close` call Signed-off-by: Zixuan James Li --- test/model/database_test.dart | 1 + 1 file changed, 1 insertion(+) diff --git a/test/model/database_test.dart b/test/model/database_test.dart index 0bfe8cd580..862e3f2109 100644 --- a/test/model/database_test.dart +++ b/test/model/database_test.dart @@ -137,6 +137,7 @@ void main() { ...accountV1.toJson(), 'ackedPushToken': null, }); + await after.close(); }); }); } From 844aba20ea9d15fc058f00aae18f8a8c863af45d Mon Sep 17 00:00:00 2001 From: Zixuan James Li Date: Sun, 29 Dec 2024 23:21:33 -0500 Subject: [PATCH 5/8] deps: Upgrade drift to 2.23.0 We will start requiring the more recent features added in releases since 2.5.0. Signed-off-by: Zixuan James Li --- pubspec.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pubspec.yaml b/pubspec.yaml index 27b7519334..519373686a 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -40,7 +40,7 @@ dependencies: convert: ^3.1.1 crypto: ^3.0.3 device_info_plus: ^11.2.0 - drift: ^2.5.0 + drift: ^2.23.0 file_picker: ^8.0.0+1 firebase_core: ^3.3.0 firebase_messaging: ^15.0.1 From 0318e069707bcfc4b45db66b43da07c173f3128c Mon Sep 17 00:00:00 2001 From: Zixuan James Li Date: Fri, 3 Jan 2025 11:30:52 +0800 Subject: [PATCH 6/8] database test: Rewrite database tests using generated helpers The tests are adapted from the test template generated from `dart run drift_dev make-migrations`. This saves us from writing the simple migration tests between schemas without data in the future. For the test that upgrade the schema with data, with the helper, while it ended up having more lines than the original, the test becomes more structured this way. Signed-off-by: Zixuan James Li --- test/model/database_test.dart | 77 +++++++++++++++++++++-------------- 1 file changed, 47 insertions(+), 30 deletions(-) diff --git a/test/model/database_test.dart b/test/model/database_test.dart index 862e3f2109..85bfd9c232 100644 --- a/test/model/database_test.dart +++ b/test/model/database_test.dart @@ -105,39 +105,56 @@ void main() { await db.close(); }, skip: true); // TODO(#1172): unskip this - test('upgrade to v2, empty', () async { - final connection = await verifier.startAt(1); - final db = AppDatabase(connection); - await verifier.migrateAndValidate(db, 2); - await db.close(); + group('migrate without data', () { + // These simple tests verify all possible schema updates with a simple (no + // data) migration. This is a quick way to ensure that written database + // migrations properly alter the schema. + const versions = GeneratedHelper.versions; + for (final (i, fromVersion) in versions.indexed) { + group('from $fromVersion', () { + for (final toVersion in versions.skip(i + 1)) { + test('to $toVersion', () async { + final schema = await verifier.schemaAt(fromVersion); + final db = AppDatabase(schema.newConnection()); + await verifier.migrateAndValidate(db, toVersion); + await db.close(); + }); + } + }); + } }); - test('upgrade to v2, with data', () async { - final schema = await verifier.schemaAt(1); - final before = v1.DatabaseAtV1(schema.newConnection()); - await before.into(before.accounts).insert(v1.AccountsCompanion.insert( - realmUrl: 'https://chat.example/', - userId: 1, - email: 'asdf@example.org', - apiKey: '1234', - zulipVersion: '6.0', - zulipMergeBase: const Value('6.0'), - zulipFeatureLevel: 42, - )); - final accountV1 = await before.select(before.accounts).watchSingle().first; - await before.close(); - - final db = AppDatabase(schema.newConnection()); - await verifier.migrateAndValidate(db, 2); - await db.close(); - - final after = v2.DatabaseAtV2(schema.newConnection()); - final account = await after.select(after.accounts).getSingle(); - check(account.toJson()).deepEquals({ - ...accountV1.toJson(), - 'ackedPushToken': null, + // Testing this can be useful for migrations that change existing columns + // (e.g. by alterating their type or constraints). Migrations that only add + // tables or columns typically don't need these advanced tests. For more + // information, see https://drift.simonbinder.eu/migrations/tests/#verifying-data-integrity + group('migrate with data', () { + test('upgrade to v2', () async { + late final v1.AccountsData oldAccountData; + await verifier.testWithDataIntegrity( + oldVersion: 1, createOld: v1.DatabaseAtV1.new, + newVersion: 2, createNew: v2.DatabaseAtV2.new, + openTestedDatabase: AppDatabase.new, + createItems: (batch, oldDb) async { + await oldDb.into(oldDb.accounts).insert(v1.AccountsCompanion.insert( + realmUrl: 'https://chat.example/', + userId: 1, + email: 'asdf@example.org', + apiKey: '1234', + zulipVersion: '6.0', + zulipMergeBase: const Value('6.0'), + zulipFeatureLevel: 42, + )); + oldAccountData = await oldDb.select(oldDb.accounts).watchSingle().first; + }, + validateItems: (newDb) async { + final account = await newDb.select(newDb.accounts).getSingle(); + check(account.toJson()).deepEquals({ + ...oldAccountData.toJson(), + 'ackedPushToken': null, + }); + }); }); - await after.close(); }); }); } From fe47b48004e5be01581bd4659df0cf88db6adc9c Mon Sep 17 00:00:00 2001 From: Zixuan James Li Date: Wed, 18 Dec 2024 15:46:28 -0500 Subject: [PATCH 7/8] db: Start generating step by step migration helper An alternative to this is `dart run drift_dev make-migrations`, which is essentially a wrapper for the `schema {dump,generate,steps}` subcommands. `make-migrations` let us manage multiple database schemas by configuring them with `build.yaml`, and it dictates the which subdirectories the generated files will be created at. Because `make-migrations` does not offer the same level of customizations to designate exactly where the output files will be, opting out from it for now. We can revisit this if it starts to offer features that are not available with the subcommands, or that we find the need for managing multiple databases. See also: https://drift.simonbinder.eu/migrations/step_by_step/#manual-generation Signed-off-by: Zixuan James Li --- lib/model/database.dart | 15 +++-- lib/model/schema_versions.g.dart | 112 +++++++++++++++++++++++++++++++ tools/check | 7 ++ 3 files changed, 128 insertions(+), 6 deletions(-) create mode 100644 lib/model/schema_versions.g.dart diff --git a/lib/model/database.dart b/lib/model/database.dart index 6f941c184b..b8a0f780fe 100644 --- a/lib/model/database.dart +++ b/lib/model/database.dart @@ -2,6 +2,8 @@ import 'package:drift/drift.dart'; import 'package:drift/remote.dart'; import 'package:sqlite3/common.dart'; +import 'schema_versions.g.dart'; + part 'database.g.dart'; /// The table of [Account] records in the app's database. @@ -84,12 +86,13 @@ class AppDatabase extends _$AppDatabase { } assert(1 <= from && from <= to && to <= schemaVersion); - if (from < 2 && 2 <= to) { - await m.addColumn(accounts, accounts.ackedPushToken); - } - // New migrations go here. - } - ); + await m.runMigrationSteps(from: from, to: to, + steps: migrationSteps( + from1To2: (m, schema) async { + await m.addColumn(schema.accounts, schema.accounts.ackedPushToken); + }, + )); + }); } Future createAccount(AccountsCompanion values) async { diff --git a/lib/model/schema_versions.g.dart b/lib/model/schema_versions.g.dart new file mode 100644 index 0000000000..300813c53e --- /dev/null +++ b/lib/model/schema_versions.g.dart @@ -0,0 +1,112 @@ +// dart format width=80 +import 'package:drift/internal/versioned_schema.dart' as i0; +import 'package:drift/drift.dart' as i1; +import 'package:drift/drift.dart'; // ignore_for_file: type=lint,unused_import + +// GENERATED BY drift_dev, DO NOT MODIFY. +final class Schema2 extends i0.VersionedSchema { + Schema2({required super.database}) : super(version: 2); + @override + late final List entities = [ + accounts, + ]; + late final Shape0 accounts = Shape0( + source: i0.VersionedTable( + entityName: 'accounts', + withoutRowId: false, + isStrict: false, + tableConstraints: [ + 'UNIQUE(realm_url, user_id)', + 'UNIQUE(realm_url, email)', + ], + columns: [ + _column_0, + _column_1, + _column_2, + _column_3, + _column_4, + _column_5, + _column_6, + _column_7, + _column_8, + ], + attachedDatabase: database, + ), + alias: null); +} + +class Shape0 extends i0.VersionedTable { + Shape0({required super.source, required super.alias}) : super.aliased(); + i1.GeneratedColumn get id => + columnsByName['id']! as i1.GeneratedColumn; + i1.GeneratedColumn get realmUrl => + columnsByName['realm_url']! as i1.GeneratedColumn; + i1.GeneratedColumn get userId => + columnsByName['user_id']! as i1.GeneratedColumn; + i1.GeneratedColumn get email => + columnsByName['email']! as i1.GeneratedColumn; + i1.GeneratedColumn get apiKey => + columnsByName['api_key']! as i1.GeneratedColumn; + i1.GeneratedColumn get zulipVersion => + columnsByName['zulip_version']! as i1.GeneratedColumn; + i1.GeneratedColumn get zulipMergeBase => + columnsByName['zulip_merge_base']! as i1.GeneratedColumn; + i1.GeneratedColumn get zulipFeatureLevel => + columnsByName['zulip_feature_level']! as i1.GeneratedColumn; + i1.GeneratedColumn get ackedPushToken => + columnsByName['acked_push_token']! as i1.GeneratedColumn; +} + +i1.GeneratedColumn _column_0(String aliasedName) => + i1.GeneratedColumn('id', aliasedName, false, + hasAutoIncrement: true, + type: i1.DriftSqlType.int, + defaultConstraints: + i1.GeneratedColumn.constraintIsAlways('PRIMARY KEY AUTOINCREMENT')); +i1.GeneratedColumn _column_1(String aliasedName) => + i1.GeneratedColumn('realm_url', aliasedName, false, + type: i1.DriftSqlType.string); +i1.GeneratedColumn _column_2(String aliasedName) => + i1.GeneratedColumn('user_id', aliasedName, false, + type: i1.DriftSqlType.int); +i1.GeneratedColumn _column_3(String aliasedName) => + i1.GeneratedColumn('email', aliasedName, false, + type: i1.DriftSqlType.string); +i1.GeneratedColumn _column_4(String aliasedName) => + i1.GeneratedColumn('api_key', aliasedName, false, + type: i1.DriftSqlType.string); +i1.GeneratedColumn _column_5(String aliasedName) => + i1.GeneratedColumn('zulip_version', aliasedName, false, + type: i1.DriftSqlType.string); +i1.GeneratedColumn _column_6(String aliasedName) => + i1.GeneratedColumn('zulip_merge_base', aliasedName, true, + type: i1.DriftSqlType.string); +i1.GeneratedColumn _column_7(String aliasedName) => + i1.GeneratedColumn('zulip_feature_level', aliasedName, false, + type: i1.DriftSqlType.int); +i1.GeneratedColumn _column_8(String aliasedName) => + i1.GeneratedColumn('acked_push_token', aliasedName, true, + type: i1.DriftSqlType.string); +i0.MigrationStepWithVersion migrationSteps({ + required Future Function(i1.Migrator m, Schema2 schema) from1To2, +}) { + return (currentVersion, database) async { + switch (currentVersion) { + case 1: + final schema = Schema2(database: database); + final migrator = i1.Migrator(database, schema); + await from1To2(migrator, schema); + return 2; + default: + throw ArgumentError.value('Unknown migration from $currentVersion'); + } + }; +} + +i1.OnUpgrade stepByStep({ + required Future Function(i1.Migrator m, Schema2 schema) from1To2, +}) => + i0.VersionedSchema.stepByStepHelper( + step: migrationSteps( + from1To2: from1To2, + )); diff --git a/tools/check b/tools/check index fefcd514ed..392c11d634 100755 --- a/tools/check +++ b/tools/check @@ -378,6 +378,7 @@ run_l10n() { run_drift() { local schema_dir=test/model/schemas/ + local migration_helper_path=lib/model/schema_versions.g.dart # Omitted from this check: # pubspec.{yaml,lock} tools/check @@ -386,6 +387,8 @@ run_drift() { check_no_uncommitted_or_untracked "${schema_dir}" \ || return + check_no_uncommitted_or_untracked "${migration_helper_path}" \ + || return dart run drift_dev schema dump \ lib/model/database.dart "${schema_dir}" \ @@ -393,8 +396,12 @@ run_drift() { dart run drift_dev schema generate --data-classes --companions \ "${schema_dir}" "${schema_dir}" \ || return + dart run drift_dev schema steps \ + "${schema_dir}" "${migration_helper_path}" \ + || return check_no_changes "schema updates" "${schema_dir}" + check_no_changes "migration helper updates" "${migration_helper_path}" } filter_flutter_pub_run_output() { From 125afeabed328c33f226631c860bf521fd3b1eb0 Mon Sep 17 00:00:00 2001 From: Zixuan James Li Date: Fri, 3 Jan 2025 09:54:19 +0800 Subject: [PATCH 8/8] db: Drop all tables on downgrade We previously missed tables that are not known to the schema. This becomes an issue if a new table is added at a newer schema level. When we go back to an earlier schema, that new table remains in the database, so subsequent attempts to upgrade to the later schema level that adds the table will fail, because it already exists. Testing for this is blocked until #1172. Signed-off-by: Zixuan James Li --- lib/model/database.dart | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/lib/model/database.dart b/lib/model/database.dart index b8a0f780fe..ae50ab2765 100644 --- a/lib/model/database.dart +++ b/lib/model/database.dart @@ -65,6 +65,26 @@ class AppDatabase extends _$AppDatabase { @override int get schemaVersion => 2; // See note. + Future _resetDatabase(Migrator m) async { + // This should only ever happen in dev. As a dev convenience, + // drop everything from the database and start over. + await m.database.transaction(() async { + final query = m.database.customSelect( + "SELECT name FROM sqlite_master WHERE type='table'"); + for (final row in await query.get()) { + final data = row.data; + final tableName = data['name'] as String; + // Skip sqlite-internal tables, https://www.sqlite.org/fileformat2.html#intschema + // https://github.com/simolus3/drift/blob/0901c984a987e54ba0b67e227adbfdcf3c08eeb4/drift_dev/lib/src/services/schema/verifier_common.dart#L9-L22 + if (tableName.startsWith('sqlite_')) continue; + // SQL injection could be a concern, but the database would've been + // compromised if the table name is corrupted. + await m.database.customStatement('DROP TABLE $tableName'); + } + await m.createAll(); + }); + } + @override MigrationStrategy get migration { return MigrationStrategy( @@ -74,14 +94,7 @@ class AppDatabase extends _$AppDatabase { onUpgrade: (Migrator m, int from, int to) async { if (from > to) { // TODO(log): log schema downgrade as an error - // This should only ever happen in dev. As a dev convenience, - // drop everything from the database and start over. - for (final entity in allSchemaEntities) { - // This will miss any entire tables (or indexes, etc.) that - // don't exist at this version. For a dev-only feature, that's OK. - await m.drop(entity); - } - await m.createAll(); + await _resetDatabase(m); return; } assert(1 <= from && from <= to && to <= schemaVersion);