From 5a8ca7e039661d0d567e34cdff581271be74c0bf Mon Sep 17 00:00:00 2001 From: Paul Woitaschek Date: Sat, 13 May 2023 11:36:20 +0200 Subject: [PATCH] Folder picker performance improvements (#1874) * Improve the performance of the folder picker by querying all the required data at once to reduce the IPC calls. * Clean up the document file logic. * Only propagate the scanner as active while searching for books, not for covers. * Add timing diagnostics. * Make use of the Caching DocumentFile in the book scanning process to greatly improve the overall performance. * Simplify the time measurement. * Prioritize the scanning based on the amount of audio files. * Abstract the CachedDocumentFile and create a file based implementation that is used in unit tests. --- app/build.gradle.kts | 1 + data/build.gradle.kts | 1 + .../voice/data/SupportedAudioFormats.kt | 14 ++++ .../voice/data/folders/AudiobookFolders.kt | 38 +++++---- documentfile/build.gradle.kts | 15 ++++ .../voice/documentfile/CachedDocumentFile.kt | 28 +++++++ .../documentfile/CachedDocumentFileFactory.kt | 7 ++ .../documentfile/FileBasedDocumentFile.kt | 25 ++++++ .../kotlin/voice/documentfile/FileContents.kt | 39 +++++++++ .../voice/documentfile/ParseContents.kt | 26 ++++++ .../documentfile/RealCachedDocumentFile.kt | 38 +++++++++ .../RealCachedDocumentFileFactory.kt | 17 ++++ .../main/kotlin/voice/documentfile/Walk.kt | 11 +++ folderPicker/build.gradle.kts | 1 + .../folderPicker/FolderPickerViewModel.kt | 19 +---- .../selectType/DocumentFileCache.kt | 50 ------------ .../selectType/SelectFolderType.kt | 2 +- .../selectType/SelectFolderTypeViewModel.kt | 21 +++-- .../SelectFolderTypeViewModelTest.kt | 4 +- new_module.main.kts | 79 +++++++++++++++++++ .../kotlin/voice/app/scanner/BookParser.kt | 14 ++-- .../kotlin/voice/app/scanner/ChapterParser.kt | 21 ++--- .../voice/app/scanner/FFProbeAnalyze.kt | 4 +- .../kotlin/voice/app/scanner/MediaAnalyzer.kt | 15 +--- .../voice/app/scanner/MediaScanTrigger.kt | 26 +++--- .../kotlin/voice/app/scanner/MediaScanner.kt | 22 ++++-- .../voice/app/scanner/MediaAnalyzerTest.kt | 6 +- .../voice/app/scanner/MediaScannerTest.kt | 9 ++- settings.gradle.kts | 1 + 29 files changed, 400 insertions(+), 154 deletions(-) create mode 100644 documentfile/build.gradle.kts create mode 100644 documentfile/src/main/kotlin/voice/documentfile/CachedDocumentFile.kt create mode 100644 documentfile/src/main/kotlin/voice/documentfile/CachedDocumentFileFactory.kt create mode 100644 documentfile/src/main/kotlin/voice/documentfile/FileBasedDocumentFile.kt create mode 100644 documentfile/src/main/kotlin/voice/documentfile/FileContents.kt create mode 100644 documentfile/src/main/kotlin/voice/documentfile/ParseContents.kt create mode 100644 documentfile/src/main/kotlin/voice/documentfile/RealCachedDocumentFile.kt create mode 100644 documentfile/src/main/kotlin/voice/documentfile/RealCachedDocumentFileFactory.kt create mode 100644 documentfile/src/main/kotlin/voice/documentfile/Walk.kt delete mode 100644 folderPicker/src/main/kotlin/voice/folderPicker/selectType/DocumentFileCache.kt create mode 100755 new_module.main.kts diff --git a/app/build.gradle.kts b/app/build.gradle.kts index be157aef71..347ee3276a 100755 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -155,6 +155,7 @@ dependencies { implementation(projects.migration) implementation(projects.search) implementation(projects.cover) + implementation(projects.documentfile) implementation(libs.appCompat) implementation(libs.recyclerView) diff --git a/data/build.gradle.kts b/data/build.gradle.kts index 8fbae6ce3e..8625afd0fb 100644 --- a/data/build.gradle.kts +++ b/data/build.gradle.kts @@ -31,6 +31,7 @@ android { dependencies { api(projects.common) + api(projects.documentfile) implementation(libs.appCompat) implementation(libs.androidxCore) implementation(libs.serialization.json) diff --git a/data/src/main/kotlin/voice/data/SupportedAudioFormats.kt b/data/src/main/kotlin/voice/data/SupportedAudioFormats.kt index 4f83b25607..e6dbcfce47 100644 --- a/data/src/main/kotlin/voice/data/SupportedAudioFormats.kt +++ b/data/src/main/kotlin/voice/data/SupportedAudioFormats.kt @@ -1,5 +1,8 @@ package voice.data +import voice.documentfile.CachedDocumentFile +import voice.documentfile.walk + val supportedAudioFormats = arrayOf( "3gp", "aac", @@ -25,3 +28,14 @@ val supportedAudioFormats = arrayOf( "webm", "xmf", ) + +fun CachedDocumentFile.isAudioFile(): Boolean { + if (!isFile) return false + val name = name ?: return false + val extension = name.substringAfterLast(".").lowercase() + return extension in supportedAudioFormats +} + +fun CachedDocumentFile.audioFileCount(): Int { + return walk().count { it.isAudioFile() } +} diff --git a/data/src/main/kotlin/voice/data/folders/AudiobookFolders.kt b/data/src/main/kotlin/voice/data/folders/AudiobookFolders.kt index 0e173740c4..7d50092394 100644 --- a/data/src/main/kotlin/voice/data/folders/AudiobookFolders.kt +++ b/data/src/main/kotlin/voice/data/folders/AudiobookFolders.kt @@ -1,10 +1,10 @@ package voice.data.folders -import android.app.Application +import android.content.Context import android.content.Intent import android.net.Uri +import android.provider.DocumentsContract import androidx.datastore.core.DataStore -import androidx.documentfile.provider.DocumentFile import kotlinx.coroutines.MainScope import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.combine @@ -13,6 +13,8 @@ import kotlinx.coroutines.launch import voice.common.pref.RootAudiobookFolders import voice.common.pref.SingleFileAudiobookFolders import voice.common.pref.SingleFolderAudiobookFolders +import voice.documentfile.CachedDocumentFile +import voice.documentfile.CachedDocumentFileFactory import voice.logging.core.Logger import javax.inject.Inject @@ -24,7 +26,8 @@ class AudiobookFolders private val singleFolderAudiobookFolders: DataStore>, @SingleFileAudiobookFolders private val singleFileAudiobookFolders: DataStore>, - private val application: Application, + private val context: Context, + private val cachedDocumentFileFactory: CachedDocumentFileFactory, ) { private val scope = MainScope() @@ -33,11 +36,8 @@ class AudiobookFolders val flows = FolderType.values() .map { folderType -> dataStore(folderType).data.map { uris -> - val documentFiles = uris.mapNotNull { uri -> - val documentFile = uri.toDocumentFile(folderType) - documentFile?.let { - DocumentFileWithUri(it, uri) - } + val documentFiles = uris.map { uri -> + DocumentFileWithUri(uri.toDocumentFile(folderType), uri) } folderType to documentFiles } @@ -47,19 +47,23 @@ class AudiobookFolders private fun Uri.toDocumentFile( folderType: FolderType, - ): DocumentFile? { - return when (folderType) { - FolderType.SingleFile -> { - DocumentFile.fromSingleUri(application, this) - } + ): CachedDocumentFile { + val uri = when (folderType) { + FolderType.SingleFile -> this FolderType.SingleFolder, FolderType.Root, - -> DocumentFile.fromTreeUri(application, this) + -> { + DocumentsContract.buildDocumentUriUsingTree( + this, + DocumentsContract.getTreeDocumentId(this), + ) + } } + return cachedDocumentFileFactory.create(uri) } fun add(uri: Uri, type: FolderType) { - application.contentResolver.takePersistableUriPermission( + context.contentResolver.takePersistableUriPermission( uri, Intent.FLAG_GRANT_READ_URI_PERMISSION or Intent.FLAG_GRANT_WRITE_URI_PERMISSION, ) @@ -72,7 +76,7 @@ class AudiobookFolders fun remove(uri: Uri, folderType: FolderType) { try { - application.contentResolver.releasePersistableUriPermission( + context.contentResolver.releasePersistableUriPermission( uri, Intent.FLAG_GRANT_READ_URI_PERMISSION or Intent.FLAG_GRANT_WRITE_URI_PERMISSION, ) @@ -94,6 +98,6 @@ class AudiobookFolders } data class DocumentFileWithUri( - val documentFile: DocumentFile, + val documentFile: CachedDocumentFile, val uri: Uri, ) diff --git a/documentfile/build.gradle.kts b/documentfile/build.gradle.kts new file mode 100644 index 0000000000..fbaeecd655 --- /dev/null +++ b/documentfile/build.gradle.kts @@ -0,0 +1,15 @@ +plugins { + id("voice.library") + alias(libs.plugins.anvil) + +} + +anvil { + generateDaggerFactories.set(true) +} + +dependencies { + implementation(projects.common) + implementation(libs.dagger.core) + implementation(libs.androidxCore) +} diff --git a/documentfile/src/main/kotlin/voice/documentfile/CachedDocumentFile.kt b/documentfile/src/main/kotlin/voice/documentfile/CachedDocumentFile.kt new file mode 100644 index 0000000000..3453c1e118 --- /dev/null +++ b/documentfile/src/main/kotlin/voice/documentfile/CachedDocumentFile.kt @@ -0,0 +1,28 @@ +package voice.documentfile + +import android.net.Uri + +interface CachedDocumentFile { + val children: List + val name: String? + val isDirectory: Boolean + val isFile: Boolean + val length: Long + val lastModified: Long + val uri: Uri +} + +fun CachedDocumentFile.nameWithoutExtension(): String { + val name = name + return if (name == null) { + uri.pathSegments.lastOrNull() + ?.dropWhile { it != ':' } + ?.removePrefix(":") + ?.takeUnless { it.isBlank() } + ?: uri.toString() + } else { + name.substringBeforeLast(".") + .takeUnless { it.isEmpty() } + ?: name + } +} diff --git a/documentfile/src/main/kotlin/voice/documentfile/CachedDocumentFileFactory.kt b/documentfile/src/main/kotlin/voice/documentfile/CachedDocumentFileFactory.kt new file mode 100644 index 0000000000..ef20bc0d99 --- /dev/null +++ b/documentfile/src/main/kotlin/voice/documentfile/CachedDocumentFileFactory.kt @@ -0,0 +1,7 @@ +package voice.documentfile + +import android.net.Uri + +interface CachedDocumentFileFactory { + fun create(uri: Uri): CachedDocumentFile +} diff --git a/documentfile/src/main/kotlin/voice/documentfile/FileBasedDocumentFile.kt b/documentfile/src/main/kotlin/voice/documentfile/FileBasedDocumentFile.kt new file mode 100644 index 0000000000..1e96c62704 --- /dev/null +++ b/documentfile/src/main/kotlin/voice/documentfile/FileBasedDocumentFile.kt @@ -0,0 +1,25 @@ +package voice.documentfile + +import android.net.Uri +import androidx.core.net.toFile +import androidx.core.net.toUri +import java.io.File + +data class FileBasedDocumentFile( + private val file: File, +) : CachedDocumentFile { + + override val children: List get() = file.listFiles()?.map { FileBasedDocumentFile(it) } ?: emptyList() + override val name: String? get() = file.name + override val isDirectory: Boolean get() = file.isDirectory + override val isFile: Boolean get() = file.isFile + override val length: Long get() = file.length() + override val lastModified: Long get() = file.lastModified() + override val uri: Uri get() = file.toUri() +} + +object FileBasedDocumentFactory : CachedDocumentFileFactory { + override fun create(uri: Uri): CachedDocumentFile { + return FileBasedDocumentFile(uri.toFile()) + } +} diff --git a/documentfile/src/main/kotlin/voice/documentfile/FileContents.kt b/documentfile/src/main/kotlin/voice/documentfile/FileContents.kt new file mode 100644 index 0000000000..e53d5ee654 --- /dev/null +++ b/documentfile/src/main/kotlin/voice/documentfile/FileContents.kt @@ -0,0 +1,39 @@ +package voice.documentfile + +import android.database.Cursor +import android.provider.DocumentsContract +import androidx.core.database.getLongOrNull +import androidx.core.database.getStringOrNull + +internal data class FileContents( + val name: String?, + val isFile: Boolean, + val isDirectory: Boolean, + val length: Long, + val lastModified: Long, +) { + + companion object { + val columns = arrayOf( + DocumentsContract.Document.COLUMN_DOCUMENT_ID, + DocumentsContract.Document.COLUMN_MIME_TYPE, + DocumentsContract.Document.COLUMN_DISPLAY_NAME, + DocumentsContract.Document.COLUMN_SIZE, + DocumentsContract.Document.COLUMN_LAST_MODIFIED, + ) + + fun readFrom(cursor: Cursor): FileContents { + val mimeType = cursor.getStringOrNull(DocumentsContract.Document.COLUMN_MIME_TYPE) + return FileContents( + name = cursor.getStringOrNull(DocumentsContract.Document.COLUMN_DISPLAY_NAME), + isFile = mimeType != null && mimeType != DocumentsContract.Document.MIME_TYPE_DIR, + isDirectory = mimeType == DocumentsContract.Document.MIME_TYPE_DIR, + length = cursor.getLongOrNull(DocumentsContract.Document.COLUMN_SIZE) ?: 0L, + lastModified = cursor.getLongOrNull(DocumentsContract.Document.COLUMN_LAST_MODIFIED) ?: 0L, + ) + } + } +} + +private fun Cursor.getStringOrNull(columnName: String): String? = getStringOrNull(getColumnIndexOrThrow(columnName)) +private fun Cursor.getLongOrNull(columnName: String): Long? = getLongOrNull(getColumnIndexOrThrow(columnName)) diff --git a/documentfile/src/main/kotlin/voice/documentfile/ParseContents.kt b/documentfile/src/main/kotlin/voice/documentfile/ParseContents.kt new file mode 100644 index 0000000000..8743c5aa14 --- /dev/null +++ b/documentfile/src/main/kotlin/voice/documentfile/ParseContents.kt @@ -0,0 +1,26 @@ +package voice.documentfile + +import android.content.Context +import android.net.Uri +import android.provider.DocumentsContract +import androidx.core.database.getStringOrNull + +internal fun parseContents(uri: Uri, context: Context): List { + val childrenUri = DocumentsContract.buildChildDocumentsUriUsingTree( + uri, + DocumentsContract.getDocumentId(uri), + ) + return context.contentResolver.query( + childrenUri, + FileContents.columns, null, null, null, + )?.use { cursor -> + val files = mutableListOf() + while (cursor.moveToNext()) { + val documentId = cursor.getStringOrNull(cursor.getColumnIndexOrThrow(DocumentsContract.Document.COLUMN_DOCUMENT_ID)) + val documentUri = DocumentsContract.buildDocumentUriUsingTree(uri, documentId) + val contents = FileContents.readFrom(cursor) + files += RealCachedDocumentFile(context, documentUri, contents) + } + files + } ?: emptyList() +} diff --git a/documentfile/src/main/kotlin/voice/documentfile/RealCachedDocumentFile.kt b/documentfile/src/main/kotlin/voice/documentfile/RealCachedDocumentFile.kt new file mode 100644 index 0000000000..c6ada02b93 --- /dev/null +++ b/documentfile/src/main/kotlin/voice/documentfile/RealCachedDocumentFile.kt @@ -0,0 +1,38 @@ +package voice.documentfile + +import android.content.Context +import android.net.Uri + +internal data class RealCachedDocumentFile( + val context: Context, + override val uri: Uri, + private val preFilledContent: FileContents?, +) : CachedDocumentFile { + + override val children: List by lazy { + if (isDirectory) { + parseContents(uri, context) + } else { + emptyList() + } + } + + private val content: FileContents? by lazy { + preFilledContent ?: context.contentResolver.query( + uri, + FileContents.columns, null, null, null, + )?.use { cursor -> + if (cursor.moveToFirst()) { + FileContents.readFrom(cursor) + } else { + null + } + } + } + + override val name: String? by lazy { content?.name } + override val isDirectory: Boolean by lazy { content?.isDirectory ?: false } + override val isFile: Boolean by lazy { content?.isFile ?: false } + override val length: Long by lazy { content?.length ?: 0L } + override val lastModified: Long by lazy { content?.lastModified ?: 0L } +} diff --git a/documentfile/src/main/kotlin/voice/documentfile/RealCachedDocumentFileFactory.kt b/documentfile/src/main/kotlin/voice/documentfile/RealCachedDocumentFileFactory.kt new file mode 100644 index 0000000000..742d9262e5 --- /dev/null +++ b/documentfile/src/main/kotlin/voice/documentfile/RealCachedDocumentFileFactory.kt @@ -0,0 +1,17 @@ +package voice.documentfile + +import android.content.Context +import android.net.Uri +import com.squareup.anvil.annotations.ContributesBinding +import voice.common.AppScope +import javax.inject.Inject + +@ContributesBinding(AppScope::class) +class RealCachedDocumentFileFactory +@Inject constructor( + private val context: Context, +) : CachedDocumentFileFactory { + override fun create(uri: Uri): CachedDocumentFile { + return RealCachedDocumentFile(context = context, uri = uri, preFilledContent = null) + } +} diff --git a/documentfile/src/main/kotlin/voice/documentfile/Walk.kt b/documentfile/src/main/kotlin/voice/documentfile/Walk.kt new file mode 100644 index 0000000000..c5bc261e50 --- /dev/null +++ b/documentfile/src/main/kotlin/voice/documentfile/Walk.kt @@ -0,0 +1,11 @@ +package voice.documentfile + +fun CachedDocumentFile.walk(): Sequence = sequence { + suspend fun SequenceScope.walk(file: CachedDocumentFile) { + yield(file) + if (file.isDirectory) { + file.children.forEach { walk(it) } + } + } + walk(this@walk) +} diff --git a/folderPicker/build.gradle.kts b/folderPicker/build.gradle.kts index b18e8ea601..ce06e66420 100644 --- a/folderPicker/build.gradle.kts +++ b/folderPicker/build.gradle.kts @@ -14,6 +14,7 @@ dependencies { implementation(projects.playback) implementation(projects.data) implementation(projects.sleepTimer) + implementation(projects.documentfile) implementation(libs.datastore) implementation(libs.coil) diff --git a/folderPicker/src/main/kotlin/voice/folderPicker/folderPicker/FolderPickerViewModel.kt b/folderPicker/src/main/kotlin/voice/folderPicker/folderPicker/FolderPickerViewModel.kt index 226efdd84b..e500fb7162 100644 --- a/folderPicker/src/main/kotlin/voice/folderPicker/folderPicker/FolderPickerViewModel.kt +++ b/folderPicker/src/main/kotlin/voice/folderPicker/folderPicker/FolderPickerViewModel.kt @@ -5,7 +5,6 @@ import androidx.compose.runtime.Composable import androidx.compose.runtime.collectAsState import androidx.compose.runtime.getValue import androidx.compose.runtime.remember -import androidx.documentfile.provider.DocumentFile import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.map @@ -14,6 +13,7 @@ import voice.common.navigation.Destination import voice.common.navigation.Navigator import voice.data.folders.AudiobookFolders import voice.data.folders.FolderType +import voice.documentfile.nameWithoutExtension import javax.inject.Inject class FolderPickerViewModel @@ -36,7 +36,7 @@ class FolderPickerViewModel folders.flatMap { (folderType, folders) -> folders.map { (documentFile, uri) -> FolderPickerViewState.Item( - name = documentFile.displayName(), + name = documentFile.nameWithoutExtension(), id = uri, folderType = folderType, ) @@ -61,18 +61,3 @@ class FolderPickerViewModel audiobookFolders.remove(item.id, item.folderType) } } - -private fun DocumentFile.displayName(): String { - val name = name - return if (name == null) { - uri.pathSegments.lastOrNull() - ?.dropWhile { it != ':' } - ?.removePrefix(":") - ?.takeUnless { it.isBlank() } - ?: uri.toString() - } else { - name.substringBeforeLast(".") - .takeUnless { it.isEmpty() } - ?: name - } -} diff --git a/folderPicker/src/main/kotlin/voice/folderPicker/selectType/DocumentFileCache.kt b/folderPicker/src/main/kotlin/voice/folderPicker/selectType/DocumentFileCache.kt deleted file mode 100644 index 8c629d6de1..0000000000 --- a/folderPicker/src/main/kotlin/voice/folderPicker/selectType/DocumentFileCache.kt +++ /dev/null @@ -1,50 +0,0 @@ -package voice.folderPicker.selectType - -import android.net.Uri -import androidx.documentfile.provider.DocumentFile -import voice.data.supportedAudioFormats - -internal class DocumentFileCache { - - private val cache = mutableMapOf() - - fun DocumentFile.cached(): CachedDocumentFile { - return cache.getOrPut(uri) { CachedDocumentFile(this) } - } - - inner class CachedDocumentFile(private val documentFile: DocumentFile) { - - val children: List by lazy { - documentFile.listFiles().map { it.cached() } - } - - val name: String? by lazy { documentFile.name } - - val isDirectory: Boolean by lazy { documentFile.isDirectory } - private val isFile: Boolean by lazy { documentFile.isFile } - val length: Long by lazy { documentFile.length() } - - private fun walk(): Sequence = sequence { - suspend fun SequenceScope.walk(file: CachedDocumentFile) { - yield(file) - if (file.isDirectory) { - file.children.forEach { walk(it) } - } - } - walk(this@CachedDocumentFile) - } - - fun isAudioFile(): Boolean { - if (!isFile) return false - val name = name ?: return false - val extension = name.substringAfterLast(".").lowercase() - return extension in supportedAudioFormats - } - - fun audioFileCount(): Int { - return walk().count { - it.isAudioFile() - } - } - } -} diff --git a/folderPicker/src/main/kotlin/voice/folderPicker/selectType/SelectFolderType.kt b/folderPicker/src/main/kotlin/voice/folderPicker/selectType/SelectFolderType.kt index d900ae9304..684cef0a13 100644 --- a/folderPicker/src/main/kotlin/voice/folderPicker/selectType/SelectFolderType.kt +++ b/folderPicker/src/main/kotlin/voice/folderPicker/selectType/SelectFolderType.kt @@ -48,7 +48,7 @@ fun SelectFolderType(uri: Uri) { val viewModel = rememberScoped { rootComponentAs().selectFolderTypeViewModel } - val documentFile = DocumentFile.fromTreeUri(LocalContext.current, uri) ?: return + val documentFile = DocumentFile.fromTreeUri(LocalContext.current, uri)!! viewModel.args = SelectFolderTypeViewModel.Args(uri, documentFile) val viewState = viewModel.viewState() SelectFolderType( diff --git a/folderPicker/src/main/kotlin/voice/folderPicker/selectType/SelectFolderTypeViewModel.kt b/folderPicker/src/main/kotlin/voice/folderPicker/selectType/SelectFolderTypeViewModel.kt index 96dd43acc7..8dbed4fb56 100644 --- a/folderPicker/src/main/kotlin/voice/folderPicker/selectType/SelectFolderTypeViewModel.kt +++ b/folderPicker/src/main/kotlin/voice/folderPicker/selectType/SelectFolderTypeViewModel.kt @@ -12,8 +12,13 @@ import androidx.documentfile.provider.DocumentFile import kotlinx.coroutines.withContext import voice.common.DispatcherProvider import voice.common.navigation.Navigator +import voice.data.audioFileCount import voice.data.folders.AudiobookFolders import voice.data.folders.FolderType +import voice.data.isAudioFile +import voice.documentfile.CachedDocumentFile +import voice.documentfile.CachedDocumentFileFactory +import voice.documentfile.nameWithoutExtension import javax.inject.Inject class SelectFolderTypeViewModel @@ -21,13 +26,14 @@ class SelectFolderTypeViewModel private val dispatcherProvider: DispatcherProvider, private val audiobookFolders: AudiobookFolders, private val navigator: Navigator, + private val documentFileFactory: CachedDocumentFileFactory, ) { internal lateinit var args: Args private var selectedFolderMode: MutableState = mutableStateOf(null) - private fun DocumentFileCache.CachedDocumentFile.defaultFolderMode(): FolderMode { + private fun CachedDocumentFile.defaultFolderMode(): FolderMode { return when { children.any { it.isAudioFile() } && children.any { it.isDirectory } -> { FolderMode.Audiobooks @@ -64,10 +70,8 @@ class SelectFolderTypeViewModel @Composable internal fun viewState(): SelectFolderTypeViewState { - val documentFile: DocumentFileCache.CachedDocumentFile = remember { - with(DocumentFileCache()) { - args.documentFile.cached() - } + val documentFile: CachedDocumentFile = remember { + documentFileFactory.create(args.documentFile.uri) } val selectedFolderMode = selectedFolderMode.value ?: documentFile.defaultFolderMode().also { selectedFolderMode.value = it @@ -114,10 +118,3 @@ class SelectFolderTypeViewModel val documentFile: DocumentFile, ) } - -private fun DocumentFileCache.CachedDocumentFile.nameWithoutExtension(): String { - val name = name ?: return "" - return name.substringBeforeLast(".") - .takeUnless { it.isBlank() } - ?: name -} diff --git a/folderPicker/src/test/kotlin/voice/folderPicker/selectType/SelectFolderTypeViewModelTest.kt b/folderPicker/src/test/kotlin/voice/folderPicker/selectType/SelectFolderTypeViewModelTest.kt index 975b1d7305..2105ba88aa 100644 --- a/folderPicker/src/test/kotlin/voice/folderPicker/selectType/SelectFolderTypeViewModelTest.kt +++ b/folderPicker/src/test/kotlin/voice/folderPicker/selectType/SelectFolderTypeViewModelTest.kt @@ -15,6 +15,7 @@ import org.junit.Test import org.junit.rules.TemporaryFolder import org.junit.runner.RunWith import voice.common.DispatcherProvider +import voice.documentfile.FileBasedDocumentFactory @RunWith(AndroidJUnit4::class) class SelectFolderTypeViewModelTest { @@ -31,7 +32,8 @@ class SelectFolderTypeViewModelTest { newFile("audiobooks/SecondBook/1.mp3") newFile("audiobooks/SecondBook/2.mp3") } - val viewModel = SelectFolderTypeViewModel(DispatcherProvider(coroutineContext, coroutineContext), mockk(), mockk()) + val viewModel = + SelectFolderTypeViewModel(DispatcherProvider(coroutineContext, coroutineContext), mockk(), mockk(), FileBasedDocumentFactory) viewModel.args = SelectFolderTypeViewModel.Args(audiobookFolder.toUri(), DocumentFile.fromFile(audiobookFolder)) viewModel.setFolderMode(FolderMode.Audiobooks) diff --git a/new_module.main.kts b/new_module.main.kts new file mode 100755 index 0000000000..9e607a1ab4 --- /dev/null +++ b/new_module.main.kts @@ -0,0 +1,79 @@ +#!/usr/bin/env kotlin +@file:DependsOn("com.github.ajalt.clikt:clikt-jvm:3.5.2") +@file:CompilerOptions("-Xopt-in=kotlin.RequiresOptIn") + +import com.github.ajalt.clikt.core.CliktCommand +import com.github.ajalt.clikt.parameters.arguments.argument +import com.github.ajalt.clikt.parameters.arguments.validate +import java.io.File + +class NewFeatureModule : CliktCommand(name = "Creates a new feature module") { + + private val name: String by argument("The new module name, i.e. :unicorn:wings") + .validate { name -> + val valid = name.startsWith(":") && name.all { it.isLetter() || it == ':' } + if (!valid) { + fail("Invalid module name") + } + } + private val components: List by lazy { + name.removePrefix(":").split(":") + } + + override fun run() { + val moduleRoot = File(components.joinToString(separator = "/")) + + val buildGradle = File(moduleRoot, "build.gradle.kts") + buildGradle.parentFile.mkdirs() + buildGradle.writeText(gradleContent()) + + val packageName = components.joinToString(separator = ".") + listOf("main", "test").forEach { sourceSet -> + val srcFolder = File(moduleRoot, "src/$sourceSet/kotlin/voice/${packageName.replace(".", "/")}") + srcFolder.mkdirs() + } + + addModuleToSettingsGradle() + } + + private fun addModuleToSettingsGradle() { + val settingsGradle = File("settings.gradle.kts") + val lines = settingsGradle.readLines().toMutableList() + if (lines.last().isBlank()) { + lines.removeLast() + } + lines += """include("$name")""" + settingsGradle.writeText(lines.joinToString(separator = "\n")) + } + + private fun String.toSnakeCase(): String { + return "([a-z])([A-Z]+)".toRegex() + .replace(this, "$1_$2") + .lowercase() + } + + private fun gradleContent(): String { + val plugins = buildList { + add("voice.library") + } + + return buildString { + appendLine("plugins {") + plugins.forEach { plugin -> + appendLine(" id(\"$plugin\")") + } + appendLine("}") + + appendLine() + appendLine( + """ + dependencies { + // todo + } + """.trimIndent(), + ) + } + } +} + +NewFeatureModule().main(args) diff --git a/scanner/src/main/kotlin/voice/app/scanner/BookParser.kt b/scanner/src/main/kotlin/voice/app/scanner/BookParser.kt index 8d6e6bd19a..981c85107c 100644 --- a/scanner/src/main/kotlin/voice/app/scanner/BookParser.kt +++ b/scanner/src/main/kotlin/voice/app/scanner/BookParser.kt @@ -1,9 +1,7 @@ package voice.app.scanner import android.app.Application -import android.content.Context import android.net.Uri -import androidx.documentfile.provider.DocumentFile import voice.common.BookId import voice.data.Book import voice.data.BookContent @@ -13,6 +11,8 @@ import voice.data.legacy.LegacyBookSettings import voice.data.repo.BookContentRepo import voice.data.repo.internals.dao.LegacyBookDao import voice.data.toUri +import voice.documentfile.CachedDocumentFile +import voice.documentfile.CachedDocumentFileFactory import voice.logging.core.Logger import java.io.File import java.time.Instant @@ -25,16 +25,14 @@ class BookParser private val legacyBookDao: LegacyBookDao, private val application: Application, private val bookmarkMigrator: BookmarkMigrator, - private val context: Context, + private val fileFactory: CachedDocumentFileFactory, ) { - suspend fun parseAndStore(chapters: List, file: DocumentFile): BookContent { + suspend fun parseAndStore(chapters: List, file: CachedDocumentFile): BookContent { val id = BookId(file.uri) return contentRepo.getOrPut(id) { val uri = chapters.first().id.toUri() - val analyzed = DocumentFile.fromSingleUri(context, uri)?.let { - mediaAnalyzer.analyze(it) - } + val analyzed = mediaAnalyzer.analyze(fileFactory.create(uri)) val filePath = file.uri.filePath() val migrationMetaData = filePath?.let { legacyBookDao.bookMetaData() @@ -97,7 +95,7 @@ class BookParser val playbackPosition: Long, ) - private fun DocumentFile.bookName(): String { + private fun CachedDocumentFile.bookName(): String { val fileName = name return if (fileName == null) { uri.toString() diff --git a/scanner/src/main/kotlin/voice/app/scanner/ChapterParser.kt b/scanner/src/main/kotlin/voice/app/scanner/ChapterParser.kt index d736dcfe3b..c7a4452fb9 100644 --- a/scanner/src/main/kotlin/voice/app/scanner/ChapterParser.kt +++ b/scanner/src/main/kotlin/voice/app/scanner/ChapterParser.kt @@ -1,10 +1,10 @@ package voice.app.scanner -import androidx.documentfile.provider.DocumentFile import voice.data.Chapter import voice.data.ChapterId +import voice.data.isAudioFile import voice.data.repo.ChapterRepo -import voice.data.supportedAudioFormats +import voice.documentfile.CachedDocumentFile import java.time.Instant import javax.inject.Inject @@ -14,18 +14,18 @@ class ChapterParser private val mediaAnalyzer: MediaAnalyzer, ) { - suspend fun parse(documentFile: DocumentFile): List { + suspend fun parse(documentFile: CachedDocumentFile): List { val result = mutableListOf() - suspend fun parseChapters(file: DocumentFile) { + suspend fun parseChapters(file: CachedDocumentFile) { if (file.isAudioFile()) { val id = ChapterId(file.uri) - val chapter = chapterRepo.getOrPut(id, Instant.ofEpochMilli(file.lastModified())) { + val chapter = chapterRepo.getOrPut(id, Instant.ofEpochMilli(file.lastModified)) { val metaData = mediaAnalyzer.analyze(file) ?: return@getOrPut null Chapter( id = id, duration = metaData.duration, - fileLastModified = Instant.ofEpochMilli(file.lastModified()), + fileLastModified = Instant.ofEpochMilli(file.lastModified), name = metaData.chapterName, markData = metaData.chapters, ) @@ -34,7 +34,7 @@ class ChapterParser result.add(chapter) } } else if (file.isDirectory) { - file.listFiles() + file.children .forEach { parseChapters(it) } @@ -45,10 +45,3 @@ class ChapterParser return result.sorted() } } - -private fun DocumentFile.isAudioFile(): Boolean { - if (!isFile) return false - val name = name ?: return false - val extension = name.substringAfterLast(".") - return extension.lowercase() in supportedAudioFormats -} diff --git a/scanner/src/main/kotlin/voice/app/scanner/FFProbeAnalyze.kt b/scanner/src/main/kotlin/voice/app/scanner/FFProbeAnalyze.kt index 257a76ca65..322f33d567 100644 --- a/scanner/src/main/kotlin/voice/app/scanner/FFProbeAnalyze.kt +++ b/scanner/src/main/kotlin/voice/app/scanner/FFProbeAnalyze.kt @@ -1,9 +1,9 @@ package voice.app.scanner import android.content.Context -import androidx.documentfile.provider.DocumentFile import kotlinx.serialization.SerializationException import kotlinx.serialization.json.Json +import voice.documentfile.CachedDocumentFile import voice.ffmpeg.ffprobe import voice.logging.core.Logger import javax.inject.Inject @@ -18,7 +18,7 @@ class FFProbeAnalyze allowStructuredMapKeys = true } - suspend fun analyze(file: DocumentFile): MetaDataScanResult? { + suspend fun analyze(file: CachedDocumentFile): MetaDataScanResult? { val result = ffprobe( input = file.uri, context = context, diff --git a/scanner/src/main/kotlin/voice/app/scanner/MediaAnalyzer.kt b/scanner/src/main/kotlin/voice/app/scanner/MediaAnalyzer.kt index 2db8f08cf1..318f625003 100644 --- a/scanner/src/main/kotlin/voice/app/scanner/MediaAnalyzer.kt +++ b/scanner/src/main/kotlin/voice/app/scanner/MediaAnalyzer.kt @@ -1,7 +1,8 @@ package voice.app.scanner -import androidx.documentfile.provider.DocumentFile import voice.data.MarkData +import voice.documentfile.CachedDocumentFile +import voice.documentfile.nameWithoutExtension import voice.logging.core.Logger import javax.inject.Inject import kotlin.time.Duration.Companion.seconds @@ -11,13 +12,13 @@ class MediaAnalyzer private val ffProbeAnalyze: FFProbeAnalyze, ) { - suspend fun analyze(file: DocumentFile): Metadata? { + suspend fun analyze(file: CachedDocumentFile): Metadata? { val result = ffProbeAnalyze.analyze(file) ?: return null val duration = result.format?.duration return if (duration != null && duration > 0 && result.streams.isNotEmpty()) { Metadata( duration = duration.seconds.inWholeMilliseconds, - chapterName = result.findTag(TagType.Title) ?: file.chapterNameFallback(), + chapterName = result.findTag(TagType.Title) ?: file.nameWithoutExtension(), author = result.findTag(TagType.Artist), bookName = result.findTag(TagType.Album), chapters = result.chapters.mapIndexed { index, metaDataChapter -> @@ -41,11 +42,3 @@ class MediaAnalyzer val chapters: List, ) } - -private fun DocumentFile.chapterNameFallback(): String? { - val name = name ?: return null - return name.substringBeforeLast(".") - .trim() - .takeUnless { it.isEmpty() } - ?: name -} diff --git a/scanner/src/main/kotlin/voice/app/scanner/MediaScanTrigger.kt b/scanner/src/main/kotlin/voice/app/scanner/MediaScanTrigger.kt index c8d5bb4c0f..642b528d04 100644 --- a/scanner/src/main/kotlin/voice/app/scanner/MediaScanTrigger.kt +++ b/scanner/src/main/kotlin/voice/app/scanner/MediaScanTrigger.kt @@ -1,6 +1,5 @@ package voice.app.scanner -import androidx.documentfile.provider.DocumentFile import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Job @@ -12,9 +11,12 @@ import kotlinx.coroutines.launch import voice.data.folders.AudiobookFolders import voice.data.folders.FolderType import voice.data.repo.BookRepository +import voice.documentfile.CachedDocumentFile +import voice.documentfile.CachedDocumentFileFactory import voice.logging.core.Logger import javax.inject.Inject import javax.inject.Singleton +import kotlin.time.measureTime @Singleton class MediaScanTrigger @@ -23,6 +25,7 @@ class MediaScanTrigger private val scanner: MediaScanner, private val coverScanner: CoverScanner, private val bookRepo: BookRepository, + private val documentFileFactory: CachedDocumentFileFactory, ) { private val _scannerActive = MutableStateFlow(false) @@ -41,17 +44,22 @@ class MediaScanTrigger _scannerActive.value = true oldJob?.cancelAndJoin() - val folders: Map> = audiobookFolders.all() - .first() - .mapValues { (_, documentFilesWithUri) -> - documentFilesWithUri.map { it.documentFile } - } - scanner.scan(folders) + measureTime { + val folders: Map> = audiobookFolders.all() + .first() + .mapValues { (_, documentFilesWithUri) -> + documentFilesWithUri.map { + documentFileFactory.create(it.documentFile.uri) + } + } + scanner.scan(folders) + }.also { + Logger.i("scan took $it") + } + _scannerActive.value = false val books = bookRepo.all() coverScanner.scan(books) - - _scannerActive.value = false } } } diff --git a/scanner/src/main/kotlin/voice/app/scanner/MediaScanner.kt b/scanner/src/main/kotlin/voice/app/scanner/MediaScanner.kt index 58bd69e79a..5c6a18f1f4 100644 --- a/scanner/src/main/kotlin/voice/app/scanner/MediaScanner.kt +++ b/scanner/src/main/kotlin/voice/app/scanner/MediaScanner.kt @@ -1,10 +1,13 @@ package voice.app.scanner -import androidx.documentfile.provider.DocumentFile import voice.common.BookId +import voice.data.audioFileCount import voice.data.folders.FolderType import voice.data.repo.BookContentRepo +import voice.documentfile.CachedDocumentFile +import voice.logging.core.Logger import javax.inject.Inject +import kotlin.time.measureTime class MediaScanner @Inject constructor( @@ -13,7 +16,7 @@ class MediaScanner private val bookParser: BookParser, ) { - suspend fun scan(folders: Map>) { + suspend fun scan(folders: Map>) { val files = folders.flatMap { (folderType, files) -> when (folderType) { FolderType.SingleFile, FolderType.SingleFolder -> { @@ -21,7 +24,7 @@ class MediaScanner } FolderType.Root -> { files.flatMap { file -> - file.listFiles().toList() + file.children } } } @@ -29,10 +32,19 @@ class MediaScanner contentRepo.setAllInactiveExcept(files.map { BookId(it.uri) }) - files.forEach { scan(it) } + files + .sortedBy { it.audioFileCount() } + .forEach { file -> + Logger.d("scanning $file") + measureTime { + scan(file) + }.also { + Logger.i("scan took $it for ${file.uri}") + } + } } - private suspend fun scan(file: DocumentFile) { + private suspend fun scan(file: CachedDocumentFile) { val chapters = chapterParser.parse(file) if (chapters.isEmpty()) return diff --git a/scanner/src/test/kotlin/voice/app/scanner/MediaAnalyzerTest.kt b/scanner/src/test/kotlin/voice/app/scanner/MediaAnalyzerTest.kt index 33f35cb626..14190f30f9 100644 --- a/scanner/src/test/kotlin/voice/app/scanner/MediaAnalyzerTest.kt +++ b/scanner/src/test/kotlin/voice/app/scanner/MediaAnalyzerTest.kt @@ -1,11 +1,11 @@ package voice.app.scanner -import androidx.documentfile.provider.DocumentFile import io.kotest.matchers.shouldBe import io.mockk.coEvery import io.mockk.mockk import kotlinx.coroutines.test.runTest import org.junit.Test +import voice.documentfile.FileBasedDocumentFile import java.io.File internal class MediaAnalyzerTest { @@ -15,7 +15,7 @@ internal class MediaAnalyzerTest { @Test fun chapterNameUsed() = runTest { - val file = DocumentFile.fromFile(File("mybook.mp3")) + val file = FileBasedDocumentFile(File("mybook.mp3")) coEvery { ffprobe.analyze(any()) } returns MetaDataScanResult( @@ -30,7 +30,7 @@ internal class MediaAnalyzerTest { @Test fun chapterFallbackDerivedFromFileName() = runTest { - val file = DocumentFile.fromFile(File("mybook.mp3")) + val file = FileBasedDocumentFile(File("mybook.mp3")) coEvery { ffprobe.analyze(any()) } returns MetaDataScanResult( diff --git a/scanner/src/test/kotlin/voice/app/scanner/MediaScannerTest.kt b/scanner/src/test/kotlin/voice/app/scanner/MediaScannerTest.kt index 5cece9acf5..f87097fd85 100644 --- a/scanner/src/test/kotlin/voice/app/scanner/MediaScannerTest.kt +++ b/scanner/src/test/kotlin/voice/app/scanner/MediaScannerTest.kt @@ -6,7 +6,6 @@ import android.net.Uri import android.webkit.MimeTypeMap import androidx.core.net.toFile import androidx.core.net.toUri -import androidx.documentfile.provider.DocumentFile import androidx.room.Room import androidx.test.core.app.ApplicationProvider import androidx.test.ext.junit.runners.AndroidJUnit4 @@ -26,6 +25,8 @@ import voice.data.repo.BookRepository import voice.data.repo.ChapterRepo import voice.data.repo.internals.AppDb import voice.data.toUri +import voice.documentfile.FileBasedDocumentFactory +import voice.documentfile.FileBasedDocumentFile import java.io.Closeable import java.io.File import java.nio.file.Files @@ -141,13 +142,13 @@ class MediaScannerTest { bookParser = BookParser( contentRepo = bookContentRepo, mediaAnalyzer = mediaAnalyzer, - application = ApplicationProvider.getApplicationContext(), legacyBookDao = db.legacyBookDao(), + application = ApplicationProvider.getApplicationContext(), bookmarkMigrator = BookmarkMigrator( legacyBookDao = db.legacyBookDao(), bookmarkDao = db.bookmarkDao(), ), - context = ApplicationProvider.getApplicationContext(), + fileFactory = FileBasedDocumentFactory, ), ) @@ -156,7 +157,7 @@ class MediaScannerTest { private val root: File = Files.createTempDirectory(this::class.java.canonicalName!!).toFile() suspend fun scan(vararg roots: File) { - scanner.scan(mapOf(FolderType.Root to roots.map(DocumentFile::fromFile))) + scanner.scan(mapOf(FolderType.Root to roots.map(::FileBasedDocumentFile))) } fun file(parent: File, name: String): Uri { diff --git a/settings.gradle.kts b/settings.gradle.kts index 4c5970ff17..07093d347c 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -61,3 +61,4 @@ include(":migration") include(":logging:core") include(":logging:debug") include(":logging:crashlytics") +include(":documentfile") \ No newline at end of file