Folder picker performance improvements (#1874)

* Improve the performance of the folder picker by querying all the required data at once to reduce the IPC calls.

* Clean up the document file logic.

* Only propagate the scanner as active while searching for books, not for covers.

* Add timing diagnostics.

* Make use of the Caching DocumentFile in the book scanning process to greatly improve the overall performance.

* Simplify the time measurement.

* Prioritize the scanning based on the amount of audio files.

* Abstract the CachedDocumentFile and create a file based implementation that is used in unit tests.

app/build.gradle.kts

@@ -155,6 +155,7 @@ dependencies {
   implementation(projects.migration)
   implementation(projects.search)
   implementation(projects.cover)
+  implementation(projects.documentfile)
 
   implementation(libs.appCompat)
   implementation(libs.recyclerView)

data/build.gradle.kts

@@ -31,6 +31,7 @@ android {
 
 dependencies {
   api(projects.common)
+  api(projects.documentfile)
   implementation(libs.appCompat)
   implementation(libs.androidxCore)
   implementation(libs.serialization.json)

data/src/main/kotlin/voice/data/SupportedAudioFormats.kt

@@ -1,5 +1,8 @@
 package voice.data
 
+import voice.documentfile.CachedDocumentFile
+import voice.documentfile.walk
+
 val supportedAudioFormats = arrayOf(
   "3gp",
   "aac",
@@ -25,3 +28,14 @@ val supportedAudioFormats = arrayOf(
   "webm",
   "xmf",
 )
+
+fun CachedDocumentFile.isAudioFile(): Boolean {
+  if (!isFile) return false
+  val name = name ?: return false
+  val extension = name.substringAfterLast(".").lowercase()
+  return extension in supportedAudioFormats
+}
+
+fun CachedDocumentFile.audioFileCount(): Int {
+  return walk().count { it.isAudioFile() }
+}

data/src/main/kotlin/voice/data/folders/AudiobookFolders.kt

@@ -1,10 +1,10 @@
 package voice.data.folders
 
-import android.app.Application
+import android.content.Context
 import android.content.Intent
 import android.net.Uri
+import android.provider.DocumentsContract
 import androidx.datastore.core.DataStore
-import androidx.documentfile.provider.DocumentFile
 import kotlinx.coroutines.MainScope
 import kotlinx.coroutines.flow.Flow
 import kotlinx.coroutines.flow.combine
@@ -13,6 +13,8 @@ import kotlinx.coroutines.launch
 import voice.common.pref.RootAudiobookFolders
 import voice.common.pref.SingleFileAudiobookFolders
 import voice.common.pref.SingleFolderAudiobookFolders
+import voice.documentfile.CachedDocumentFile
+import voice.documentfile.CachedDocumentFileFactory
 import voice.logging.core.Logger
 import javax.inject.Inject
 
@@ -24,7 +26,8 @@ class AudiobookFolders
   private val singleFolderAudiobookFolders: DataStore<List<@JvmSuppressWildcards Uri>>,
   @SingleFileAudiobookFolders
   private val singleFileAudiobookFolders: DataStore<List<@JvmSuppressWildcards Uri>>,
-  private val application: Application,
+  private val context: Context,
+  private val cachedDocumentFileFactory: CachedDocumentFileFactory,
 ) {
 
   private val scope = MainScope()
@@ -33,11 +36,8 @@ class AudiobookFolders
     val flows = FolderType.values()
       .map { folderType ->
         dataStore(folderType).data.map { uris ->
-          val documentFiles = uris.mapNotNull { uri ->
-            val documentFile = uri.toDocumentFile(folderType)
-            documentFile?.let {
-              DocumentFileWithUri(it, uri)
-            }
+          val documentFiles = uris.map { uri ->
+            DocumentFileWithUri(uri.toDocumentFile(folderType), uri)
           }
           folderType to documentFiles
         }
@@ -47,19 +47,23 @@ class AudiobookFolders
 
   private fun Uri.toDocumentFile(
     folderType: FolderType,
-  ): DocumentFile? {
-    return when (folderType) {
-      FolderType.SingleFile -> {
-        DocumentFile.fromSingleUri(application, this)
-      }
+  ): CachedDocumentFile {
+    val uri = when (folderType) {
+      FolderType.SingleFile -> this
       FolderType.SingleFolder,
       FolderType.Root,
-      -> DocumentFile.fromTreeUri(application, this)
+      -> {
+        DocumentsContract.buildDocumentUriUsingTree(
+          this,
+          DocumentsContract.getTreeDocumentId(this),
+        )
+      }
     }
+    return cachedDocumentFileFactory.create(uri)
   }
 
   fun add(uri: Uri, type: FolderType) {
-    application.contentResolver.takePersistableUriPermission(
+    context.contentResolver.takePersistableUriPermission(
       uri,
       Intent.FLAG_GRANT_READ_URI_PERMISSION or Intent.FLAG_GRANT_WRITE_URI_PERMISSION,
     )
@@ -72,7 +76,7 @@ class AudiobookFolders
 
   fun remove(uri: Uri, folderType: FolderType) {
     try {
-      application.contentResolver.releasePersistableUriPermission(
+      context.contentResolver.releasePersistableUriPermission(
         uri,
         Intent.FLAG_GRANT_READ_URI_PERMISSION or Intent.FLAG_GRANT_WRITE_URI_PERMISSION,
       )
@@ -94,6 +98,6 @@ class AudiobookFolders
 }
 
 data class DocumentFileWithUri(
-  val documentFile: DocumentFile,
+  val documentFile: CachedDocumentFile,
   val uri: Uri,
 )

documentfile/build.gradle.kts

@@ -0,0 +1,15 @@
+plugins {
+  id("voice.library")
+  alias(libs.plugins.anvil)
+
+}
+
+anvil {
+  generateDaggerFactories.set(true)
+}
+
+dependencies {
+  implementation(projects.common)
+  implementation(libs.dagger.core)
+  implementation(libs.androidxCore)
+}

documentfile/src/main/kotlin/voice/documentfile/CachedDocumentFile.kt

@@ -0,0 +1,28 @@
+package voice.documentfile
+
+import android.net.Uri
+
+interface CachedDocumentFile {
+  val children: List<CachedDocumentFile>
+  val name: String?
+  val isDirectory: Boolean
+  val isFile: Boolean
+  val length: Long
+  val lastModified: Long
+  val uri: Uri
+}
+
+fun CachedDocumentFile.nameWithoutExtension(): String {
+  val name = name
+  return if (name == null) {
+    uri.pathSegments.lastOrNull()
+      ?.dropWhile { it != ':' }
+      ?.removePrefix(":")
+      ?.takeUnless { it.isBlank() }
+      ?: uri.toString()
+  } else {
+    name.substringBeforeLast(".")
+      .takeUnless { it.isEmpty() }
+      ?: name
+  }
+}

documentfile/src/main/kotlin/voice/documentfile/CachedDocumentFileFactory.kt

@@ -0,0 +1,7 @@
+package voice.documentfile
+
+import android.net.Uri
+
+interface CachedDocumentFileFactory {
+  fun create(uri: Uri): CachedDocumentFile
+}

documentfile/src/main/kotlin/voice/documentfile/FileBasedDocumentFile.kt

@@ -0,0 +1,25 @@
+package voice.documentfile
+
+import android.net.Uri
+import androidx.core.net.toFile
+import androidx.core.net.toUri
+import java.io.File
+
+data class FileBasedDocumentFile(
+  private val file: File,
+) : CachedDocumentFile {
+
+  override val children: List<CachedDocumentFile> get() = file.listFiles()?.map { FileBasedDocumentFile(it) } ?: emptyList()
+  override val name: String? get() = file.name
+  override val isDirectory: Boolean get() = file.isDirectory
+  override val isFile: Boolean get() = file.isFile
+  override val length: Long get() = file.length()
+  override val lastModified: Long get() = file.lastModified()
+  override val uri: Uri get() = file.toUri()
+}
+
+object FileBasedDocumentFactory : CachedDocumentFileFactory {
+  override fun create(uri: Uri): CachedDocumentFile {
+    return FileBasedDocumentFile(uri.toFile())
+  }
+}

documentfile/src/main/kotlin/voice/documentfile/FileContents.kt

@@ -0,0 +1,39 @@
+package voice.documentfile
+
+import android.database.Cursor
+import android.provider.DocumentsContract
+import androidx.core.database.getLongOrNull
+import androidx.core.database.getStringOrNull
+
+internal data class FileContents(
+  val name: String?,
+  val isFile: Boolean,
+  val isDirectory: Boolean,
+  val length: Long,
+  val lastModified: Long,
+) {
+
+  companion object {
+    val columns = arrayOf(
+      DocumentsContract.Document.COLUMN_DOCUMENT_ID,
+      DocumentsContract.Document.COLUMN_MIME_TYPE,
+      DocumentsContract.Document.COLUMN_DISPLAY_NAME,
+      DocumentsContract.Document.COLUMN_SIZE,
+      DocumentsContract.Document.COLUMN_LAST_MODIFIED,
+    )
+
+    fun readFrom(cursor: Cursor): FileContents {
+      val mimeType = cursor.getStringOrNull(DocumentsContract.Document.COLUMN_MIME_TYPE)
+      return FileContents(
+        name = cursor.getStringOrNull(DocumentsContract.Document.COLUMN_DISPLAY_NAME),
+        isFile = mimeType != null && mimeType != DocumentsContract.Document.MIME_TYPE_DIR,
+        isDirectory = mimeType == DocumentsContract.Document.MIME_TYPE_DIR,
+        length = cursor.getLongOrNull(DocumentsContract.Document.COLUMN_SIZE) ?: 0L,
+        lastModified = cursor.getLongOrNull(DocumentsContract.Document.COLUMN_LAST_MODIFIED) ?: 0L,
+      )
+    }
+  }
+}
+
+private fun Cursor.getStringOrNull(columnName: String): String? = getStringOrNull(getColumnIndexOrThrow(columnName))
+private fun Cursor.getLongOrNull(columnName: String): Long? = getLongOrNull(getColumnIndexOrThrow(columnName))

documentfile/src/main/kotlin/voice/documentfile/ParseContents.kt

@@ -0,0 +1,26 @@
+package voice.documentfile
+
+import android.content.Context
+import android.net.Uri
+import android.provider.DocumentsContract
+import androidx.core.database.getStringOrNull
+
+internal fun parseContents(uri: Uri, context: Context): List<CachedDocumentFile> {
+  val childrenUri = DocumentsContract.buildChildDocumentsUriUsingTree(
+    uri,
+    DocumentsContract.getDocumentId(uri),
+  )
+  return context.contentResolver.query(
+    childrenUri,
+    FileContents.columns, null, null, null,
+  )?.use { cursor ->
+    val files = mutableListOf<CachedDocumentFile>()
+    while (cursor.moveToNext()) {
+      val documentId = cursor.getStringOrNull(cursor.getColumnIndexOrThrow(DocumentsContract.Document.COLUMN_DOCUMENT_ID))
+      val documentUri = DocumentsContract.buildDocumentUriUsingTree(uri, documentId)
+      val contents = FileContents.readFrom(cursor)
+      files += RealCachedDocumentFile(context, documentUri, contents)
+    }
+    files
+  } ?: emptyList()
+}

documentfile/src/main/kotlin/voice/documentfile/RealCachedDocumentFile.kt

@@ -0,0 +1,38 @@
+package voice.documentfile
+
+import android.content.Context
+import android.net.Uri
+
+internal data class RealCachedDocumentFile(
+  val context: Context,
+  override val uri: Uri,
+  private val preFilledContent: FileContents?,
+) : CachedDocumentFile {
+
+  override val children: List<CachedDocumentFile> by lazy {
+    if (isDirectory) {
+      parseContents(uri, context)
+    } else {
+      emptyList()
+    }
+  }
+
+  private val content: FileContents? by lazy {
+    preFilledContent ?: context.contentResolver.query(
+      uri,
+      FileContents.columns, null, null, null,
+    )?.use { cursor ->
+      if (cursor.moveToFirst()) {
+        FileContents.readFrom(cursor)
+      } else {
+        null
+      }
+    }
+  }
+
+  override val name: String? by lazy { content?.name }
+  override val isDirectory: Boolean by lazy { content?.isDirectory ?: false }
+  override val isFile: Boolean by lazy { content?.isFile ?: false }
+  override val length: Long by lazy { content?.length ?: 0L }
+  override val lastModified: Long by lazy { content?.lastModified ?: 0L }
+}

documentfile/src/main/kotlin/voice/documentfile/RealCachedDocumentFileFactory.kt

@@ -0,0 +1,17 @@
+package voice.documentfile
+
+import android.content.Context
+import android.net.Uri
+import com.squareup.anvil.annotations.ContributesBinding
+import voice.common.AppScope
+import javax.inject.Inject
+
+@ContributesBinding(AppScope::class)
+class RealCachedDocumentFileFactory
+@Inject constructor(
+  private val context: Context,
+) : CachedDocumentFileFactory {
+  override fun create(uri: Uri): CachedDocumentFile {
+    return RealCachedDocumentFile(context = context, uri = uri, preFilledContent = null)
+  }
+}

documentfile/src/main/kotlin/voice/documentfile/Walk.kt

@@ -0,0 +1,11 @@
+package voice.documentfile
+
+fun CachedDocumentFile.walk(): Sequence<CachedDocumentFile> = sequence {
+  suspend fun SequenceScope<CachedDocumentFile>.walk(file: CachedDocumentFile) {
+    yield(file)
+    if (file.isDirectory) {
+      file.children.forEach { walk(it) }
+    }
+  }
+  walk(this@walk)
+}

folderPicker/build.gradle.kts

@@ -14,6 +14,7 @@ dependencies {
   implementation(projects.playback)
   implementation(projects.data)
   implementation(projects.sleepTimer)
+  implementation(projects.documentfile)
 
   implementation(libs.datastore)
   implementation(libs.coil)