Queue many chapters without probing each one on disk

This commit is contained in:
AntsyLich
2026-10-11 14:54:32 +06:00
parent ba16158b13
commit f02bf5ba7e
3 changed files with 77 additions and 27 deletions
@@ -1,5 +1,6 @@
package eu.kanade.tachiyomi.data.download package eu.kanade.tachiyomi.data.download
import android.content.ContentResolver
import android.content.Context import android.content.Context
import com.hippo.unifile.UniFile import com.hippo.unifile.UniFile
import dev.zacsweers.metro.AppScope import dev.zacsweers.metro.AppScope
@@ -9,6 +10,9 @@ import eu.kanade.tachiyomi.source.Source
import eu.kanade.tachiyomi.util.lang.Hash.md5 import eu.kanade.tachiyomi.util.lang.Hash.md5
import eu.kanade.tachiyomi.util.storage.DiskUtil import eu.kanade.tachiyomi.util.storage.DiskUtil
import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.async
import kotlinx.coroutines.awaitAll
import kotlinx.coroutines.coroutineScope
import kotlinx.coroutines.withContext import kotlinx.coroutines.withContext
import logcat.LogPriority import logcat.LogPriority
import tachiyomi.core.common.i18n.stringResource import tachiyomi.core.common.i18n.stringResource
@@ -132,11 +136,60 @@ class DownloadProvider(
source: Source, source: Source,
): Pair<UniFile?, List<UniFile>> = withContext(Dispatchers.IO) { ): Pair<UniFile?, List<UniFile>> = withContext(Dispatchers.IO) {
val mangaDir = findMangaDir(manga.title, source) ?: return@withContext null to emptyList() val mangaDir = findMangaDir(manga.title, source) ?: return@withContext null to emptyList()
mangaDir to chapters.mapNotNull { chapter -> mangaDir to findChapterDirs(chapters, mangaDir).values.toList()
getValidChapterDirNames(chapter.name, chapter.scanlator, chapter.url).asSequence() }
.mapNotNull { mangaDir.findFile(it) }
.firstOrNull() /**
* Returns the ids of the chapters whose download directory exists.
*
* @param chapters the chapters to query.
* @param manga the manga of the chapters.
* @param source the source of the chapters.
*/
suspend fun findDownloadedChapterIds(
chapters: List<Chapter>,
manga: Manga,
source: Source,
): Set<Long> = withContext(Dispatchers.IO) {
val mangaDir = findMangaDir(manga.title, source) ?: return@withContext emptySet()
findChapterDirs(chapters, mangaDir).keys
}
/**
* Returns the download directory of each chapter that has one, keyed by chapter id.
*
* A chapter is looked for under every name it may have been downloaded with, with the chapters looked up in
* parallel. On a storage access framework directory each lookup is a content resolver query, so from
* [LIST_CHAPTER_DIRS_MIN_CHAPTERS] chapters on the directory is listed once instead and the names are matched
* against the listing, ignoring case as the lookups do. Plain file directories are always looked up by name, as
* their listing can leave out files another app created.
*
* @param chapters the chapters to query.
* @param mangaDir the download directory of the chapters' manga.
*/
private suspend fun findChapterDirs(chapters: List<Chapter>, mangaDir: UniFile): Map<Long, UniFile> {
val isDocumentTree = mangaDir.uri.scheme == ContentResolver.SCHEME_CONTENT
if (isDocumentTree && chapters.size >= LIST_CHAPTER_DIRS_MIN_CHAPTERS) {
val chapterDirs = mangaDir.listFiles().orEmpty()
.filter { it.name != null }
.associateByTo(sortedMapOf(String.CASE_INSENSITIVE_ORDER)) { it.name!! }
return chapters
.mapNotNull { chapter -> chapter.findDir { chapterDirs[it] }?.let { chapter.id to it } }
.toMap()
} }
return coroutineScope {
chapters
.map { chapter ->
async(chapterDirLookupDispatcher) { chapter.findDir(mangaDir::findFile)?.let { chapter.id to it } }
}
.awaitAll()
.filterNotNull()
.toMap()
}
}
private fun Chapter.findDir(find: (String) -> UniFile?): UniFile? {
return getValidChapterDirNames(name, scanlator, url).firstNotNullOfOrNull(find)
} }
/** /**
@@ -257,6 +310,12 @@ class DownloadProvider(
} }
} }
private const val LIST_CHAPTER_DIRS_MIN_CHAPTERS = 50
private const val CHAPTER_DIR_LOOKUP_PARALLELISM = 8
private val chapterDirLookupDispatcher = Dispatchers.IO.limitedParallelism(CHAPTER_DIR_LOOKUP_PARALLELISM)
/** /**
* The parts of a chapter's directory name, each computed once however many name variations are built from them. * The parts of a chapter's directory name, each computed once however many name variations are built from them.
*/ */
@@ -357,14 +357,16 @@ class Downloader(
val source = sourceManager.get(manga.source) as? HttpSource ?: return false val source = sourceManager.get(manga.source) as? HttpSource ?: return false
val wasEmpty = queueState.value.isEmpty() val wasEmpty = queueState.value.isEmpty()
val chaptersToQueue = withContext(Dispatchers.IO) { val chaptersToQueue = withContext(Dispatchers.IO) {
val downloadedChapterIds = provider.findDownloadedChapterIds(chapters, manga, source)
val queuedChapterIds = queueState.value.mapTo(HashSet()) { it.chapter.id }
chapters chapters
// Filter out those already downloaded. // Filter out those already downloaded.
.filter { provider.findChapterDir(it.name, it.scanlator, it.url, manga.title, source) == null } .filter { it.id !in downloadedChapterIds }
.asSequence() .asSequence()
// Add chapters to queue from the start. // Add chapters to queue from the start.
.sortedByDescending { it.sourceOrder } .sortedByDescending { it.sourceOrder }
// Filter out those already enqueued. // Filter out those already enqueued.
.filter { chapter -> queueState.value.none { it.chapter.id == chapter.id } } .filter { it.id !in queuedChapterIds }
// Create a download for each one. // Create a download for each one.
.map { Download(source, manga, it) } .map { Download(source, manga, it) }
.toList() .toList()
@@ -765,7 +767,8 @@ class Downloader(
private fun addAllToQueue(downloads: List<Download>) { private fun addAllToQueue(downloads: List<Download>) {
_queueState.update { queue -> _queueState.update { queue ->
// Callers filter off the queue's thread, so a chapter may have been queued in the meantime // Callers filter off the queue's thread, so a chapter may have been queued in the meantime
val toAdd = downloads.filter { download -> queue.none { it.chapter.id == download.chapter.id } } val queuedChapterIds = queue.mapTo(HashSet()) { it.chapter.id }
val toAdd = downloads.filter { it.chapter.id !in queuedChapterIds }
toAdd.forEach { download -> toAdd.forEach { download ->
download.status = Download.State.QUEUE download.status = Download.State.QUEUE
} }
@@ -513,16 +513,7 @@ class LibraryViewModel(
appScope.launch { appScope.launch {
mangas.forEach { manga -> mangas.forEach { manga ->
val chapters = getNextChapters.await(manga.id) val chapters = getNextChapters.await(manga.id)
.fastFilterNot { chapter -> .filterNotQueuedOrDownloaded(manga)
downloadManager.getQueuedDownloadOrNull(chapter.id) != null ||
downloadManager.isChapterDownloaded(
chapter.name,
chapter.scanlator,
chapter.url,
manga.title,
manga.source,
)
}
.let { if (amount != null) it.take(amount) else it } .let { if (amount != null) it.take(amount) else it }
downloadManager.downloadChapters(manga, chapters) downloadManager.downloadChapters(manga, chapters)
@@ -535,21 +526,18 @@ class LibraryViewModel(
appScope.launch { appScope.launch {
mangas.forEach { manga -> mangas.forEach { manga ->
val chapters = getBookmarkedChaptersByMangaId.await(manga.id) val chapters = getBookmarkedChaptersByMangaId.await(manga.id)
.fastFilterNot { chapter -> .filterNotQueuedOrDownloaded(manga)
downloadManager.getQueuedDownloadOrNull(chapter.id) != null ||
downloadManager.isChapterDownloaded(
chapter.name,
chapter.scanlator,
chapter.url,
manga.title,
manga.source,
)
}
downloadManager.downloadChapters(manga, chapters) downloadManager.downloadChapters(manga, chapters)
} }
} }
} }
private fun List<Chapter>.filterNotQueuedOrDownloaded(manga: Manga): List<Chapter> {
val queuedDownloads = downloadManager.getQueuedDownloadsByChapterId()
val downloadedChapterIds = downloadManager.getDownloadedChapterIds(this, manga)
return fastFilterNot { it.id in queuedDownloads || it.id in downloadedChapterIds }
}
/** /**
* Marks mangas' chapters read status. * Marks mangas' chapters read status.
*/ */