Queue many chapters without probing each one on disk
This commit is contained in:
@@ -1,5 +1,6 @@
|
|||||||
package eu.kanade.tachiyomi.data.download
|
package eu.kanade.tachiyomi.data.download
|
||||||
|
|
||||||
|
import android.content.ContentResolver
|
||||||
import android.content.Context
|
import android.content.Context
|
||||||
import com.hippo.unifile.UniFile
|
import com.hippo.unifile.UniFile
|
||||||
import dev.zacsweers.metro.AppScope
|
import dev.zacsweers.metro.AppScope
|
||||||
@@ -9,6 +10,9 @@ import eu.kanade.tachiyomi.source.Source
|
|||||||
import eu.kanade.tachiyomi.util.lang.Hash.md5
|
import eu.kanade.tachiyomi.util.lang.Hash.md5
|
||||||
import eu.kanade.tachiyomi.util.storage.DiskUtil
|
import eu.kanade.tachiyomi.util.storage.DiskUtil
|
||||||
import kotlinx.coroutines.Dispatchers
|
import kotlinx.coroutines.Dispatchers
|
||||||
|
import kotlinx.coroutines.async
|
||||||
|
import kotlinx.coroutines.awaitAll
|
||||||
|
import kotlinx.coroutines.coroutineScope
|
||||||
import kotlinx.coroutines.withContext
|
import kotlinx.coroutines.withContext
|
||||||
import logcat.LogPriority
|
import logcat.LogPriority
|
||||||
import tachiyomi.core.common.i18n.stringResource
|
import tachiyomi.core.common.i18n.stringResource
|
||||||
@@ -132,11 +136,60 @@ class DownloadProvider(
|
|||||||
source: Source,
|
source: Source,
|
||||||
): Pair<UniFile?, List<UniFile>> = withContext(Dispatchers.IO) {
|
): Pair<UniFile?, List<UniFile>> = withContext(Dispatchers.IO) {
|
||||||
val mangaDir = findMangaDir(manga.title, source) ?: return@withContext null to emptyList()
|
val mangaDir = findMangaDir(manga.title, source) ?: return@withContext null to emptyList()
|
||||||
mangaDir to chapters.mapNotNull { chapter ->
|
mangaDir to findChapterDirs(chapters, mangaDir).values.toList()
|
||||||
getValidChapterDirNames(chapter.name, chapter.scanlator, chapter.url).asSequence()
|
}
|
||||||
.mapNotNull { mangaDir.findFile(it) }
|
|
||||||
.firstOrNull()
|
/**
|
||||||
|
* Returns the ids of the chapters whose download directory exists.
|
||||||
|
*
|
||||||
|
* @param chapters the chapters to query.
|
||||||
|
* @param manga the manga of the chapters.
|
||||||
|
* @param source the source of the chapters.
|
||||||
|
*/
|
||||||
|
suspend fun findDownloadedChapterIds(
|
||||||
|
chapters: List<Chapter>,
|
||||||
|
manga: Manga,
|
||||||
|
source: Source,
|
||||||
|
): Set<Long> = withContext(Dispatchers.IO) {
|
||||||
|
val mangaDir = findMangaDir(manga.title, source) ?: return@withContext emptySet()
|
||||||
|
findChapterDirs(chapters, mangaDir).keys
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns the download directory of each chapter that has one, keyed by chapter id.
|
||||||
|
*
|
||||||
|
* A chapter is looked for under every name it may have been downloaded with, with the chapters looked up in
|
||||||
|
* parallel. On a storage access framework directory each lookup is a content resolver query, so from
|
||||||
|
* [LIST_CHAPTER_DIRS_MIN_CHAPTERS] chapters on the directory is listed once instead and the names are matched
|
||||||
|
* against the listing, ignoring case as the lookups do. Plain file directories are always looked up by name, as
|
||||||
|
* their listing can leave out files another app created.
|
||||||
|
*
|
||||||
|
* @param chapters the chapters to query.
|
||||||
|
* @param mangaDir the download directory of the chapters' manga.
|
||||||
|
*/
|
||||||
|
private suspend fun findChapterDirs(chapters: List<Chapter>, mangaDir: UniFile): Map<Long, UniFile> {
|
||||||
|
val isDocumentTree = mangaDir.uri.scheme == ContentResolver.SCHEME_CONTENT
|
||||||
|
if (isDocumentTree && chapters.size >= LIST_CHAPTER_DIRS_MIN_CHAPTERS) {
|
||||||
|
val chapterDirs = mangaDir.listFiles().orEmpty()
|
||||||
|
.filter { it.name != null }
|
||||||
|
.associateByTo(sortedMapOf(String.CASE_INSENSITIVE_ORDER)) { it.name!! }
|
||||||
|
return chapters
|
||||||
|
.mapNotNull { chapter -> chapter.findDir { chapterDirs[it] }?.let { chapter.id to it } }
|
||||||
|
.toMap()
|
||||||
}
|
}
|
||||||
|
return coroutineScope {
|
||||||
|
chapters
|
||||||
|
.map { chapter ->
|
||||||
|
async(chapterDirLookupDispatcher) { chapter.findDir(mangaDir::findFile)?.let { chapter.id to it } }
|
||||||
|
}
|
||||||
|
.awaitAll()
|
||||||
|
.filterNotNull()
|
||||||
|
.toMap()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun Chapter.findDir(find: (String) -> UniFile?): UniFile? {
|
||||||
|
return getValidChapterDirNames(name, scanlator, url).firstNotNullOfOrNull(find)
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -257,6 +310,12 @@ class DownloadProvider(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private const val LIST_CHAPTER_DIRS_MIN_CHAPTERS = 50
|
||||||
|
|
||||||
|
private const val CHAPTER_DIR_LOOKUP_PARALLELISM = 8
|
||||||
|
|
||||||
|
private val chapterDirLookupDispatcher = Dispatchers.IO.limitedParallelism(CHAPTER_DIR_LOOKUP_PARALLELISM)
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The parts of a chapter's directory name, each computed once however many name variations are built from them.
|
* The parts of a chapter's directory name, each computed once however many name variations are built from them.
|
||||||
*/
|
*/
|
||||||
|
|||||||
@@ -357,14 +357,16 @@ class Downloader(
|
|||||||
val source = sourceManager.get(manga.source) as? HttpSource ?: return false
|
val source = sourceManager.get(manga.source) as? HttpSource ?: return false
|
||||||
val wasEmpty = queueState.value.isEmpty()
|
val wasEmpty = queueState.value.isEmpty()
|
||||||
val chaptersToQueue = withContext(Dispatchers.IO) {
|
val chaptersToQueue = withContext(Dispatchers.IO) {
|
||||||
|
val downloadedChapterIds = provider.findDownloadedChapterIds(chapters, manga, source)
|
||||||
|
val queuedChapterIds = queueState.value.mapTo(HashSet()) { it.chapter.id }
|
||||||
chapters
|
chapters
|
||||||
// Filter out those already downloaded.
|
// Filter out those already downloaded.
|
||||||
.filter { provider.findChapterDir(it.name, it.scanlator, it.url, manga.title, source) == null }
|
.filter { it.id !in downloadedChapterIds }
|
||||||
.asSequence()
|
.asSequence()
|
||||||
// Add chapters to queue from the start.
|
// Add chapters to queue from the start.
|
||||||
.sortedByDescending { it.sourceOrder }
|
.sortedByDescending { it.sourceOrder }
|
||||||
// Filter out those already enqueued.
|
// Filter out those already enqueued.
|
||||||
.filter { chapter -> queueState.value.none { it.chapter.id == chapter.id } }
|
.filter { it.id !in queuedChapterIds }
|
||||||
// Create a download for each one.
|
// Create a download for each one.
|
||||||
.map { Download(source, manga, it) }
|
.map { Download(source, manga, it) }
|
||||||
.toList()
|
.toList()
|
||||||
@@ -765,7 +767,8 @@ class Downloader(
|
|||||||
private fun addAllToQueue(downloads: List<Download>) {
|
private fun addAllToQueue(downloads: List<Download>) {
|
||||||
_queueState.update { queue ->
|
_queueState.update { queue ->
|
||||||
// Callers filter off the queue's thread, so a chapter may have been queued in the meantime
|
// Callers filter off the queue's thread, so a chapter may have been queued in the meantime
|
||||||
val toAdd = downloads.filter { download -> queue.none { it.chapter.id == download.chapter.id } }
|
val queuedChapterIds = queue.mapTo(HashSet()) { it.chapter.id }
|
||||||
|
val toAdd = downloads.filter { it.chapter.id !in queuedChapterIds }
|
||||||
toAdd.forEach { download ->
|
toAdd.forEach { download ->
|
||||||
download.status = Download.State.QUEUE
|
download.status = Download.State.QUEUE
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -513,16 +513,7 @@ class LibraryViewModel(
|
|||||||
appScope.launch {
|
appScope.launch {
|
||||||
mangas.forEach { manga ->
|
mangas.forEach { manga ->
|
||||||
val chapters = getNextChapters.await(manga.id)
|
val chapters = getNextChapters.await(manga.id)
|
||||||
.fastFilterNot { chapter ->
|
.filterNotQueuedOrDownloaded(manga)
|
||||||
downloadManager.getQueuedDownloadOrNull(chapter.id) != null ||
|
|
||||||
downloadManager.isChapterDownloaded(
|
|
||||||
chapter.name,
|
|
||||||
chapter.scanlator,
|
|
||||||
chapter.url,
|
|
||||||
manga.title,
|
|
||||||
manga.source,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
.let { if (amount != null) it.take(amount) else it }
|
.let { if (amount != null) it.take(amount) else it }
|
||||||
|
|
||||||
downloadManager.downloadChapters(manga, chapters)
|
downloadManager.downloadChapters(manga, chapters)
|
||||||
@@ -535,21 +526,18 @@ class LibraryViewModel(
|
|||||||
appScope.launch {
|
appScope.launch {
|
||||||
mangas.forEach { manga ->
|
mangas.forEach { manga ->
|
||||||
val chapters = getBookmarkedChaptersByMangaId.await(manga.id)
|
val chapters = getBookmarkedChaptersByMangaId.await(manga.id)
|
||||||
.fastFilterNot { chapter ->
|
.filterNotQueuedOrDownloaded(manga)
|
||||||
downloadManager.getQueuedDownloadOrNull(chapter.id) != null ||
|
|
||||||
downloadManager.isChapterDownloaded(
|
|
||||||
chapter.name,
|
|
||||||
chapter.scanlator,
|
|
||||||
chapter.url,
|
|
||||||
manga.title,
|
|
||||||
manga.source,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
downloadManager.downloadChapters(manga, chapters)
|
downloadManager.downloadChapters(manga, chapters)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private fun List<Chapter>.filterNotQueuedOrDownloaded(manga: Manga): List<Chapter> {
|
||||||
|
val queuedDownloads = downloadManager.getQueuedDownloadsByChapterId()
|
||||||
|
val downloadedChapterIds = downloadManager.getDownloadedChapterIds(this, manga)
|
||||||
|
return fastFilterNot { it.id in queuedDownloads || it.id in downloadedChapterIds }
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Marks mangas' chapters read status.
|
* Marks mangas' chapters read status.
|
||||||
*/
|
*/
|
||||||
|
|||||||
Reference in New Issue
Block a user