Add option to customize concurrent downloads, increase page concurrency (#2637)
This commit is contained in:
@@ -191,15 +191,17 @@ class Downloader(
|
||||
if (isRunning) return
|
||||
|
||||
downloaderJob = scope.launch {
|
||||
val activeDownloadsFlow = queueState.transformLatest { queue ->
|
||||
val activeDownloadsFlow = combine(
|
||||
queueState,
|
||||
downloadPreferences.parallelSourceLimit().changes(),
|
||||
) { a, b -> a to b }.transformLatest { (queue, parallelCount) ->
|
||||
while (true) {
|
||||
val activeDownloads = queue.asSequence()
|
||||
// Ignore completed downloads, leave them in the queue
|
||||
.filter { it.status.value <= Download.State.DOWNLOADING.value }
|
||||
.groupBy { it.source }
|
||||
.toList()
|
||||
// Concurrently download from 5 different sources
|
||||
.take(5)
|
||||
.take(parallelCount)
|
||||
.map { (_, downloads) -> downloads.first() }
|
||||
emit(activeDownloads)
|
||||
|
||||
@@ -211,7 +213,8 @@ class Downloader(
|
||||
}.filter { it }
|
||||
activeDownloadsErroredFlow.first()
|
||||
}
|
||||
}.distinctUntilChanged()
|
||||
}
|
||||
.distinctUntilChanged()
|
||||
|
||||
// Use supervisorScope to cancel child jobs when the downloader job is cancelled
|
||||
supervisorScope {
|
||||
@@ -366,24 +369,23 @@ class Downloader(
|
||||
download.status = Download.State.DOWNLOADING
|
||||
|
||||
// Start downloading images, consider we can have downloaded images already
|
||||
// Concurrently do 2 pages at a time
|
||||
pageList.asFlow()
|
||||
.flatMapMerge(concurrency = 2) { page ->
|
||||
flow {
|
||||
// Fetch image URL if necessary
|
||||
if (page.imageUrl.isNullOrEmpty()) {
|
||||
page.status = Page.State.LoadPage
|
||||
try {
|
||||
page.imageUrl = download.source.getImageUrl(page)
|
||||
} catch (e: Throwable) {
|
||||
page.status = Page.State.Error(e)
|
||||
}
|
||||
pageList.asFlow().flatMapMerge(concurrency = downloadPreferences.parallelPageLimit().get()) { page ->
|
||||
flow {
|
||||
// Fetch image URL if necessary
|
||||
if (page.imageUrl.isNullOrEmpty()) {
|
||||
page.status = Page.State.LoadPage
|
||||
try {
|
||||
page.imageUrl = download.source.getImageUrl(page)
|
||||
} catch (e: Throwable) {
|
||||
page.status = Page.State.Error(e)
|
||||
}
|
||||
}
|
||||
|
||||
withIOContext { getOrDownloadImage(page, download, tmpDir) }
|
||||
emit(page)
|
||||
}.flowOn(Dispatchers.IO)
|
||||
withIOContext { getOrDownloadImage(page, download, tmpDir) }
|
||||
emit(page)
|
||||
}
|
||||
.flowOn(Dispatchers.IO)
|
||||
}
|
||||
.collect {
|
||||
// Do when page is downloaded.
|
||||
notifier.onProgressChange(download)
|
||||
|
||||
Reference in New Issue
Block a user