Stream entries when creating and restoring backups
Closes #3850 Co-authored-by: NGB-Was-Taken <76197326+NGB-Was-Taken@users.noreply.github.com>
This commit is contained in:
@@ -32,6 +32,9 @@ The format is a modified version of [Keep a Changelog](https://keepachangelog.co
|
|||||||
### Changed
|
### Changed
|
||||||
- Use a new decoder for images Android can't decode itself, like AVIF, JPEG XL and HEIF, and support JPEG 2000 images ([@wwww-wwww](https://github.com/wwww-wwww)) ([#3786](https://github.com/mihonapp/mihon/pull/3786))
|
- Use a new decoder for images Android can't decode itself, like AVIF, JPEG XL and HEIF, and support JPEG 2000 images ([@wwww-wwww](https://github.com/wwww-wwww)) ([#3786](https://github.com/mihonapp/mihon/pull/3786))
|
||||||
|
|
||||||
|
### Improved
|
||||||
|
- Create and restore backups faster and with less memory ([@NGB-Was-Taken](https://github.com/NGB-Was-Taken), [@AntsyLich](https://github.com/AntsyLich))
|
||||||
|
|
||||||
### Removed
|
### Removed
|
||||||
- Remove "Custom hardware bitmap threshold" and "Custom display profile" advanced settings ([@wwww-wwww](https://github.com/wwww-wwww)) ([#3786](https://github.com/mihonapp/mihon/pull/3786))
|
- Remove "Custom hardware bitmap threshold" and "Custom display profile" advanced settings ([@wwww-wwww](https://github.com/wwww-wwww)) ([#3786](https://github.com/mihonapp/mihon/pull/3786))
|
||||||
|
|
||||||
|
|||||||
@@ -4,8 +4,11 @@ import android.content.Context
|
|||||||
import android.net.Uri
|
import android.net.Uri
|
||||||
import dev.zacsweers.metro.Inject
|
import dev.zacsweers.metro.Inject
|
||||||
import eu.kanade.tachiyomi.data.backup.models.Backup
|
import eu.kanade.tachiyomi.data.backup.models.Backup
|
||||||
import kotlinx.serialization.SerializationException
|
import eu.kanade.tachiyomi.data.backup.models.BackupManga
|
||||||
|
import kotlinx.coroutines.flow.Flow
|
||||||
|
import kotlinx.coroutines.flow.flow
|
||||||
import kotlinx.serialization.protobuf.ProtoBuf
|
import kotlinx.serialization.protobuf.ProtoBuf
|
||||||
|
import okio.BufferedSource
|
||||||
import okio.buffer
|
import okio.buffer
|
||||||
import okio.gzip
|
import okio.gzip
|
||||||
import okio.source
|
import okio.source
|
||||||
@@ -19,9 +22,38 @@ class BackupDecoder(
|
|||||||
private val parser: ProtoBuf,
|
private val parser: ProtoBuf,
|
||||||
) {
|
) {
|
||||||
/**
|
/**
|
||||||
* Decode a potentially-gzipped backup.
|
* Decode everything but the entries of a potentially-gzipped backup, counting the entries without decoding them.
|
||||||
*/
|
*/
|
||||||
fun decode(uri: Uri): Backup {
|
fun decodeMetadata(uri: Uri): Metadata {
|
||||||
|
return read(uri) { reader ->
|
||||||
|
orInvalid {
|
||||||
|
var entryCount = 0
|
||||||
|
while (reader.nextEntry()) {
|
||||||
|
reader.skipEntry()
|
||||||
|
entryCount++
|
||||||
|
}
|
||||||
|
Metadata(entryCount, reader.readMetadata())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Decode the entries of a potentially-gzipped backup one at a time, skipping those whose index [include] rejects.
|
||||||
|
*/
|
||||||
|
fun decodeEntries(uri: Uri, include: (index: Int) -> Boolean = { true }): Flow<BackupManga> = flow {
|
||||||
|
read(uri) { reader ->
|
||||||
|
var index = 0
|
||||||
|
while (orInvalid { reader.nextEntry() }) {
|
||||||
|
if (include(index++)) {
|
||||||
|
emit(orInvalid { reader.readEntry() })
|
||||||
|
} else {
|
||||||
|
orInvalid { reader.skipEntry() }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private inline fun <T> read(uri: Uri, block: (BackupReader) -> T): T {
|
||||||
return context.contentResolver.openInputStream(uri)!!.use { inputStream ->
|
return context.contentResolver.openInputStream(uri)!!.use { inputStream ->
|
||||||
val source = inputStream.source().buffer()
|
val source = inputStream.source().buffer()
|
||||||
|
|
||||||
@@ -29,22 +61,32 @@ class BackupDecoder(
|
|||||||
require(2)
|
require(2)
|
||||||
}
|
}
|
||||||
val id1id2 = peeked.readShort()
|
val id1id2 = peeked.readShort()
|
||||||
val backupString = when (id1id2.toInt()) {
|
val backupSource: BufferedSource = when (id1id2.toInt()) {
|
||||||
0x1f8b -> source.gzip().buffer() // 0x1f8b is gzip magic bytes
|
0x1f8b -> source.gzip().buffer() // 0x1f8b is gzip magic bytes
|
||||||
MAGIC_JSON_SIGNATURE1, MAGIC_JSON_SIGNATURE2, MAGIC_JSON_SIGNATURE3 -> {
|
MAGIC_JSON_SIGNATURE1, MAGIC_JSON_SIGNATURE2, MAGIC_JSON_SIGNATURE3 -> {
|
||||||
throw IOException(context.stringResource(MR.strings.invalid_backup_file_json))
|
throw IOException(context.stringResource(MR.strings.invalid_backup_file_json))
|
||||||
}
|
}
|
||||||
else -> source
|
else -> source
|
||||||
}.use { it.readByteArray() }
|
|
||||||
|
|
||||||
try {
|
|
||||||
parser.decodeFromByteArray(Backup.serializer(), backupString)
|
|
||||||
} catch (_: SerializationException) {
|
|
||||||
throw IOException(context.stringResource(MR.strings.invalid_backup_file_unknown))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
backupSource.use { block(BackupReader(it, parser)) }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Only around reading, so the JSON error above and anything thrown downstream of emit keep their own messages
|
||||||
|
private inline fun <T> orInvalid(block: () -> T): T {
|
||||||
|
return try {
|
||||||
|
block()
|
||||||
|
} catch (e: IOException) {
|
||||||
|
throw IOException(context.stringResource(MR.strings.invalid_backup_file_unknown), e)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
data class Metadata(
|
||||||
|
val entryCount: Int,
|
||||||
|
val backup: Backup,
|
||||||
|
)
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
private const val MAGIC_JSON_SIGNATURE1 = 0x7b7d // `{}`
|
private const val MAGIC_JSON_SIGNATURE1 = 0x7b7d // `{}`
|
||||||
private const val MAGIC_JSON_SIGNATURE2 = 0x7b22 // `{"`
|
private const val MAGIC_JSON_SIGNATURE2 = 0x7b22 // `{"`
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package eu.kanade.tachiyomi.data.backup
|
|||||||
import android.net.Uri
|
import android.net.Uri
|
||||||
import dev.zacsweers.metro.Inject
|
import dev.zacsweers.metro.Inject
|
||||||
import eu.kanade.tachiyomi.data.track.TrackerManager
|
import eu.kanade.tachiyomi.data.track.TrackerManager
|
||||||
|
import kotlinx.coroutines.CancellationException
|
||||||
import tachiyomi.domain.source.service.SourceManager
|
import tachiyomi.domain.source.service.SourceManager
|
||||||
|
|
||||||
@Inject
|
@Inject
|
||||||
@@ -18,8 +19,14 @@ class BackupFileValidator(
|
|||||||
* @return List of missing sources or missing trackers.
|
* @return List of missing sources or missing trackers.
|
||||||
*/
|
*/
|
||||||
suspend fun validate(uri: Uri): Results {
|
suspend fun validate(uri: Uri): Results {
|
||||||
|
val trackerIds = mutableSetOf<Int>()
|
||||||
val backup = try {
|
val backup = try {
|
||||||
backupDecoder.decode(uri)
|
backupDecoder.decodeEntries(uri).collect { manga ->
|
||||||
|
manga.tracking.mapTo(trackerIds) { it.syncId }
|
||||||
|
}
|
||||||
|
backupDecoder.decodeMetadata(uri).backup
|
||||||
|
} catch (e: CancellationException) {
|
||||||
|
throw e
|
||||||
} catch (e: Exception) {
|
} catch (e: Exception) {
|
||||||
throw IllegalStateException(e)
|
throw IllegalStateException(e)
|
||||||
}
|
}
|
||||||
@@ -38,11 +45,7 @@ class BackupFileValidator(
|
|||||||
.distinct()
|
.distinct()
|
||||||
.sorted()
|
.sorted()
|
||||||
|
|
||||||
val trackers = backup.backupManga
|
val missingTrackers = trackerIds
|
||||||
.flatMap { it.tracking }
|
|
||||||
.map { it.syncId }
|
|
||||||
.distinct()
|
|
||||||
val missingTrackers = trackers
|
|
||||||
.mapNotNull { trackerManager.get(it.toLong()) }
|
.mapNotNull { trackerManager.get(it.toLong()) }
|
||||||
.filter { !it.isLoggedIn }
|
.filter { !it.isLoggedIn }
|
||||||
.map { it.name }
|
.map { it.name }
|
||||||
|
|||||||
@@ -0,0 +1,153 @@
|
|||||||
|
package eu.kanade.tachiyomi.data.backup
|
||||||
|
|
||||||
|
import eu.kanade.tachiyomi.data.backup.models.Backup
|
||||||
|
import eu.kanade.tachiyomi.data.backup.models.BackupManga
|
||||||
|
import kotlinx.serialization.DeserializationStrategy
|
||||||
|
import kotlinx.serialization.SerializationException
|
||||||
|
import kotlinx.serialization.protobuf.ProtoBuf
|
||||||
|
import okio.Buffer
|
||||||
|
import okio.BufferedSink
|
||||||
|
import okio.BufferedSource
|
||||||
|
import java.io.IOException
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Writes the same bytes as encoding a whole [Backup], without holding all of its entries at once.
|
||||||
|
*/
|
||||||
|
class BackupWriter(
|
||||||
|
private val sink: BufferedSink,
|
||||||
|
private val parser: ProtoBuf,
|
||||||
|
) {
|
||||||
|
var isEmpty = true
|
||||||
|
private set
|
||||||
|
|
||||||
|
fun writeEntry(entry: BackupManga) {
|
||||||
|
val bytes = parser.encodeToByteArray(BackupManga.serializer(), entry)
|
||||||
|
sink.writeVarint(ENTRY_TAG)
|
||||||
|
sink.writeVarint(bytes.size.toLong())
|
||||||
|
sink.write(bytes)
|
||||||
|
isEmpty = false
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Writes the other fields after the entries, as [metadata] without entries of its own.
|
||||||
|
*/
|
||||||
|
fun writeMetadata(metadata: Backup) {
|
||||||
|
val bytes = parser.encodeToByteArray(Backup.serializer(), metadata)
|
||||||
|
sink.write(bytes)
|
||||||
|
if (bytes.isNotEmpty()) isEmpty = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reads an encoded [Backup] one entry at a time. Every other field is set aside for [readMetadata], wherever it
|
||||||
|
* appears.
|
||||||
|
*
|
||||||
|
* Malformed or truncated input throws an [IOException].
|
||||||
|
*/
|
||||||
|
class BackupReader(
|
||||||
|
private val source: BufferedSource,
|
||||||
|
private val parser: ProtoBuf,
|
||||||
|
) {
|
||||||
|
private val metadata = Buffer()
|
||||||
|
private var entryLength = NO_ENTRY
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Moves to the next entry, which [readEntry] or [skipEntry] then consumes.
|
||||||
|
*
|
||||||
|
* @return false once the input is exhausted.
|
||||||
|
*/
|
||||||
|
fun nextEntry(): Boolean {
|
||||||
|
check(entryLength == NO_ENTRY)
|
||||||
|
while (!source.exhausted()) {
|
||||||
|
val tag = source.readVarint()
|
||||||
|
val fieldNumber = tag ushr 3
|
||||||
|
val wireType = (tag and 0x7).toInt()
|
||||||
|
if (fieldNumber == 0L) throw IOException("Invalid field number 0")
|
||||||
|
|
||||||
|
if (tag == ENTRY_TAG) {
|
||||||
|
entryLength = source.readLength()
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
metadata.writeVarint(tag)
|
||||||
|
when (wireType) {
|
||||||
|
WIRE_TYPE_VARINT -> metadata.writeVarint(source.readVarint())
|
||||||
|
WIRE_TYPE_FIXED64 -> metadata.write(source, 8)
|
||||||
|
WIRE_TYPE_LENGTH_DELIMITED -> {
|
||||||
|
val length = source.readLength()
|
||||||
|
metadata.writeVarint(length)
|
||||||
|
metadata.write(source, length)
|
||||||
|
}
|
||||||
|
WIRE_TYPE_FIXED32 -> metadata.write(source, 4)
|
||||||
|
else -> throw IOException("Unsupported wire type $wireType")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
fun readEntry(): BackupManga {
|
||||||
|
val bytes = source.readByteArray(takeEntryLength())
|
||||||
|
return decode(BackupManga.serializer(), bytes)
|
||||||
|
}
|
||||||
|
|
||||||
|
fun skipEntry() {
|
||||||
|
source.skip(takeEntryLength())
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Skips the remaining entries and decodes everything else as a [Backup] without entries.
|
||||||
|
*/
|
||||||
|
fun readMetadata(): Backup {
|
||||||
|
if (entryLength != NO_ENTRY) skipEntry()
|
||||||
|
while (nextEntry()) skipEntry()
|
||||||
|
return decode(Backup.serializer(), metadata.readByteArray())
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun takeEntryLength(): Long {
|
||||||
|
check(entryLength != NO_ENTRY)
|
||||||
|
return entryLength.also { entryLength = NO_ENTRY }
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun <T> decode(deserializer: DeserializationStrategy<T>, bytes: ByteArray): T {
|
||||||
|
return try {
|
||||||
|
parser.decodeFromByteArray(deserializer, bytes)
|
||||||
|
} catch (e: SerializationException) {
|
||||||
|
throw IOException(e)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun BufferedSource.readVarint(): Long {
|
||||||
|
var result = 0L
|
||||||
|
for (shift in 0 until Long.SIZE_BITS step 7) {
|
||||||
|
val byte = readByte().toInt()
|
||||||
|
result = result or ((byte and 0x7F).toLong() shl shift)
|
||||||
|
if ((byte and 0x80) == 0) return result
|
||||||
|
}
|
||||||
|
throw IOException("Malformed varint")
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun BufferedSource.readLength(): Long {
|
||||||
|
val length = readVarint()
|
||||||
|
if (length !in 0..Int.MAX_VALUE) throw IOException("Invalid length $length")
|
||||||
|
return length
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun BufferedSink.writeVarint(value: Long) {
|
||||||
|
var remaining = value
|
||||||
|
while ((remaining and 0x7FL.inv()) != 0L) {
|
||||||
|
writeByte((remaining and 0x7F).toInt() or 0x80)
|
||||||
|
remaining = remaining ushr 7
|
||||||
|
}
|
||||||
|
writeByte(remaining.toInt())
|
||||||
|
}
|
||||||
|
|
||||||
|
private const val WIRE_TYPE_VARINT = 0
|
||||||
|
private const val WIRE_TYPE_FIXED64 = 1
|
||||||
|
private const val WIRE_TYPE_LENGTH_DELIMITED = 2
|
||||||
|
private const val WIRE_TYPE_FIXED32 = 5
|
||||||
|
|
||||||
|
// Backup.backupManga is field 1, and each entry is length-delimited
|
||||||
|
private const val ENTRY_TAG = 1L shl 3 or WIRE_TYPE_LENGTH_DELIMITED.toLong()
|
||||||
|
|
||||||
|
private const val NO_ENTRY = -1L
|
||||||
@@ -8,6 +8,7 @@ import dev.zacsweers.metro.AssistedFactory
|
|||||||
import dev.zacsweers.metro.AssistedInject
|
import dev.zacsweers.metro.AssistedInject
|
||||||
import eu.kanade.tachiyomi.BuildConfig
|
import eu.kanade.tachiyomi.BuildConfig
|
||||||
import eu.kanade.tachiyomi.data.backup.BackupFileValidator
|
import eu.kanade.tachiyomi.data.backup.BackupFileValidator
|
||||||
|
import eu.kanade.tachiyomi.data.backup.BackupWriter
|
||||||
import eu.kanade.tachiyomi.data.backup.create.creators.CategoriesBackupCreator
|
import eu.kanade.tachiyomi.data.backup.create.creators.CategoriesBackupCreator
|
||||||
import eu.kanade.tachiyomi.data.backup.create.creators.ExtensionStoresBackupCreator
|
import eu.kanade.tachiyomi.data.backup.create.creators.ExtensionStoresBackupCreator
|
||||||
import eu.kanade.tachiyomi.data.backup.create.creators.MangaBackupCreator
|
import eu.kanade.tachiyomi.data.backup.create.creators.MangaBackupCreator
|
||||||
@@ -20,6 +21,8 @@ import eu.kanade.tachiyomi.data.backup.models.BackupManga
|
|||||||
import eu.kanade.tachiyomi.data.backup.models.BackupPreference
|
import eu.kanade.tachiyomi.data.backup.models.BackupPreference
|
||||||
import eu.kanade.tachiyomi.data.backup.models.BackupSource
|
import eu.kanade.tachiyomi.data.backup.models.BackupSource
|
||||||
import eu.kanade.tachiyomi.data.backup.models.BackupSourcePreferences
|
import eu.kanade.tachiyomi.data.backup.models.BackupSourcePreferences
|
||||||
|
import kotlinx.coroutines.flow.Flow
|
||||||
|
import kotlinx.coroutines.flow.emptyFlow
|
||||||
import kotlinx.serialization.protobuf.ProtoBuf
|
import kotlinx.serialization.protobuf.ProtoBuf
|
||||||
import logcat.LogPriority
|
import logcat.LogPriority
|
||||||
import okio.buffer
|
import okio.buffer
|
||||||
@@ -85,27 +88,31 @@ class BackupCreator(
|
|||||||
val nonFavoriteManga = if (options.readEntries) mangaRepository.getReadMangaNotInLibrary() else emptyList()
|
val nonFavoriteManga = if (options.readEntries) mangaRepository.getReadMangaNotInLibrary() else emptyList()
|
||||||
val backupManga = backupMangas(getFavorites.await() + nonFavoriteManga, options)
|
val backupManga = backupMangas(getFavorites.await() + nonFavoriteManga, options)
|
||||||
|
|
||||||
val backup = Backup(
|
|
||||||
backupManga = backupManga,
|
|
||||||
backupCategories = backupCategories(options),
|
|
||||||
backupSources = backupSources(backupManga),
|
|
||||||
backupPreferences = backupAppPreferences(options),
|
|
||||||
backupExtensionStores = backupExtensionStores(options),
|
|
||||||
backupSourcePreferences = backupSourcePreferences(options),
|
|
||||||
)
|
|
||||||
|
|
||||||
val byteArray = parser.encodeToByteArray(Backup.serializer(), backup)
|
|
||||||
if (byteArray.isEmpty()) {
|
|
||||||
throw IllegalStateException(context.stringResource(MR.strings.empty_backup_error))
|
|
||||||
}
|
|
||||||
|
|
||||||
file.openOutputStream()
|
file.openOutputStream()
|
||||||
.also {
|
.also {
|
||||||
// Force overwrite old file
|
// Force overwrite old file
|
||||||
(it as? FileOutputStream)?.channel?.truncate(0)
|
(it as? FileOutputStream)?.channel?.truncate(0)
|
||||||
}
|
}
|
||||||
.sink().gzip().buffer().use {
|
.sink().gzip().buffer().use {
|
||||||
it.write(byteArray)
|
val writer = BackupWriter(it, parser)
|
||||||
|
val sourceIds = mutableSetOf<Long>()
|
||||||
|
backupManga.collect { manga ->
|
||||||
|
writer.writeEntry(manga)
|
||||||
|
sourceIds += manga.source
|
||||||
|
}
|
||||||
|
writer.writeMetadata(
|
||||||
|
Backup(
|
||||||
|
backupManga = emptyList(),
|
||||||
|
backupCategories = backupCategories(options),
|
||||||
|
backupSources = backupSources(sourceIds),
|
||||||
|
backupPreferences = backupAppPreferences(options),
|
||||||
|
backupExtensionStores = backupExtensionStores(options),
|
||||||
|
backupSourcePreferences = backupSourcePreferences(options),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
if (writer.isEmpty) {
|
||||||
|
throw IllegalStateException(context.stringResource(MR.strings.empty_backup_error))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
val fileUri = file.uri
|
val fileUri = file.uri
|
||||||
|
|
||||||
@@ -130,14 +137,14 @@ class BackupCreator(
|
|||||||
return categoriesBackupCreator()
|
return categoriesBackupCreator()
|
||||||
}
|
}
|
||||||
|
|
||||||
private suspend fun backupMangas(mangas: List<Manga>, options: BackupOptions): List<BackupManga> {
|
private fun backupMangas(mangas: List<Manga>, options: BackupOptions): Flow<BackupManga> {
|
||||||
if (!options.libraryEntries) return emptyList()
|
if (!options.libraryEntries) return emptyFlow()
|
||||||
|
|
||||||
return mangaBackupCreator(mangas, options)
|
return mangaBackupCreator(mangas, options)
|
||||||
}
|
}
|
||||||
|
|
||||||
private suspend fun backupSources(mangas: List<BackupManga>): List<BackupSource> {
|
private suspend fun backupSources(sourceIds: Set<Long>): List<BackupSource> {
|
||||||
return sourcesBackupCreator(mangas)
|
return sourcesBackupCreator(sourceIds)
|
||||||
}
|
}
|
||||||
|
|
||||||
private fun backupAppPreferences(options: BackupOptions): List<BackupPreference> {
|
private fun backupAppPreferences(options: BackupOptions): List<BackupPreference> {
|
||||||
|
|||||||
+50
-39
@@ -7,11 +7,13 @@ import eu.kanade.tachiyomi.data.backup.models.BackupManga
|
|||||||
import eu.kanade.tachiyomi.data.backup.models.toBackupChapter
|
import eu.kanade.tachiyomi.data.backup.models.toBackupChapter
|
||||||
import eu.kanade.tachiyomi.data.backup.models.toBackupTracking
|
import eu.kanade.tachiyomi.data.backup.models.toBackupTracking
|
||||||
import eu.kanade.tachiyomi.ui.reader.setting.ReadingMode
|
import eu.kanade.tachiyomi.ui.reader.setting.ReadingMode
|
||||||
|
import kotlinx.coroutines.flow.Flow
|
||||||
|
import kotlinx.coroutines.flow.flow
|
||||||
import mihon.core.common.extensions.toByteArray
|
import mihon.core.common.extensions.toByteArray
|
||||||
import mihon.core.common.extensions.toEpochMillisOrZero
|
import mihon.core.common.extensions.toEpochMillisOrZero
|
||||||
import tachiyomi.domain.category.interactor.GetCategories
|
import tachiyomi.domain.category.repository.CategoryRepository
|
||||||
import tachiyomi.domain.chapter.repository.ChapterRepository
|
import tachiyomi.domain.chapter.repository.ChapterRepository
|
||||||
import tachiyomi.domain.history.interactor.GetHistory
|
import tachiyomi.domain.history.repository.HistoryRepository
|
||||||
import tachiyomi.domain.manga.model.Manga
|
import tachiyomi.domain.manga.model.Manga
|
||||||
import tachiyomi.domain.manga.repository.MangaRepository
|
import tachiyomi.domain.manga.repository.MangaRepository
|
||||||
import tachiyomi.domain.track.repository.TrackRepository
|
import tachiyomi.domain.track.repository.TrackRepository
|
||||||
@@ -20,62 +22,71 @@ import tachiyomi.domain.track.repository.TrackRepository
|
|||||||
class MangaBackupCreator(
|
class MangaBackupCreator(
|
||||||
private val mangaRepository: MangaRepository,
|
private val mangaRepository: MangaRepository,
|
||||||
private val chapterRepository: ChapterRepository,
|
private val chapterRepository: ChapterRepository,
|
||||||
|
private val categoryRepository: CategoryRepository,
|
||||||
|
private val historyRepository: HistoryRepository,
|
||||||
private val trackRepository: TrackRepository,
|
private val trackRepository: TrackRepository,
|
||||||
private val getCategories: GetCategories,
|
|
||||||
private val getHistory: GetHistory,
|
|
||||||
) {
|
) {
|
||||||
|
|
||||||
suspend operator fun invoke(mangas: List<Manga>, options: BackupOptions): List<BackupManga> {
|
/**
|
||||||
return mangas.map {
|
* Builds the entries a chunk at a time, so only one chunk's chapters are held at once.
|
||||||
backupManga(it, options)
|
*/
|
||||||
|
operator fun invoke(mangas: List<Manga>, options: BackupOptions): Flow<BackupManga> = flow {
|
||||||
|
mangas.chunked(CHUNK_SIZE).forEach { chunk ->
|
||||||
|
backupMangas(chunk, options).forEach { emit(it) }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private suspend fun backupManga(manga: Manga, options: BackupOptions): BackupManga {
|
private suspend fun backupMangas(mangas: List<Manga>, options: BackupOptions): List<BackupManga> {
|
||||||
// Entry for this manga
|
val mangaIds = mangas.map { it.id }
|
||||||
val mangaObject = manga.toBackupManga()
|
|
||||||
|
|
||||||
mangaObject.excludedScanlators = mangaRepository.getExcludedScanlators(manga.id).toList()
|
val excludedScanlatorsByMangaId = mangaRepository.getExcludedScanlatorsByMangaIds(mangaIds)
|
||||||
|
// History is backed up by chapter URL, so it needs the chapters too
|
||||||
if (options.chapters) {
|
val chaptersByMangaId = if (options.chapters || options.history) {
|
||||||
// Backup all the chapters
|
chapterRepository.getChaptersByMangaIds(mangaIds)
|
||||||
chapterRepository.getChapterByMangaId(manga.id, applyScanlatorFilter = false)
|
} else {
|
||||||
.map { it.toBackupChapter() }
|
emptyMap()
|
||||||
.takeUnless { it.isEmpty() }
|
|
||||||
?.let { mangaObject.chapters = it }
|
|
||||||
}
|
}
|
||||||
|
val categoriesByMangaId = if (options.categories) {
|
||||||
|
categoryRepository.getCategoriesByMangaIds(mangaIds)
|
||||||
|
} else {
|
||||||
|
emptyMap()
|
||||||
|
}
|
||||||
|
val tracksByMangaId = if (options.tracking) trackRepository.getTracksByMangaIds(mangaIds) else emptyMap()
|
||||||
|
val historyByMangaId = if (options.history) historyRepository.getHistoryByMangaIds(mangaIds) else emptyMap()
|
||||||
|
|
||||||
if (options.categories) {
|
return mangas.map { manga ->
|
||||||
// Backup categories for this manga
|
val mangaObject = manga.toBackupManga()
|
||||||
val categoriesForManga = getCategories.await(manga.id)
|
val chapters = chaptersByMangaId[manga.id].orEmpty()
|
||||||
if (categoriesForManga.isNotEmpty()) {
|
|
||||||
mangaObject.categories = categoriesForManga.map { it.order }
|
mangaObject.excludedScanlators = excludedScanlatorsByMangaId[manga.id].orEmpty()
|
||||||
|
|
||||||
|
if (options.chapters) {
|
||||||
|
mangaObject.chapters = chapters.map { it.toBackupChapter() }
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if (options.tracking) {
|
if (options.categories) {
|
||||||
val tracks = trackRepository.getTracksByMangaId(manga.id).map { it.toBackupTracking() }
|
mangaObject.categories = categoriesByMangaId[manga.id].orEmpty().map { it.order }
|
||||||
if (tracks.isNotEmpty()) {
|
|
||||||
mangaObject.tracking = tracks
|
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if (options.history) {
|
if (options.tracking) {
|
||||||
val historyByMangaId = getHistory.await(manga.id)
|
mangaObject.tracking = tracksByMangaId[manga.id].orEmpty().map { it.toBackupTracking() }
|
||||||
if (historyByMangaId.isNotEmpty()) {
|
}
|
||||||
val chapterUrlsById = chapterRepository.getChapterByMangaId(manga.id).associate { it.id to it.url }
|
|
||||||
val history = historyByMangaId.mapNotNull { history ->
|
if (options.history) {
|
||||||
|
val chapterUrlsById = chapters.associate { it.id to it.url }
|
||||||
|
mangaObject.history = historyByMangaId[manga.id].orEmpty().mapNotNull { history ->
|
||||||
// A chapter removed since its history was read takes that history with it
|
// A chapter removed since its history was read takes that history with it
|
||||||
val url = chapterUrlsById[history.chapterId] ?: return@mapNotNull null
|
val url = chapterUrlsById[history.chapterId] ?: return@mapNotNull null
|
||||||
BackupHistory(url, history.readAt.toEpochMillisOrZero(), history.readDuration)
|
BackupHistory(url, history.readAt.toEpochMillisOrZero(), history.readDuration)
|
||||||
}
|
}
|
||||||
if (history.isNotEmpty()) {
|
|
||||||
mangaObject.history = history
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
return mangaObject
|
mangaObject
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private companion object {
|
||||||
|
const val CHUNK_SIZE = 100
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+2
-6
@@ -1,7 +1,6 @@
|
|||||||
package eu.kanade.tachiyomi.data.backup.create.creators
|
package eu.kanade.tachiyomi.data.backup.create.creators
|
||||||
|
|
||||||
import dev.zacsweers.metro.Inject
|
import dev.zacsweers.metro.Inject
|
||||||
import eu.kanade.tachiyomi.data.backup.models.BackupManga
|
|
||||||
import eu.kanade.tachiyomi.data.backup.models.BackupSource
|
import eu.kanade.tachiyomi.data.backup.models.BackupSource
|
||||||
import eu.kanade.tachiyomi.source.Source
|
import eu.kanade.tachiyomi.source.Source
|
||||||
import tachiyomi.domain.source.service.SourceManager
|
import tachiyomi.domain.source.service.SourceManager
|
||||||
@@ -11,11 +10,8 @@ class SourcesBackupCreator(
|
|||||||
private val sourceManager: SourceManager,
|
private val sourceManager: SourceManager,
|
||||||
) {
|
) {
|
||||||
|
|
||||||
suspend operator fun invoke(mangas: List<BackupManga>): List<BackupSource> {
|
suspend operator fun invoke(sourceIds: Set<Long>): List<BackupSource> {
|
||||||
return mangas
|
return sourceIds.map { sourceManager.getOrStub(it).toBackupSource() }
|
||||||
.map(BackupManga::source)
|
|
||||||
.distinct()
|
|
||||||
.map { sourceManager.getOrStub(it).toBackupSource() }
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -9,7 +9,6 @@ import eu.kanade.tachiyomi.data.backup.BackupDecoder
|
|||||||
import eu.kanade.tachiyomi.data.backup.BackupNotifier
|
import eu.kanade.tachiyomi.data.backup.BackupNotifier
|
||||||
import eu.kanade.tachiyomi.data.backup.models.BackupCategory
|
import eu.kanade.tachiyomi.data.backup.models.BackupCategory
|
||||||
import eu.kanade.tachiyomi.data.backup.models.BackupExtensionStore
|
import eu.kanade.tachiyomi.data.backup.models.BackupExtensionStore
|
||||||
import eu.kanade.tachiyomi.data.backup.models.BackupManga
|
|
||||||
import eu.kanade.tachiyomi.data.backup.models.BackupPreference
|
import eu.kanade.tachiyomi.data.backup.models.BackupPreference
|
||||||
import eu.kanade.tachiyomi.data.backup.models.BackupSource
|
import eu.kanade.tachiyomi.data.backup.models.BackupSource
|
||||||
import eu.kanade.tachiyomi.data.backup.models.BackupSourcePreferences
|
import eu.kanade.tachiyomi.data.backup.models.BackupSourcePreferences
|
||||||
@@ -23,6 +22,12 @@ import kotlinx.coroutines.CoroutineScope
|
|||||||
import kotlinx.coroutines.Job
|
import kotlinx.coroutines.Job
|
||||||
import kotlinx.coroutines.coroutineScope
|
import kotlinx.coroutines.coroutineScope
|
||||||
import kotlinx.coroutines.ensureActive
|
import kotlinx.coroutines.ensureActive
|
||||||
|
import kotlinx.coroutines.flow.chunked
|
||||||
|
import kotlinx.coroutines.flow.emitAll
|
||||||
|
import kotlinx.coroutines.flow.filter
|
||||||
|
import kotlinx.coroutines.flow.flow
|
||||||
|
import kotlinx.coroutines.flow.map
|
||||||
|
import kotlinx.coroutines.flow.withIndex
|
||||||
import kotlinx.coroutines.launch
|
import kotlinx.coroutines.launch
|
||||||
import logcat.LogPriority
|
import logcat.LogPriority
|
||||||
import tachiyomi.core.common.i18n.stringResource
|
import tachiyomi.core.common.i18n.stringResource
|
||||||
@@ -32,6 +37,7 @@ import tachiyomi.domain.source.service.SourceManager
|
|||||||
import tachiyomi.i18n.MR
|
import tachiyomi.i18n.MR
|
||||||
import java.io.File
|
import java.io.File
|
||||||
import java.text.SimpleDateFormat
|
import java.text.SimpleDateFormat
|
||||||
|
import java.util.BitSet
|
||||||
import java.util.Date
|
import java.util.Date
|
||||||
import java.util.Locale
|
import java.util.Locale
|
||||||
import java.util.concurrent.CopyOnWriteArrayList
|
import java.util.concurrent.CopyOnWriteArrayList
|
||||||
@@ -97,7 +103,7 @@ class BackupRestorer(
|
|||||||
}
|
}
|
||||||
|
|
||||||
private suspend fun restoreFromFile(uri: Uri, options: RestoreOptions) {
|
private suspend fun restoreFromFile(uri: Uri, options: RestoreOptions) {
|
||||||
val backup = backupDecoder.decode(uri)
|
val (entryCount, backup) = backupDecoder.decodeMetadata(uri)
|
||||||
|
|
||||||
// Store source mapping for error messages
|
// Store source mapping for error messages
|
||||||
val backupMaps = backup.backupSources
|
val backupMaps = backup.backupSources
|
||||||
@@ -105,7 +111,7 @@ class BackupRestorer(
|
|||||||
|
|
||||||
if (options.libraryEntries) {
|
if (options.libraryEntries) {
|
||||||
restoreSourceNames(backupMaps)
|
restoreSourceNames(backupMaps)
|
||||||
restoreAmount += backup.backupManga.size
|
restoreAmount += entryCount
|
||||||
}
|
}
|
||||||
if (options.categories) {
|
if (options.categories) {
|
||||||
restoreAmount += 1
|
restoreAmount += 1
|
||||||
@@ -138,7 +144,7 @@ class BackupRestorer(
|
|||||||
}
|
}
|
||||||
if (options.libraryEntries) {
|
if (options.libraryEntries) {
|
||||||
restoreManga(
|
restoreManga(
|
||||||
backup.backupManga,
|
uri,
|
||||||
if (options.categories) backup.backupCategories else emptyList(),
|
if (options.categories) backup.backupCategories else emptyList(),
|
||||||
restoreCategoriesJob,
|
restoreCategoriesJob,
|
||||||
)
|
)
|
||||||
@@ -175,14 +181,31 @@ class BackupRestorer(
|
|||||||
}
|
}
|
||||||
|
|
||||||
private fun CoroutineScope.restoreManga(
|
private fun CoroutineScope.restoreManga(
|
||||||
backupMangas: List<BackupManga>,
|
uri: Uri,
|
||||||
backupCategories: List<BackupCategory>,
|
backupCategories: List<BackupCategory>,
|
||||||
categoriesRestoreJob: Job?,
|
categoriesRestoreJob: Job?,
|
||||||
) = launch {
|
) = launch {
|
||||||
categoriesRestoreJob?.join()
|
categoriesRestoreJob?.join()
|
||||||
mangaRestorer.sortByNew(backupMangas)
|
|
||||||
|
// New entries go first. Rather than hold the others back in memory, a second pass reads them again.
|
||||||
|
val isExisting = mangaRestorer.getExistingMatcher()
|
||||||
|
val existingIndices = BitSet()
|
||||||
|
val newMangas = backupDecoder.decodeEntries(uri)
|
||||||
|
.withIndex()
|
||||||
|
.filter { (index, manga) ->
|
||||||
|
val exists = isExisting(manga)
|
||||||
|
if (exists) existingIndices.set(index)
|
||||||
|
!exists
|
||||||
|
}
|
||||||
|
.map { it.value }
|
||||||
|
val backupMangas = flow {
|
||||||
|
emitAll(newMangas)
|
||||||
|
emitAll(backupDecoder.decodeEntries(uri, existingIndices::get))
|
||||||
|
}
|
||||||
|
|
||||||
|
backupMangas
|
||||||
.chunked(100)
|
.chunked(100)
|
||||||
.forEach { chunk ->
|
.collect { chunk ->
|
||||||
val restoredAsBatch = try {
|
val restoredAsBatch = try {
|
||||||
ensureActive()
|
ensureActive()
|
||||||
mangaRestorer.restore(chunk, backupCategories)
|
mangaRestorer.restore(chunk, backupCategories)
|
||||||
|
|||||||
+5
-3
@@ -23,11 +23,13 @@ class MangaRestorer(
|
|||||||
private val now = Clock.System.now().toLocalDateTime(timeZone)
|
private val now = Clock.System.now().toLocalDateTime(timeZone)
|
||||||
private val currentFetchWindow = fetchInterval.getWindow(now.date, timeZone)
|
private val currentFetchWindow = fetchInterval.getWindow(now.date, timeZone)
|
||||||
|
|
||||||
suspend fun sortByNew(backupMangas: List<BackupManga>): List<BackupManga> {
|
/**
|
||||||
|
* Returns whether an entry was already in the database at the time of this call.
|
||||||
|
*/
|
||||||
|
suspend fun getExistingMatcher(): (BackupManga) -> Boolean {
|
||||||
val urlsBySource = restoreRepository.getMangaUrlsBySourceId()
|
val urlsBySource = restoreRepository.getMangaUrlsBySourceId()
|
||||||
|
|
||||||
return backupMangas
|
return { it.url in urlsBySource[it.source].orEmpty() }
|
||||||
.sortedBy { it.url in urlsBySource[it.source].orEmpty() }
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -0,0 +1,202 @@
|
|||||||
|
package eu.kanade.tachiyomi.data.backup
|
||||||
|
|
||||||
|
import eu.kanade.tachiyomi.data.backup.models.Backup
|
||||||
|
import eu.kanade.tachiyomi.data.backup.models.BackupCategory
|
||||||
|
import eu.kanade.tachiyomi.data.backup.models.BackupChapter
|
||||||
|
import eu.kanade.tachiyomi.data.backup.models.BackupExtensionStore
|
||||||
|
import eu.kanade.tachiyomi.data.backup.models.BackupHistory
|
||||||
|
import eu.kanade.tachiyomi.data.backup.models.BackupManga
|
||||||
|
import eu.kanade.tachiyomi.data.backup.models.BackupPreference
|
||||||
|
import eu.kanade.tachiyomi.data.backup.models.BackupSource
|
||||||
|
import eu.kanade.tachiyomi.data.backup.models.BackupSourcePreferences
|
||||||
|
import eu.kanade.tachiyomi.data.backup.models.BackupTracking
|
||||||
|
import eu.kanade.tachiyomi.data.backup.models.BooleanPreferenceValue
|
||||||
|
import eu.kanade.tachiyomi.data.backup.models.StringPreferenceValue
|
||||||
|
import eu.kanade.tachiyomi.data.backup.models.StringSetPreferenceValue
|
||||||
|
import io.kotest.matchers.shouldBe
|
||||||
|
import kotlinx.serialization.protobuf.ProtoBuf
|
||||||
|
import okio.Buffer
|
||||||
|
import org.junit.jupiter.api.Test
|
||||||
|
import org.junit.jupiter.api.assertThrows
|
||||||
|
import org.junit.jupiter.api.parallel.Execution
|
||||||
|
import org.junit.jupiter.api.parallel.ExecutionMode
|
||||||
|
import java.io.IOException
|
||||||
|
|
||||||
|
@Execution(ExecutionMode.CONCURRENT)
|
||||||
|
class BackupStreamTest {
|
||||||
|
|
||||||
|
private val parser = ProtoBuf
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `writer output is the plain encoding`() {
|
||||||
|
val backup = backup()
|
||||||
|
|
||||||
|
val bytes = write(backup)
|
||||||
|
|
||||||
|
bytes shouldBe encode(backup)
|
||||||
|
encode(parser.decodeFromByteArray(Backup.serializer(), bytes)) shouldBe encode(backup)
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `reader reads the plain encoding`() {
|
||||||
|
val backup = backup()
|
||||||
|
|
||||||
|
val (entries, metadata) = read(encode(backup))
|
||||||
|
|
||||||
|
entries.map(::encode) shouldBe backup.backupManga.map(::encode)
|
||||||
|
encode(metadata) shouldBe encode(backup.copy(backupManga = emptyList()))
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `reader counts entries without decoding them`() {
|
||||||
|
val backup = backup()
|
||||||
|
val reader = BackupReader(Buffer().write(encode(backup)), parser)
|
||||||
|
|
||||||
|
var count = 0
|
||||||
|
while (reader.nextEntry()) {
|
||||||
|
reader.skipEntry()
|
||||||
|
count++
|
||||||
|
}
|
||||||
|
|
||||||
|
count shouldBe backup.backupManga.size
|
||||||
|
encode(reader.readMetadata()) shouldBe encode(backup.copy(backupManga = emptyList()))
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `reader takes fields in any order`() {
|
||||||
|
val backup = backup()
|
||||||
|
val (first, second, third) = backup.backupManga
|
||||||
|
val input = Buffer()
|
||||||
|
.write(encode(Backup(emptyList(), backupSources = backup.backupSources)))
|
||||||
|
.write(encode(Backup(listOf(first))))
|
||||||
|
// Fields Backup doesn't know, one of each remaining wire type
|
||||||
|
.writeByte(3 shl 3 or 0).writeByte(0x96).writeByte(0x01)
|
||||||
|
.writeByte(4 shl 3 or 1).writeLongLe(42)
|
||||||
|
.write(encode(Backup(listOf(second), backupCategories = backup.backupCategories)))
|
||||||
|
.writeByte(5 shl 3 or 5).writeIntLe(42)
|
||||||
|
.write(encode(Backup(emptyList(), backupPreferences = backup.backupPreferences)))
|
||||||
|
.write(encode(Backup(listOf(third), backupExtensionStores = backup.backupExtensionStores)))
|
||||||
|
.write(encode(Backup(emptyList(), backupSourcePreferences = backup.backupSourcePreferences)))
|
||||||
|
.readByteArray()
|
||||||
|
|
||||||
|
val (entries, metadata) = read(input)
|
||||||
|
|
||||||
|
entries.map(::encode) shouldBe backup.backupManga.map(::encode)
|
||||||
|
encode(metadata) shouldBe encode(backup.copy(backupManga = emptyList()))
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `empty backup`() {
|
||||||
|
val sink = Buffer()
|
||||||
|
val writer = BackupWriter(sink, parser)
|
||||||
|
writer.writeMetadata(Backup(emptyList()))
|
||||||
|
|
||||||
|
writer.isEmpty shouldBe true
|
||||||
|
sink.size shouldBe 0L
|
||||||
|
|
||||||
|
val (entries, metadata) = read(ByteArray(0))
|
||||||
|
entries shouldBe emptyList()
|
||||||
|
encode(metadata) shouldBe ByteArray(0)
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `truncated input throws`() {
|
||||||
|
val bytes = encode(backup())
|
||||||
|
|
||||||
|
// Cut inside the last entry, inside the metadata, and right after a tag
|
||||||
|
val entryEnd = encode(Backup(backup().backupManga)).size
|
||||||
|
listOf(entryEnd - 1, bytes.size - 1, entryEnd + 1).forEach { size ->
|
||||||
|
assertThrows<IOException> { read(bytes.copyOf(size)) }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `garbage input throws`() {
|
||||||
|
listOf(
|
||||||
|
// Wire type 7 doesn't exist
|
||||||
|
byteArrayOf(0x0F),
|
||||||
|
// Field number 0
|
||||||
|
byteArrayOf(0x02, 0x00),
|
||||||
|
// A varint that never ends
|
||||||
|
ByteArray(11) { 0xFF.toByte() },
|
||||||
|
// An entry longer than the input
|
||||||
|
byteArrayOf(0x0A, 0x10, 0x01),
|
||||||
|
// An entry that isn't a BackupManga
|
||||||
|
byteArrayOf(0x0A, 0x02, 0x0F, 0x00),
|
||||||
|
).forEach { input ->
|
||||||
|
assertThrows<IOException> { read(input) }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun write(backup: Backup): ByteArray {
|
||||||
|
val sink = Buffer()
|
||||||
|
val writer = BackupWriter(sink, parser)
|
||||||
|
backup.backupManga.forEach(writer::writeEntry)
|
||||||
|
writer.writeMetadata(backup.copy(backupManga = emptyList()))
|
||||||
|
writer.isEmpty shouldBe false
|
||||||
|
return sink.readByteArray()
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun read(bytes: ByteArray): Pair<List<BackupManga>, Backup> {
|
||||||
|
val reader = BackupReader(Buffer().write(bytes), parser)
|
||||||
|
val entries = buildList {
|
||||||
|
while (reader.nextEntry()) add(reader.readEntry())
|
||||||
|
}
|
||||||
|
return entries to reader.readMetadata()
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun encode(backup: Backup) = parser.encodeToByteArray(Backup.serializer(), backup)
|
||||||
|
|
||||||
|
private fun encode(manga: BackupManga) = parser.encodeToByteArray(BackupManga.serializer(), manga)
|
||||||
|
|
||||||
|
private fun backup() = Backup(
|
||||||
|
backupManga = listOf(
|
||||||
|
BackupManga(
|
||||||
|
source = 1L,
|
||||||
|
url = "/manga/1",
|
||||||
|
title = "First",
|
||||||
|
genre = listOf("Action", "Drama"),
|
||||||
|
chapters = listOf(
|
||||||
|
BackupChapter(url = "/chapter/1", name = "Chapter 1", read = true, chapterNumber = 1f),
|
||||||
|
BackupChapter(url = "/chapter/2", name = "Chapter 2", bookmark = true, lastPageRead = 4),
|
||||||
|
),
|
||||||
|
categories = listOf(0L, 1L),
|
||||||
|
tracking = listOf(BackupTracking(syncId = 2, libraryId = 3L, title = "First", mediaId = 7L)),
|
||||||
|
history = listOf(BackupHistory(url = "/chapter/1", lastRead = 1_700_000_000_000L, readDuration = 60)),
|
||||||
|
excludedScanlators = listOf("Group"),
|
||||||
|
),
|
||||||
|
BackupManga(source = 2L, url = "/manga/2", title = "Second", favorite = false),
|
||||||
|
BackupManga(
|
||||||
|
source = 1L,
|
||||||
|
url = "/manga/3",
|
||||||
|
title = "Third",
|
||||||
|
description = "x".repeat(300),
|
||||||
|
chapters = List(200) { BackupChapter(url = "/chapter/3/$it", name = "Chapter $it") },
|
||||||
|
),
|
||||||
|
),
|
||||||
|
backupCategories = listOf(
|
||||||
|
BackupCategory(name = "Reading", order = 0),
|
||||||
|
BackupCategory(name = "Done", order = 1),
|
||||||
|
),
|
||||||
|
backupSources = listOf(BackupSource(name = "One", sourceId = 1L), BackupSource(name = "Two", sourceId = 2L)),
|
||||||
|
backupPreferences = listOf(
|
||||||
|
BackupPreference("flag", BooleanPreferenceValue(true)),
|
||||||
|
BackupPreference("tags", StringSetPreferenceValue(setOf("a", "b"))),
|
||||||
|
),
|
||||||
|
backupSourcePreferences = listOf(
|
||||||
|
BackupSourcePreferences("source_1", listOf(BackupPreference("lang", StringPreferenceValue("en")))),
|
||||||
|
),
|
||||||
|
backupExtensionStores = listOf(
|
||||||
|
BackupExtensionStore(
|
||||||
|
indexUrl = "https://example.org/index.min.json",
|
||||||
|
name = "Store",
|
||||||
|
badgeLabel = null,
|
||||||
|
signingKey = "key",
|
||||||
|
contactWebsite = "https://example.org",
|
||||||
|
contactDiscord = null,
|
||||||
|
isLegacy = false,
|
||||||
|
extensionListUrl = null,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
}
|
||||||
@@ -50,6 +50,15 @@ class CategoryRepositoryImpl(
|
|||||||
.subscribeToList()
|
.subscribeToList()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
override suspend fun getCategoriesByMangaIds(mangaIds: List<Long>): Map<Long, List<Category>> {
|
||||||
|
return database.categoryQueries
|
||||||
|
.getCategoriesByMangaIds(mangaIds) { mangaId, id, name, order, flags ->
|
||||||
|
mangaId to mapCategory(id, name, order, flags)
|
||||||
|
}
|
||||||
|
.awaitAsList()
|
||||||
|
.groupBy({ it.first }, { it.second })
|
||||||
|
}
|
||||||
|
|
||||||
override suspend fun insert(category: NewCategory) {
|
override suspend fun insert(category: NewCategory) {
|
||||||
database.categoryQueries.insert(name = category.name, flags = category.flags)
|
database.categoryQueries.insert(name = category.name, flags = category.flags)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -113,6 +113,13 @@ class ChapterRepositoryImpl(
|
|||||||
.awaitAsList()
|
.awaitAsList()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
override suspend fun getChaptersByMangaIds(mangaIds: List<Long>): Map<Long, List<Chapter>> {
|
||||||
|
return database.chapterQueries
|
||||||
|
.getChaptersByMangaIds(mangaIds, ::mapChapter)
|
||||||
|
.awaitAsList()
|
||||||
|
.groupBy { it.mangaId }
|
||||||
|
}
|
||||||
|
|
||||||
override suspend fun getChapterSummaryByMangaId(mangaId: Long): ChapterSummary {
|
override suspend fun getChapterSummaryByMangaId(mangaId: Long): ChapterSummary {
|
||||||
return database.chapterQueries
|
return database.chapterQueries
|
||||||
.getChapterSummaryByMangaId(mangaId) { total, latestChapterNumber ->
|
.getChapterSummaryByMangaId(mangaId) { total, latestChapterNumber ->
|
||||||
|
|||||||
@@ -48,6 +48,15 @@ class HistoryRepositoryImpl(
|
|||||||
.awaitAsList()
|
.awaitAsList()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
override suspend fun getHistoryByMangaIds(mangaIds: List<Long>): Map<Long, List<History>> {
|
||||||
|
return database.historyQueries
|
||||||
|
.getHistoryByMangaIds(mangaIds) { mangaId, id, chapterId, readAt, readDuration ->
|
||||||
|
mangaId to HistoryMapper.mapHistory(id, chapterId, readAt, readDuration)
|
||||||
|
}
|
||||||
|
.awaitAsList()
|
||||||
|
.groupBy({ it.first }, { it.second })
|
||||||
|
}
|
||||||
|
|
||||||
override suspend fun resetHistory(historyId: Long) {
|
override suspend fun resetHistory(historyId: Long) {
|
||||||
try {
|
try {
|
||||||
database.historyQueries.resetHistoryById(historyId)
|
database.historyQueries.resetHistoryById(historyId)
|
||||||
|
|||||||
@@ -144,6 +144,13 @@ class MangaRepositoryImpl(
|
|||||||
.toSet()
|
.toSet()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
override suspend fun getExcludedScanlatorsByMangaIds(mangaIds: List<Long>): Map<Long, List<String>> {
|
||||||
|
return database.excluded_scanlatorQueries
|
||||||
|
.getExcludedScanlatorsByMangaIds(mangaIds)
|
||||||
|
.awaitAsList()
|
||||||
|
.groupBy({ it.manga_id }, { it.scanlator })
|
||||||
|
}
|
||||||
|
|
||||||
override fun getExcludedScanlatorsAsFlow(mangaId: Long): Flow<Set<String>> {
|
override fun getExcludedScanlatorsAsFlow(mangaId: Long): Flow<Set<String>> {
|
||||||
return database.excluded_scanlatorQueries
|
return database.excluded_scanlatorQueries
|
||||||
.getExcludedScanlatorsByMangaId(mangaId)
|
.getExcludedScanlatorsByMangaId(mangaId)
|
||||||
|
|||||||
@@ -31,6 +31,13 @@ class TrackRepositoryImpl(
|
|||||||
.awaitAsList()
|
.awaitAsList()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
override suspend fun getTracksByMangaIds(mangaIds: List<Long>): Map<Long, List<Track>> {
|
||||||
|
return database.manga_trackQueries
|
||||||
|
.getTracksByMangaIds(mangaIds, TrackMapper::mapTrack)
|
||||||
|
.awaitAsList()
|
||||||
|
.groupBy { it.mangaId }
|
||||||
|
}
|
||||||
|
|
||||||
override suspend fun getTracks(): List<Track> {
|
override suspend fun getTracks(): List<Track> {
|
||||||
return database.manga_trackQueries
|
return database.manga_trackQueries
|
||||||
.getTracks(TrackMapper::mapTrack)
|
.getTracks(TrackMapper::mapTrack)
|
||||||
|
|||||||
@@ -34,6 +34,14 @@ ON C.id = MC.category_id
|
|||||||
WHERE MC.manga_id = :mangaId
|
WHERE MC.manga_id = :mangaId
|
||||||
ORDER BY C.`order`;
|
ORDER BY C.`order`;
|
||||||
|
|
||||||
|
getCategoriesByMangaIds:
|
||||||
|
SELECT MC.manga_id, C.id, C.name, C.`order`, C.flags
|
||||||
|
FROM category C
|
||||||
|
JOIN manga_category MC
|
||||||
|
ON C.id = MC.category_id
|
||||||
|
WHERE MC.manga_id IN :mangaIds
|
||||||
|
ORDER BY MC.manga_id, C.`order`;
|
||||||
|
|
||||||
insert:
|
insert:
|
||||||
-- The position is taken inside the insert, so two categories created at once cannot claim the same one
|
-- The position is taken inside the insert, so two categories created at once cannot claim the same one
|
||||||
INSERT INTO category(name, `order`, flags)
|
INSERT INTO category(name, `order`, flags)
|
||||||
|
|||||||
@@ -48,6 +48,16 @@ AND (
|
|||||||
OR ES.scanlator IS NULL
|
OR ES.scanlator IS NULL
|
||||||
);
|
);
|
||||||
|
|
||||||
|
getChaptersByMangaIds:
|
||||||
|
-- The unique index's order, which getChaptersByMangaId returns too, so backups keep their chapter order
|
||||||
|
SELECT
|
||||||
|
id, manga_id, remote_url, remote_name, remote_scanlator, remote_chapter_number,
|
||||||
|
remote_date_upload, remote_order, remote_memo, user_read, user_bookmark,
|
||||||
|
user_last_page_read, state_date_fetch
|
||||||
|
FROM chapter
|
||||||
|
WHERE manga_id IN :mangaIds
|
||||||
|
ORDER BY manga_id, remote_url;
|
||||||
|
|
||||||
getChapterSummaryByMangaId:
|
getChapterSummaryByMangaId:
|
||||||
SELECT count(*) AS total, max(remote_chapter_number) AS latestChapterNumber
|
SELECT count(*) AS total, max(remote_chapter_number) AS latestChapterNumber
|
||||||
FROM chapter
|
FROM chapter
|
||||||
|
|||||||
@@ -21,3 +21,9 @@ getExcludedScanlatorsByMangaId:
|
|||||||
SELECT scanlator
|
SELECT scanlator
|
||||||
FROM excluded_scanlator
|
FROM excluded_scanlator
|
||||||
WHERE manga_id = :mangaId;
|
WHERE manga_id = :mangaId;
|
||||||
|
|
||||||
|
getExcludedScanlatorsByMangaIds:
|
||||||
|
SELECT manga_id, scanlator
|
||||||
|
FROM excluded_scanlator
|
||||||
|
WHERE manga_id IN :mangaIds
|
||||||
|
ORDER BY manga_id, scanlator;
|
||||||
|
|||||||
@@ -20,6 +20,12 @@ SELECT id, chapter_id, read_at, read_duration
|
|||||||
FROM history
|
FROM history
|
||||||
WHERE manga_id = :mangaId;
|
WHERE manga_id = :mangaId;
|
||||||
|
|
||||||
|
getHistoryByMangaIds:
|
||||||
|
SELECT manga_id, id, chapter_id, read_at, read_duration
|
||||||
|
FROM history
|
||||||
|
WHERE manga_id IN :mangaIds
|
||||||
|
ORDER BY manga_id, id;
|
||||||
|
|
||||||
resetHistoryById:
|
resetHistoryById:
|
||||||
UPDATE history
|
UPDATE history
|
||||||
SET read_at = NULL
|
SET read_at = NULL
|
||||||
|
|||||||
@@ -48,6 +48,14 @@ SELECT
|
|||||||
FROM manga_track
|
FROM manga_track
|
||||||
WHERE manga_id = :mangaId;
|
WHERE manga_id = :mangaId;
|
||||||
|
|
||||||
|
getTracksByMangaIds:
|
||||||
|
SELECT
|
||||||
|
id, manga_id, tracker_id, remote_id, remote_url, library_id, title, last_chapter_read,
|
||||||
|
total_chapters, status, score, start_date, finish_date, private
|
||||||
|
FROM manga_track
|
||||||
|
WHERE manga_id IN :mangaIds
|
||||||
|
ORDER BY manga_id, tracker_id;
|
||||||
|
|
||||||
upsert:
|
upsert:
|
||||||
INSERT INTO manga_track(
|
INSERT INTO manga_track(
|
||||||
manga_id, tracker_id, remote_id, remote_url, library_id, title, last_chapter_read,
|
manga_id, tracker_id, remote_id, remote_url, library_id, title, last_chapter_read,
|
||||||
|
|||||||
@@ -16,6 +16,8 @@ interface CategoryRepository {
|
|||||||
|
|
||||||
fun getCategoriesByMangaIdAsFlow(mangaId: Long): Flow<List<Category>>
|
fun getCategoriesByMangaIdAsFlow(mangaId: Long): Flow<List<Category>>
|
||||||
|
|
||||||
|
suspend fun getCategoriesByMangaIds(mangaIds: List<Long>): Map<Long, List<Category>>
|
||||||
|
|
||||||
suspend fun insert(category: NewCategory)
|
suspend fun insert(category: NewCategory)
|
||||||
|
|
||||||
suspend fun insertAll(categories: List<NewCategory>)
|
suspend fun insertAll(categories: List<NewCategory>)
|
||||||
|
|||||||
@@ -20,6 +20,8 @@ interface ChapterRepository {
|
|||||||
|
|
||||||
suspend fun getChapterByMangaId(mangaId: Long, applyScanlatorFilter: Boolean = false): List<Chapter>
|
suspend fun getChapterByMangaId(mangaId: Long, applyScanlatorFilter: Boolean = false): List<Chapter>
|
||||||
|
|
||||||
|
suspend fun getChaptersByMangaIds(mangaIds: List<Long>): Map<Long, List<Chapter>>
|
||||||
|
|
||||||
suspend fun getChapterSummaryByMangaId(mangaId: Long): ChapterSummary
|
suspend fun getChapterSummaryByMangaId(mangaId: Long): ChapterSummary
|
||||||
|
|
||||||
suspend fun getScanlatorsByMangaId(mangaId: Long): List<String>
|
suspend fun getScanlatorsByMangaId(mangaId: Long): List<String>
|
||||||
|
|||||||
@@ -15,6 +15,8 @@ interface HistoryRepository {
|
|||||||
|
|
||||||
suspend fun getHistoryByMangaId(mangaId: Long): List<History>
|
suspend fun getHistoryByMangaId(mangaId: Long): List<History>
|
||||||
|
|
||||||
|
suspend fun getHistoryByMangaIds(mangaIds: List<Long>): Map<Long, List<History>>
|
||||||
|
|
||||||
suspend fun resetHistory(historyId: Long)
|
suspend fun resetHistory(historyId: Long)
|
||||||
|
|
||||||
suspend fun resetHistoryByMangaId(mangaId: Long)
|
suspend fun resetHistoryByMangaId(mangaId: Long)
|
||||||
|
|||||||
@@ -43,6 +43,8 @@ interface MangaRepository {
|
|||||||
|
|
||||||
suspend fun getExcludedScanlators(mangaId: Long): Set<String>
|
suspend fun getExcludedScanlators(mangaId: Long): Set<String>
|
||||||
|
|
||||||
|
suspend fun getExcludedScanlatorsByMangaIds(mangaIds: List<Long>): Map<Long, List<String>>
|
||||||
|
|
||||||
fun getExcludedScanlatorsAsFlow(mangaId: Long): Flow<Set<String>>
|
fun getExcludedScanlatorsAsFlow(mangaId: Long): Flow<Set<String>>
|
||||||
|
|
||||||
suspend fun setExcludedScanlators(mangaId: Long, excludedScanlators: Set<String>)
|
suspend fun setExcludedScanlators(mangaId: Long, excludedScanlators: Set<String>)
|
||||||
|
|||||||
@@ -9,6 +9,8 @@ interface TrackRepository {
|
|||||||
|
|
||||||
suspend fun getTracksByMangaId(mangaId: Long): List<Track>
|
suspend fun getTracksByMangaId(mangaId: Long): List<Track>
|
||||||
|
|
||||||
|
suspend fun getTracksByMangaIds(mangaIds: List<Long>): Map<Long, List<Track>>
|
||||||
|
|
||||||
suspend fun getTracks(): List<Track>
|
suspend fun getTracks(): List<Track>
|
||||||
|
|
||||||
fun getTracksAsFlow(): Flow<List<Track>>
|
fun getTracksAsFlow(): Flow<List<Track>>
|
||||||
|
|||||||
Reference in New Issue
Block a user