Stream entries when creating and restoring backups

Closes #3850

Co-authored-by: NGB-Was-Taken <76197326+NGB-Was-Taken@users.noreply.github.com>
This commit is contained in:
AntsyLich
2026-10-12 15:55:59 +06:00
parent 5a5ff22c37
commit 771ba33a33
25 changed files with 619 additions and 90 deletions
+3
View File
@@ -32,6 +32,9 @@ The format is a modified version of [Keep a Changelog](https://keepachangelog.co
### Changed ### Changed
- Use a new decoder for images Android can't decode itself, like AVIF, JPEG XL and HEIF, and support JPEG 2000 images ([@wwww-wwww](https://github.com/wwww-wwww)) ([#3786](https://github.com/mihonapp/mihon/pull/3786)) - Use a new decoder for images Android can't decode itself, like AVIF, JPEG XL and HEIF, and support JPEG 2000 images ([@wwww-wwww](https://github.com/wwww-wwww)) ([#3786](https://github.com/mihonapp/mihon/pull/3786))
### Improved
- Create and restore backups faster and with less memory ([@NGB-Was-Taken](https://github.com/NGB-Was-Taken), [@AntsyLich](https://github.com/AntsyLich))
### Removed ### Removed
- Remove "Custom hardware bitmap threshold" and "Custom display profile" advanced settings ([@wwww-wwww](https://github.com/wwww-wwww)) ([#3786](https://github.com/mihonapp/mihon/pull/3786)) - Remove "Custom hardware bitmap threshold" and "Custom display profile" advanced settings ([@wwww-wwww](https://github.com/wwww-wwww)) ([#3786](https://github.com/mihonapp/mihon/pull/3786))
@@ -4,8 +4,11 @@ import android.content.Context
import android.net.Uri import android.net.Uri
import dev.zacsweers.metro.Inject import dev.zacsweers.metro.Inject
import eu.kanade.tachiyomi.data.backup.models.Backup import eu.kanade.tachiyomi.data.backup.models.Backup
import kotlinx.serialization.SerializationException import eu.kanade.tachiyomi.data.backup.models.BackupManga
import kotlinx.coroutines.flow.Flow
import kotlinx.coroutines.flow.flow
import kotlinx.serialization.protobuf.ProtoBuf import kotlinx.serialization.protobuf.ProtoBuf
import okio.BufferedSource
import okio.buffer import okio.buffer
import okio.gzip import okio.gzip
import okio.source import okio.source
@@ -19,9 +22,38 @@ class BackupDecoder(
private val parser: ProtoBuf, private val parser: ProtoBuf,
) { ) {
/** /**
* Decode a potentially-gzipped backup. * Decode everything but the entries of a potentially-gzipped backup, counting the entries without decoding them.
*/ */
fun decode(uri: Uri): Backup { fun decodeMetadata(uri: Uri): Metadata {
return read(uri) { reader ->
orInvalid {
var entryCount = 0
while (reader.nextEntry()) {
reader.skipEntry()
entryCount++
}
Metadata(entryCount, reader.readMetadata())
}
}
}
/**
* Decode the entries of a potentially-gzipped backup one at a time, skipping those whose index [include] rejects.
*/
fun decodeEntries(uri: Uri, include: (index: Int) -> Boolean = { true }): Flow<BackupManga> = flow {
read(uri) { reader ->
var index = 0
while (orInvalid { reader.nextEntry() }) {
if (include(index++)) {
emit(orInvalid { reader.readEntry() })
} else {
orInvalid { reader.skipEntry() }
}
}
}
}
private inline fun <T> read(uri: Uri, block: (BackupReader) -> T): T {
return context.contentResolver.openInputStream(uri)!!.use { inputStream -> return context.contentResolver.openInputStream(uri)!!.use { inputStream ->
val source = inputStream.source().buffer() val source = inputStream.source().buffer()
@@ -29,22 +61,32 @@ class BackupDecoder(
require(2) require(2)
} }
val id1id2 = peeked.readShort() val id1id2 = peeked.readShort()
val backupString = when (id1id2.toInt()) { val backupSource: BufferedSource = when (id1id2.toInt()) {
0x1f8b -> source.gzip().buffer() // 0x1f8b is gzip magic bytes 0x1f8b -> source.gzip().buffer() // 0x1f8b is gzip magic bytes
MAGIC_JSON_SIGNATURE1, MAGIC_JSON_SIGNATURE2, MAGIC_JSON_SIGNATURE3 -> { MAGIC_JSON_SIGNATURE1, MAGIC_JSON_SIGNATURE2, MAGIC_JSON_SIGNATURE3 -> {
throw IOException(context.stringResource(MR.strings.invalid_backup_file_json)) throw IOException(context.stringResource(MR.strings.invalid_backup_file_json))
} }
else -> source else -> source
}.use { it.readByteArray() }
try {
parser.decodeFromByteArray(Backup.serializer(), backupString)
} catch (_: SerializationException) {
throw IOException(context.stringResource(MR.strings.invalid_backup_file_unknown))
} }
backupSource.use { block(BackupReader(it, parser)) }
} }
} }
// Only around reading, so the JSON error above and anything thrown downstream of emit keep their own messages
private inline fun <T> orInvalid(block: () -> T): T {
return try {
block()
} catch (e: IOException) {
throw IOException(context.stringResource(MR.strings.invalid_backup_file_unknown), e)
}
}
data class Metadata(
val entryCount: Int,
val backup: Backup,
)
companion object { companion object {
private const val MAGIC_JSON_SIGNATURE1 = 0x7b7d // `{}` private const val MAGIC_JSON_SIGNATURE1 = 0x7b7d // `{}`
private const val MAGIC_JSON_SIGNATURE2 = 0x7b22 // `{"` private const val MAGIC_JSON_SIGNATURE2 = 0x7b22 // `{"`
@@ -3,6 +3,7 @@ package eu.kanade.tachiyomi.data.backup
import android.net.Uri import android.net.Uri
import dev.zacsweers.metro.Inject import dev.zacsweers.metro.Inject
import eu.kanade.tachiyomi.data.track.TrackerManager import eu.kanade.tachiyomi.data.track.TrackerManager
import kotlinx.coroutines.CancellationException
import tachiyomi.domain.source.service.SourceManager import tachiyomi.domain.source.service.SourceManager
@Inject @Inject
@@ -18,8 +19,14 @@ class BackupFileValidator(
* @return List of missing sources or missing trackers. * @return List of missing sources or missing trackers.
*/ */
suspend fun validate(uri: Uri): Results { suspend fun validate(uri: Uri): Results {
val trackerIds = mutableSetOf<Int>()
val backup = try { val backup = try {
backupDecoder.decode(uri) backupDecoder.decodeEntries(uri).collect { manga ->
manga.tracking.mapTo(trackerIds) { it.syncId }
}
backupDecoder.decodeMetadata(uri).backup
} catch (e: CancellationException) {
throw e
} catch (e: Exception) { } catch (e: Exception) {
throw IllegalStateException(e) throw IllegalStateException(e)
} }
@@ -38,11 +45,7 @@ class BackupFileValidator(
.distinct() .distinct()
.sorted() .sorted()
val trackers = backup.backupManga val missingTrackers = trackerIds
.flatMap { it.tracking }
.map { it.syncId }
.distinct()
val missingTrackers = trackers
.mapNotNull { trackerManager.get(it.toLong()) } .mapNotNull { trackerManager.get(it.toLong()) }
.filter { !it.isLoggedIn } .filter { !it.isLoggedIn }
.map { it.name } .map { it.name }
@@ -0,0 +1,153 @@
package eu.kanade.tachiyomi.data.backup
import eu.kanade.tachiyomi.data.backup.models.Backup
import eu.kanade.tachiyomi.data.backup.models.BackupManga
import kotlinx.serialization.DeserializationStrategy
import kotlinx.serialization.SerializationException
import kotlinx.serialization.protobuf.ProtoBuf
import okio.Buffer
import okio.BufferedSink
import okio.BufferedSource
import java.io.IOException
/**
* Writes the same bytes as encoding a whole [Backup], without holding all of its entries at once.
*/
class BackupWriter(
private val sink: BufferedSink,
private val parser: ProtoBuf,
) {
var isEmpty = true
private set
fun writeEntry(entry: BackupManga) {
val bytes = parser.encodeToByteArray(BackupManga.serializer(), entry)
sink.writeVarint(ENTRY_TAG)
sink.writeVarint(bytes.size.toLong())
sink.write(bytes)
isEmpty = false
}
/**
* Writes the other fields after the entries, as [metadata] without entries of its own.
*/
fun writeMetadata(metadata: Backup) {
val bytes = parser.encodeToByteArray(Backup.serializer(), metadata)
sink.write(bytes)
if (bytes.isNotEmpty()) isEmpty = false
}
}
/**
* Reads an encoded [Backup] one entry at a time. Every other field is set aside for [readMetadata], wherever it
* appears.
*
* Malformed or truncated input throws an [IOException].
*/
class BackupReader(
private val source: BufferedSource,
private val parser: ProtoBuf,
) {
private val metadata = Buffer()
private var entryLength = NO_ENTRY
/**
* Moves to the next entry, which [readEntry] or [skipEntry] then consumes.
*
* @return false once the input is exhausted.
*/
fun nextEntry(): Boolean {
check(entryLength == NO_ENTRY)
while (!source.exhausted()) {
val tag = source.readVarint()
val fieldNumber = tag ushr 3
val wireType = (tag and 0x7).toInt()
if (fieldNumber == 0L) throw IOException("Invalid field number 0")
if (tag == ENTRY_TAG) {
entryLength = source.readLength()
return true
}
metadata.writeVarint(tag)
when (wireType) {
WIRE_TYPE_VARINT -> metadata.writeVarint(source.readVarint())
WIRE_TYPE_FIXED64 -> metadata.write(source, 8)
WIRE_TYPE_LENGTH_DELIMITED -> {
val length = source.readLength()
metadata.writeVarint(length)
metadata.write(source, length)
}
WIRE_TYPE_FIXED32 -> metadata.write(source, 4)
else -> throw IOException("Unsupported wire type $wireType")
}
}
return false
}
fun readEntry(): BackupManga {
val bytes = source.readByteArray(takeEntryLength())
return decode(BackupManga.serializer(), bytes)
}
fun skipEntry() {
source.skip(takeEntryLength())
}
/**
* Skips the remaining entries and decodes everything else as a [Backup] without entries.
*/
fun readMetadata(): Backup {
if (entryLength != NO_ENTRY) skipEntry()
while (nextEntry()) skipEntry()
return decode(Backup.serializer(), metadata.readByteArray())
}
private fun takeEntryLength(): Long {
check(entryLength != NO_ENTRY)
return entryLength.also { entryLength = NO_ENTRY }
}
private fun <T> decode(deserializer: DeserializationStrategy<T>, bytes: ByteArray): T {
return try {
parser.decodeFromByteArray(deserializer, bytes)
} catch (e: SerializationException) {
throw IOException(e)
}
}
}
private fun BufferedSource.readVarint(): Long {
var result = 0L
for (shift in 0 until Long.SIZE_BITS step 7) {
val byte = readByte().toInt()
result = result or ((byte and 0x7F).toLong() shl shift)
if ((byte and 0x80) == 0) return result
}
throw IOException("Malformed varint")
}
private fun BufferedSource.readLength(): Long {
val length = readVarint()
if (length !in 0..Int.MAX_VALUE) throw IOException("Invalid length $length")
return length
}
private fun BufferedSink.writeVarint(value: Long) {
var remaining = value
while ((remaining and 0x7FL.inv()) != 0L) {
writeByte((remaining and 0x7F).toInt() or 0x80)
remaining = remaining ushr 7
}
writeByte(remaining.toInt())
}
private const val WIRE_TYPE_VARINT = 0
private const val WIRE_TYPE_FIXED64 = 1
private const val WIRE_TYPE_LENGTH_DELIMITED = 2
private const val WIRE_TYPE_FIXED32 = 5
// Backup.backupManga is field 1, and each entry is length-delimited
private const val ENTRY_TAG = 1L shl 3 or WIRE_TYPE_LENGTH_DELIMITED.toLong()
private const val NO_ENTRY = -1L
@@ -8,6 +8,7 @@ import dev.zacsweers.metro.AssistedFactory
import dev.zacsweers.metro.AssistedInject import dev.zacsweers.metro.AssistedInject
import eu.kanade.tachiyomi.BuildConfig import eu.kanade.tachiyomi.BuildConfig
import eu.kanade.tachiyomi.data.backup.BackupFileValidator import eu.kanade.tachiyomi.data.backup.BackupFileValidator
import eu.kanade.tachiyomi.data.backup.BackupWriter
import eu.kanade.tachiyomi.data.backup.create.creators.CategoriesBackupCreator import eu.kanade.tachiyomi.data.backup.create.creators.CategoriesBackupCreator
import eu.kanade.tachiyomi.data.backup.create.creators.ExtensionStoresBackupCreator import eu.kanade.tachiyomi.data.backup.create.creators.ExtensionStoresBackupCreator
import eu.kanade.tachiyomi.data.backup.create.creators.MangaBackupCreator import eu.kanade.tachiyomi.data.backup.create.creators.MangaBackupCreator
@@ -20,6 +21,8 @@ import eu.kanade.tachiyomi.data.backup.models.BackupManga
import eu.kanade.tachiyomi.data.backup.models.BackupPreference import eu.kanade.tachiyomi.data.backup.models.BackupPreference
import eu.kanade.tachiyomi.data.backup.models.BackupSource import eu.kanade.tachiyomi.data.backup.models.BackupSource
import eu.kanade.tachiyomi.data.backup.models.BackupSourcePreferences import eu.kanade.tachiyomi.data.backup.models.BackupSourcePreferences
import kotlinx.coroutines.flow.Flow
import kotlinx.coroutines.flow.emptyFlow
import kotlinx.serialization.protobuf.ProtoBuf import kotlinx.serialization.protobuf.ProtoBuf
import logcat.LogPriority import logcat.LogPriority
import okio.buffer import okio.buffer
@@ -85,27 +88,31 @@ class BackupCreator(
val nonFavoriteManga = if (options.readEntries) mangaRepository.getReadMangaNotInLibrary() else emptyList() val nonFavoriteManga = if (options.readEntries) mangaRepository.getReadMangaNotInLibrary() else emptyList()
val backupManga = backupMangas(getFavorites.await() + nonFavoriteManga, options) val backupManga = backupMangas(getFavorites.await() + nonFavoriteManga, options)
val backup = Backup(
backupManga = backupManga,
backupCategories = backupCategories(options),
backupSources = backupSources(backupManga),
backupPreferences = backupAppPreferences(options),
backupExtensionStores = backupExtensionStores(options),
backupSourcePreferences = backupSourcePreferences(options),
)
val byteArray = parser.encodeToByteArray(Backup.serializer(), backup)
if (byteArray.isEmpty()) {
throw IllegalStateException(context.stringResource(MR.strings.empty_backup_error))
}
file.openOutputStream() file.openOutputStream()
.also { .also {
// Force overwrite old file // Force overwrite old file
(it as? FileOutputStream)?.channel?.truncate(0) (it as? FileOutputStream)?.channel?.truncate(0)
} }
.sink().gzip().buffer().use { .sink().gzip().buffer().use {
it.write(byteArray) val writer = BackupWriter(it, parser)
val sourceIds = mutableSetOf<Long>()
backupManga.collect { manga ->
writer.writeEntry(manga)
sourceIds += manga.source
}
writer.writeMetadata(
Backup(
backupManga = emptyList(),
backupCategories = backupCategories(options),
backupSources = backupSources(sourceIds),
backupPreferences = backupAppPreferences(options),
backupExtensionStores = backupExtensionStores(options),
backupSourcePreferences = backupSourcePreferences(options),
),
)
if (writer.isEmpty) {
throw IllegalStateException(context.stringResource(MR.strings.empty_backup_error))
}
} }
val fileUri = file.uri val fileUri = file.uri
@@ -130,14 +137,14 @@ class BackupCreator(
return categoriesBackupCreator() return categoriesBackupCreator()
} }
private suspend fun backupMangas(mangas: List<Manga>, options: BackupOptions): List<BackupManga> { private fun backupMangas(mangas: List<Manga>, options: BackupOptions): Flow<BackupManga> {
if (!options.libraryEntries) return emptyList() if (!options.libraryEntries) return emptyFlow()
return mangaBackupCreator(mangas, options) return mangaBackupCreator(mangas, options)
} }
private suspend fun backupSources(mangas: List<BackupManga>): List<BackupSource> { private suspend fun backupSources(sourceIds: Set<Long>): List<BackupSource> {
return sourcesBackupCreator(mangas) return sourcesBackupCreator(sourceIds)
} }
private fun backupAppPreferences(options: BackupOptions): List<BackupPreference> { private fun backupAppPreferences(options: BackupOptions): List<BackupPreference> {
@@ -7,11 +7,13 @@ import eu.kanade.tachiyomi.data.backup.models.BackupManga
import eu.kanade.tachiyomi.data.backup.models.toBackupChapter import eu.kanade.tachiyomi.data.backup.models.toBackupChapter
import eu.kanade.tachiyomi.data.backup.models.toBackupTracking import eu.kanade.tachiyomi.data.backup.models.toBackupTracking
import eu.kanade.tachiyomi.ui.reader.setting.ReadingMode import eu.kanade.tachiyomi.ui.reader.setting.ReadingMode
import kotlinx.coroutines.flow.Flow
import kotlinx.coroutines.flow.flow
import mihon.core.common.extensions.toByteArray import mihon.core.common.extensions.toByteArray
import mihon.core.common.extensions.toEpochMillisOrZero import mihon.core.common.extensions.toEpochMillisOrZero
import tachiyomi.domain.category.interactor.GetCategories import tachiyomi.domain.category.repository.CategoryRepository
import tachiyomi.domain.chapter.repository.ChapterRepository import tachiyomi.domain.chapter.repository.ChapterRepository
import tachiyomi.domain.history.interactor.GetHistory import tachiyomi.domain.history.repository.HistoryRepository
import tachiyomi.domain.manga.model.Manga import tachiyomi.domain.manga.model.Manga
import tachiyomi.domain.manga.repository.MangaRepository import tachiyomi.domain.manga.repository.MangaRepository
import tachiyomi.domain.track.repository.TrackRepository import tachiyomi.domain.track.repository.TrackRepository
@@ -20,62 +22,71 @@ import tachiyomi.domain.track.repository.TrackRepository
class MangaBackupCreator( class MangaBackupCreator(
private val mangaRepository: MangaRepository, private val mangaRepository: MangaRepository,
private val chapterRepository: ChapterRepository, private val chapterRepository: ChapterRepository,
private val categoryRepository: CategoryRepository,
private val historyRepository: HistoryRepository,
private val trackRepository: TrackRepository, private val trackRepository: TrackRepository,
private val getCategories: GetCategories,
private val getHistory: GetHistory,
) { ) {
suspend operator fun invoke(mangas: List<Manga>, options: BackupOptions): List<BackupManga> { /**
return mangas.map { * Builds the entries a chunk at a time, so only one chunk's chapters are held at once.
backupManga(it, options) */
operator fun invoke(mangas: List<Manga>, options: BackupOptions): Flow<BackupManga> = flow {
mangas.chunked(CHUNK_SIZE).forEach { chunk ->
backupMangas(chunk, options).forEach { emit(it) }
} }
} }
private suspend fun backupManga(manga: Manga, options: BackupOptions): BackupManga { private suspend fun backupMangas(mangas: List<Manga>, options: BackupOptions): List<BackupManga> {
// Entry for this manga val mangaIds = mangas.map { it.id }
val mangaObject = manga.toBackupManga()
mangaObject.excludedScanlators = mangaRepository.getExcludedScanlators(manga.id).toList() val excludedScanlatorsByMangaId = mangaRepository.getExcludedScanlatorsByMangaIds(mangaIds)
// History is backed up by chapter URL, so it needs the chapters too
if (options.chapters) { val chaptersByMangaId = if (options.chapters || options.history) {
// Backup all the chapters chapterRepository.getChaptersByMangaIds(mangaIds)
chapterRepository.getChapterByMangaId(manga.id, applyScanlatorFilter = false) } else {
.map { it.toBackupChapter() } emptyMap()
.takeUnless { it.isEmpty() }
?.let { mangaObject.chapters = it }
} }
val categoriesByMangaId = if (options.categories) {
categoryRepository.getCategoriesByMangaIds(mangaIds)
} else {
emptyMap()
}
val tracksByMangaId = if (options.tracking) trackRepository.getTracksByMangaIds(mangaIds) else emptyMap()
val historyByMangaId = if (options.history) historyRepository.getHistoryByMangaIds(mangaIds) else emptyMap()
if (options.categories) { return mangas.map { manga ->
// Backup categories for this manga val mangaObject = manga.toBackupManga()
val categoriesForManga = getCategories.await(manga.id) val chapters = chaptersByMangaId[manga.id].orEmpty()
if (categoriesForManga.isNotEmpty()) {
mangaObject.categories = categoriesForManga.map { it.order } mangaObject.excludedScanlators = excludedScanlatorsByMangaId[manga.id].orEmpty()
if (options.chapters) {
mangaObject.chapters = chapters.map { it.toBackupChapter() }
} }
}
if (options.tracking) { if (options.categories) {
val tracks = trackRepository.getTracksByMangaId(manga.id).map { it.toBackupTracking() } mangaObject.categories = categoriesByMangaId[manga.id].orEmpty().map { it.order }
if (tracks.isNotEmpty()) {
mangaObject.tracking = tracks
} }
}
if (options.history) { if (options.tracking) {
val historyByMangaId = getHistory.await(manga.id) mangaObject.tracking = tracksByMangaId[manga.id].orEmpty().map { it.toBackupTracking() }
if (historyByMangaId.isNotEmpty()) { }
val chapterUrlsById = chapterRepository.getChapterByMangaId(manga.id).associate { it.id to it.url }
val history = historyByMangaId.mapNotNull { history -> if (options.history) {
val chapterUrlsById = chapters.associate { it.id to it.url }
mangaObject.history = historyByMangaId[manga.id].orEmpty().mapNotNull { history ->
// A chapter removed since its history was read takes that history with it // A chapter removed since its history was read takes that history with it
val url = chapterUrlsById[history.chapterId] ?: return@mapNotNull null val url = chapterUrlsById[history.chapterId] ?: return@mapNotNull null
BackupHistory(url, history.readAt.toEpochMillisOrZero(), history.readDuration) BackupHistory(url, history.readAt.toEpochMillisOrZero(), history.readDuration)
} }
if (history.isNotEmpty()) {
mangaObject.history = history
}
} }
}
return mangaObject mangaObject
}
}
private companion object {
const val CHUNK_SIZE = 100
} }
} }
@@ -1,7 +1,6 @@
package eu.kanade.tachiyomi.data.backup.create.creators package eu.kanade.tachiyomi.data.backup.create.creators
import dev.zacsweers.metro.Inject import dev.zacsweers.metro.Inject
import eu.kanade.tachiyomi.data.backup.models.BackupManga
import eu.kanade.tachiyomi.data.backup.models.BackupSource import eu.kanade.tachiyomi.data.backup.models.BackupSource
import eu.kanade.tachiyomi.source.Source import eu.kanade.tachiyomi.source.Source
import tachiyomi.domain.source.service.SourceManager import tachiyomi.domain.source.service.SourceManager
@@ -11,11 +10,8 @@ class SourcesBackupCreator(
private val sourceManager: SourceManager, private val sourceManager: SourceManager,
) { ) {
suspend operator fun invoke(mangas: List<BackupManga>): List<BackupSource> { suspend operator fun invoke(sourceIds: Set<Long>): List<BackupSource> {
return mangas return sourceIds.map { sourceManager.getOrStub(it).toBackupSource() }
.map(BackupManga::source)
.distinct()
.map { sourceManager.getOrStub(it).toBackupSource() }
} }
} }
@@ -9,7 +9,6 @@ import eu.kanade.tachiyomi.data.backup.BackupDecoder
import eu.kanade.tachiyomi.data.backup.BackupNotifier import eu.kanade.tachiyomi.data.backup.BackupNotifier
import eu.kanade.tachiyomi.data.backup.models.BackupCategory import eu.kanade.tachiyomi.data.backup.models.BackupCategory
import eu.kanade.tachiyomi.data.backup.models.BackupExtensionStore import eu.kanade.tachiyomi.data.backup.models.BackupExtensionStore
import eu.kanade.tachiyomi.data.backup.models.BackupManga
import eu.kanade.tachiyomi.data.backup.models.BackupPreference import eu.kanade.tachiyomi.data.backup.models.BackupPreference
import eu.kanade.tachiyomi.data.backup.models.BackupSource import eu.kanade.tachiyomi.data.backup.models.BackupSource
import eu.kanade.tachiyomi.data.backup.models.BackupSourcePreferences import eu.kanade.tachiyomi.data.backup.models.BackupSourcePreferences
@@ -23,6 +22,12 @@ import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Job import kotlinx.coroutines.Job
import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.coroutineScope
import kotlinx.coroutines.ensureActive import kotlinx.coroutines.ensureActive
import kotlinx.coroutines.flow.chunked
import kotlinx.coroutines.flow.emitAll
import kotlinx.coroutines.flow.filter
import kotlinx.coroutines.flow.flow
import kotlinx.coroutines.flow.map
import kotlinx.coroutines.flow.withIndex
import kotlinx.coroutines.launch import kotlinx.coroutines.launch
import logcat.LogPriority import logcat.LogPriority
import tachiyomi.core.common.i18n.stringResource import tachiyomi.core.common.i18n.stringResource
@@ -32,6 +37,7 @@ import tachiyomi.domain.source.service.SourceManager
import tachiyomi.i18n.MR import tachiyomi.i18n.MR
import java.io.File import java.io.File
import java.text.SimpleDateFormat import java.text.SimpleDateFormat
import java.util.BitSet
import java.util.Date import java.util.Date
import java.util.Locale import java.util.Locale
import java.util.concurrent.CopyOnWriteArrayList import java.util.concurrent.CopyOnWriteArrayList
@@ -97,7 +103,7 @@ class BackupRestorer(
} }
private suspend fun restoreFromFile(uri: Uri, options: RestoreOptions) { private suspend fun restoreFromFile(uri: Uri, options: RestoreOptions) {
val backup = backupDecoder.decode(uri) val (entryCount, backup) = backupDecoder.decodeMetadata(uri)
// Store source mapping for error messages // Store source mapping for error messages
val backupMaps = backup.backupSources val backupMaps = backup.backupSources
@@ -105,7 +111,7 @@ class BackupRestorer(
if (options.libraryEntries) { if (options.libraryEntries) {
restoreSourceNames(backupMaps) restoreSourceNames(backupMaps)
restoreAmount += backup.backupManga.size restoreAmount += entryCount
} }
if (options.categories) { if (options.categories) {
restoreAmount += 1 restoreAmount += 1
@@ -138,7 +144,7 @@ class BackupRestorer(
} }
if (options.libraryEntries) { if (options.libraryEntries) {
restoreManga( restoreManga(
backup.backupManga, uri,
if (options.categories) backup.backupCategories else emptyList(), if (options.categories) backup.backupCategories else emptyList(),
restoreCategoriesJob, restoreCategoriesJob,
) )
@@ -175,14 +181,31 @@ class BackupRestorer(
} }
private fun CoroutineScope.restoreManga( private fun CoroutineScope.restoreManga(
backupMangas: List<BackupManga>, uri: Uri,
backupCategories: List<BackupCategory>, backupCategories: List<BackupCategory>,
categoriesRestoreJob: Job?, categoriesRestoreJob: Job?,
) = launch { ) = launch {
categoriesRestoreJob?.join() categoriesRestoreJob?.join()
mangaRestorer.sortByNew(backupMangas)
// New entries go first. Rather than hold the others back in memory, a second pass reads them again.
val isExisting = mangaRestorer.getExistingMatcher()
val existingIndices = BitSet()
val newMangas = backupDecoder.decodeEntries(uri)
.withIndex()
.filter { (index, manga) ->
val exists = isExisting(manga)
if (exists) existingIndices.set(index)
!exists
}
.map { it.value }
val backupMangas = flow {
emitAll(newMangas)
emitAll(backupDecoder.decodeEntries(uri, existingIndices::get))
}
backupMangas
.chunked(100) .chunked(100)
.forEach { chunk -> .collect { chunk ->
val restoredAsBatch = try { val restoredAsBatch = try {
ensureActive() ensureActive()
mangaRestorer.restore(chunk, backupCategories) mangaRestorer.restore(chunk, backupCategories)
@@ -23,11 +23,13 @@ class MangaRestorer(
private val now = Clock.System.now().toLocalDateTime(timeZone) private val now = Clock.System.now().toLocalDateTime(timeZone)
private val currentFetchWindow = fetchInterval.getWindow(now.date, timeZone) private val currentFetchWindow = fetchInterval.getWindow(now.date, timeZone)
suspend fun sortByNew(backupMangas: List<BackupManga>): List<BackupManga> { /**
* Returns whether an entry was already in the database at the time of this call.
*/
suspend fun getExistingMatcher(): (BackupManga) -> Boolean {
val urlsBySource = restoreRepository.getMangaUrlsBySourceId() val urlsBySource = restoreRepository.getMangaUrlsBySourceId()
return backupMangas return { it.url in urlsBySource[it.source].orEmpty() }
.sortedBy { it.url in urlsBySource[it.source].orEmpty() }
} }
/** /**
@@ -0,0 +1,202 @@
package eu.kanade.tachiyomi.data.backup
import eu.kanade.tachiyomi.data.backup.models.Backup
import eu.kanade.tachiyomi.data.backup.models.BackupCategory
import eu.kanade.tachiyomi.data.backup.models.BackupChapter
import eu.kanade.tachiyomi.data.backup.models.BackupExtensionStore
import eu.kanade.tachiyomi.data.backup.models.BackupHistory
import eu.kanade.tachiyomi.data.backup.models.BackupManga
import eu.kanade.tachiyomi.data.backup.models.BackupPreference
import eu.kanade.tachiyomi.data.backup.models.BackupSource
import eu.kanade.tachiyomi.data.backup.models.BackupSourcePreferences
import eu.kanade.tachiyomi.data.backup.models.BackupTracking
import eu.kanade.tachiyomi.data.backup.models.BooleanPreferenceValue
import eu.kanade.tachiyomi.data.backup.models.StringPreferenceValue
import eu.kanade.tachiyomi.data.backup.models.StringSetPreferenceValue
import io.kotest.matchers.shouldBe
import kotlinx.serialization.protobuf.ProtoBuf
import okio.Buffer
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.assertThrows
import org.junit.jupiter.api.parallel.Execution
import org.junit.jupiter.api.parallel.ExecutionMode
import java.io.IOException
@Execution(ExecutionMode.CONCURRENT)
class BackupStreamTest {
private val parser = ProtoBuf
@Test
fun `writer output is the plain encoding`() {
val backup = backup()
val bytes = write(backup)
bytes shouldBe encode(backup)
encode(parser.decodeFromByteArray(Backup.serializer(), bytes)) shouldBe encode(backup)
}
@Test
fun `reader reads the plain encoding`() {
val backup = backup()
val (entries, metadata) = read(encode(backup))
entries.map(::encode) shouldBe backup.backupManga.map(::encode)
encode(metadata) shouldBe encode(backup.copy(backupManga = emptyList()))
}
@Test
fun `reader counts entries without decoding them`() {
val backup = backup()
val reader = BackupReader(Buffer().write(encode(backup)), parser)
var count = 0
while (reader.nextEntry()) {
reader.skipEntry()
count++
}
count shouldBe backup.backupManga.size
encode(reader.readMetadata()) shouldBe encode(backup.copy(backupManga = emptyList()))
}
@Test
fun `reader takes fields in any order`() {
val backup = backup()
val (first, second, third) = backup.backupManga
val input = Buffer()
.write(encode(Backup(emptyList(), backupSources = backup.backupSources)))
.write(encode(Backup(listOf(first))))
// Fields Backup doesn't know, one of each remaining wire type
.writeByte(3 shl 3 or 0).writeByte(0x96).writeByte(0x01)
.writeByte(4 shl 3 or 1).writeLongLe(42)
.write(encode(Backup(listOf(second), backupCategories = backup.backupCategories)))
.writeByte(5 shl 3 or 5).writeIntLe(42)
.write(encode(Backup(emptyList(), backupPreferences = backup.backupPreferences)))
.write(encode(Backup(listOf(third), backupExtensionStores = backup.backupExtensionStores)))
.write(encode(Backup(emptyList(), backupSourcePreferences = backup.backupSourcePreferences)))
.readByteArray()
val (entries, metadata) = read(input)
entries.map(::encode) shouldBe backup.backupManga.map(::encode)
encode(metadata) shouldBe encode(backup.copy(backupManga = emptyList()))
}
@Test
fun `empty backup`() {
val sink = Buffer()
val writer = BackupWriter(sink, parser)
writer.writeMetadata(Backup(emptyList()))
writer.isEmpty shouldBe true
sink.size shouldBe 0L
val (entries, metadata) = read(ByteArray(0))
entries shouldBe emptyList()
encode(metadata) shouldBe ByteArray(0)
}
@Test
fun `truncated input throws`() {
val bytes = encode(backup())
// Cut inside the last entry, inside the metadata, and right after a tag
val entryEnd = encode(Backup(backup().backupManga)).size
listOf(entryEnd - 1, bytes.size - 1, entryEnd + 1).forEach { size ->
assertThrows<IOException> { read(bytes.copyOf(size)) }
}
}
@Test
fun `garbage input throws`() {
listOf(
// Wire type 7 doesn't exist
byteArrayOf(0x0F),
// Field number 0
byteArrayOf(0x02, 0x00),
// A varint that never ends
ByteArray(11) { 0xFF.toByte() },
// An entry longer than the input
byteArrayOf(0x0A, 0x10, 0x01),
// An entry that isn't a BackupManga
byteArrayOf(0x0A, 0x02, 0x0F, 0x00),
).forEach { input ->
assertThrows<IOException> { read(input) }
}
}
private fun write(backup: Backup): ByteArray {
val sink = Buffer()
val writer = BackupWriter(sink, parser)
backup.backupManga.forEach(writer::writeEntry)
writer.writeMetadata(backup.copy(backupManga = emptyList()))
writer.isEmpty shouldBe false
return sink.readByteArray()
}
private fun read(bytes: ByteArray): Pair<List<BackupManga>, Backup> {
val reader = BackupReader(Buffer().write(bytes), parser)
val entries = buildList {
while (reader.nextEntry()) add(reader.readEntry())
}
return entries to reader.readMetadata()
}
private fun encode(backup: Backup) = parser.encodeToByteArray(Backup.serializer(), backup)
private fun encode(manga: BackupManga) = parser.encodeToByteArray(BackupManga.serializer(), manga)
private fun backup() = Backup(
backupManga = listOf(
BackupManga(
source = 1L,
url = "/manga/1",
title = "First",
genre = listOf("Action", "Drama"),
chapters = listOf(
BackupChapter(url = "/chapter/1", name = "Chapter 1", read = true, chapterNumber = 1f),
BackupChapter(url = "/chapter/2", name = "Chapter 2", bookmark = true, lastPageRead = 4),
),
categories = listOf(0L, 1L),
tracking = listOf(BackupTracking(syncId = 2, libraryId = 3L, title = "First", mediaId = 7L)),
history = listOf(BackupHistory(url = "/chapter/1", lastRead = 1_700_000_000_000L, readDuration = 60)),
excludedScanlators = listOf("Group"),
),
BackupManga(source = 2L, url = "/manga/2", title = "Second", favorite = false),
BackupManga(
source = 1L,
url = "/manga/3",
title = "Third",
description = "x".repeat(300),
chapters = List(200) { BackupChapter(url = "/chapter/3/$it", name = "Chapter $it") },
),
),
backupCategories = listOf(
BackupCategory(name = "Reading", order = 0),
BackupCategory(name = "Done", order = 1),
),
backupSources = listOf(BackupSource(name = "One", sourceId = 1L), BackupSource(name = "Two", sourceId = 2L)),
backupPreferences = listOf(
BackupPreference("flag", BooleanPreferenceValue(true)),
BackupPreference("tags", StringSetPreferenceValue(setOf("a", "b"))),
),
backupSourcePreferences = listOf(
BackupSourcePreferences("source_1", listOf(BackupPreference("lang", StringPreferenceValue("en")))),
),
backupExtensionStores = listOf(
BackupExtensionStore(
indexUrl = "https://example.org/index.min.json",
name = "Store",
badgeLabel = null,
signingKey = "key",
contactWebsite = "https://example.org",
contactDiscord = null,
isLegacy = false,
extensionListUrl = null,
),
),
)
}
@@ -50,6 +50,15 @@ class CategoryRepositoryImpl(
.subscribeToList() .subscribeToList()
} }
override suspend fun getCategoriesByMangaIds(mangaIds: List<Long>): Map<Long, List<Category>> {
return database.categoryQueries
.getCategoriesByMangaIds(mangaIds) { mangaId, id, name, order, flags ->
mangaId to mapCategory(id, name, order, flags)
}
.awaitAsList()
.groupBy({ it.first }, { it.second })
}
override suspend fun insert(category: NewCategory) { override suspend fun insert(category: NewCategory) {
database.categoryQueries.insert(name = category.name, flags = category.flags) database.categoryQueries.insert(name = category.name, flags = category.flags)
} }
@@ -113,6 +113,13 @@ class ChapterRepositoryImpl(
.awaitAsList() .awaitAsList()
} }
override suspend fun getChaptersByMangaIds(mangaIds: List<Long>): Map<Long, List<Chapter>> {
return database.chapterQueries
.getChaptersByMangaIds(mangaIds, ::mapChapter)
.awaitAsList()
.groupBy { it.mangaId }
}
override suspend fun getChapterSummaryByMangaId(mangaId: Long): ChapterSummary { override suspend fun getChapterSummaryByMangaId(mangaId: Long): ChapterSummary {
return database.chapterQueries return database.chapterQueries
.getChapterSummaryByMangaId(mangaId) { total, latestChapterNumber -> .getChapterSummaryByMangaId(mangaId) { total, latestChapterNumber ->
@@ -48,6 +48,15 @@ class HistoryRepositoryImpl(
.awaitAsList() .awaitAsList()
} }
override suspend fun getHistoryByMangaIds(mangaIds: List<Long>): Map<Long, List<History>> {
return database.historyQueries
.getHistoryByMangaIds(mangaIds) { mangaId, id, chapterId, readAt, readDuration ->
mangaId to HistoryMapper.mapHistory(id, chapterId, readAt, readDuration)
}
.awaitAsList()
.groupBy({ it.first }, { it.second })
}
override suspend fun resetHistory(historyId: Long) { override suspend fun resetHistory(historyId: Long) {
try { try {
database.historyQueries.resetHistoryById(historyId) database.historyQueries.resetHistoryById(historyId)
@@ -144,6 +144,13 @@ class MangaRepositoryImpl(
.toSet() .toSet()
} }
override suspend fun getExcludedScanlatorsByMangaIds(mangaIds: List<Long>): Map<Long, List<String>> {
return database.excluded_scanlatorQueries
.getExcludedScanlatorsByMangaIds(mangaIds)
.awaitAsList()
.groupBy({ it.manga_id }, { it.scanlator })
}
override fun getExcludedScanlatorsAsFlow(mangaId: Long): Flow<Set<String>> { override fun getExcludedScanlatorsAsFlow(mangaId: Long): Flow<Set<String>> {
return database.excluded_scanlatorQueries return database.excluded_scanlatorQueries
.getExcludedScanlatorsByMangaId(mangaId) .getExcludedScanlatorsByMangaId(mangaId)
@@ -31,6 +31,13 @@ class TrackRepositoryImpl(
.awaitAsList() .awaitAsList()
} }
override suspend fun getTracksByMangaIds(mangaIds: List<Long>): Map<Long, List<Track>> {
return database.manga_trackQueries
.getTracksByMangaIds(mangaIds, TrackMapper::mapTrack)
.awaitAsList()
.groupBy { it.mangaId }
}
override suspend fun getTracks(): List<Track> { override suspend fun getTracks(): List<Track> {
return database.manga_trackQueries return database.manga_trackQueries
.getTracks(TrackMapper::mapTrack) .getTracks(TrackMapper::mapTrack)
@@ -34,6 +34,14 @@ ON C.id = MC.category_id
WHERE MC.manga_id = :mangaId WHERE MC.manga_id = :mangaId
ORDER BY C.`order`; ORDER BY C.`order`;
getCategoriesByMangaIds:
SELECT MC.manga_id, C.id, C.name, C.`order`, C.flags
FROM category C
JOIN manga_category MC
ON C.id = MC.category_id
WHERE MC.manga_id IN :mangaIds
ORDER BY MC.manga_id, C.`order`;
insert: insert:
-- The position is taken inside the insert, so two categories created at once cannot claim the same one -- The position is taken inside the insert, so two categories created at once cannot claim the same one
INSERT INTO category(name, `order`, flags) INSERT INTO category(name, `order`, flags)
@@ -48,6 +48,16 @@ AND (
OR ES.scanlator IS NULL OR ES.scanlator IS NULL
); );
getChaptersByMangaIds:
-- The unique index's order, which getChaptersByMangaId returns too, so backups keep their chapter order
SELECT
id, manga_id, remote_url, remote_name, remote_scanlator, remote_chapter_number,
remote_date_upload, remote_order, remote_memo, user_read, user_bookmark,
user_last_page_read, state_date_fetch
FROM chapter
WHERE manga_id IN :mangaIds
ORDER BY manga_id, remote_url;
getChapterSummaryByMangaId: getChapterSummaryByMangaId:
SELECT count(*) AS total, max(remote_chapter_number) AS latestChapterNumber SELECT count(*) AS total, max(remote_chapter_number) AS latestChapterNumber
FROM chapter FROM chapter
@@ -21,3 +21,9 @@ getExcludedScanlatorsByMangaId:
SELECT scanlator SELECT scanlator
FROM excluded_scanlator FROM excluded_scanlator
WHERE manga_id = :mangaId; WHERE manga_id = :mangaId;
getExcludedScanlatorsByMangaIds:
SELECT manga_id, scanlator
FROM excluded_scanlator
WHERE manga_id IN :mangaIds
ORDER BY manga_id, scanlator;
@@ -20,6 +20,12 @@ SELECT id, chapter_id, read_at, read_duration
FROM history FROM history
WHERE manga_id = :mangaId; WHERE manga_id = :mangaId;
getHistoryByMangaIds:
SELECT manga_id, id, chapter_id, read_at, read_duration
FROM history
WHERE manga_id IN :mangaIds
ORDER BY manga_id, id;
resetHistoryById: resetHistoryById:
UPDATE history UPDATE history
SET read_at = NULL SET read_at = NULL
@@ -48,6 +48,14 @@ SELECT
FROM manga_track FROM manga_track
WHERE manga_id = :mangaId; WHERE manga_id = :mangaId;
getTracksByMangaIds:
SELECT
id, manga_id, tracker_id, remote_id, remote_url, library_id, title, last_chapter_read,
total_chapters, status, score, start_date, finish_date, private
FROM manga_track
WHERE manga_id IN :mangaIds
ORDER BY manga_id, tracker_id;
upsert: upsert:
INSERT INTO manga_track( INSERT INTO manga_track(
manga_id, tracker_id, remote_id, remote_url, library_id, title, last_chapter_read, manga_id, tracker_id, remote_id, remote_url, library_id, title, last_chapter_read,
@@ -16,6 +16,8 @@ interface CategoryRepository {
fun getCategoriesByMangaIdAsFlow(mangaId: Long): Flow<List<Category>> fun getCategoriesByMangaIdAsFlow(mangaId: Long): Flow<List<Category>>
suspend fun getCategoriesByMangaIds(mangaIds: List<Long>): Map<Long, List<Category>>
suspend fun insert(category: NewCategory) suspend fun insert(category: NewCategory)
suspend fun insertAll(categories: List<NewCategory>) suspend fun insertAll(categories: List<NewCategory>)
@@ -20,6 +20,8 @@ interface ChapterRepository {
suspend fun getChapterByMangaId(mangaId: Long, applyScanlatorFilter: Boolean = false): List<Chapter> suspend fun getChapterByMangaId(mangaId: Long, applyScanlatorFilter: Boolean = false): List<Chapter>
suspend fun getChaptersByMangaIds(mangaIds: List<Long>): Map<Long, List<Chapter>>
suspend fun getChapterSummaryByMangaId(mangaId: Long): ChapterSummary suspend fun getChapterSummaryByMangaId(mangaId: Long): ChapterSummary
suspend fun getScanlatorsByMangaId(mangaId: Long): List<String> suspend fun getScanlatorsByMangaId(mangaId: Long): List<String>
@@ -15,6 +15,8 @@ interface HistoryRepository {
suspend fun getHistoryByMangaId(mangaId: Long): List<History> suspend fun getHistoryByMangaId(mangaId: Long): List<History>
suspend fun getHistoryByMangaIds(mangaIds: List<Long>): Map<Long, List<History>>
suspend fun resetHistory(historyId: Long) suspend fun resetHistory(historyId: Long)
suspend fun resetHistoryByMangaId(mangaId: Long) suspend fun resetHistoryByMangaId(mangaId: Long)
@@ -43,6 +43,8 @@ interface MangaRepository {
suspend fun getExcludedScanlators(mangaId: Long): Set<String> suspend fun getExcludedScanlators(mangaId: Long): Set<String>
suspend fun getExcludedScanlatorsByMangaIds(mangaIds: List<Long>): Map<Long, List<String>>
fun getExcludedScanlatorsAsFlow(mangaId: Long): Flow<Set<String>> fun getExcludedScanlatorsAsFlow(mangaId: Long): Flow<Set<String>>
suspend fun setExcludedScanlators(mangaId: Long, excludedScanlators: Set<String>) suspend fun setExcludedScanlators(mangaId: Long, excludedScanlators: Set<String>)
@@ -9,6 +9,8 @@ interface TrackRepository {
suspend fun getTracksByMangaId(mangaId: Long): List<Track> suspend fun getTracksByMangaId(mangaId: Long): List<Track>
suspend fun getTracksByMangaIds(mangaIds: List<Long>): Map<Long, List<Track>>
suspend fun getTracks(): List<Track> suspend fun getTracks(): List<Track>
fun getTracksAsFlow(): Flow<List<Track>> fun getTracksAsFlow(): Flow<List<Track>>