Change extension repo to extension store and add support for newer extension index format (#3349)

This commit is contained in:
AntsyLich
2026-06-03 06:35:44 +06:00
committed by GitHub
parent 748211c19b
commit b446d2a6b1
64 changed files with 1279 additions and 1218 deletions
@@ -0,0 +1,7 @@
package mihon.data.extension.model
import mihon.domain.extension.model.ExtensionStore
interface BaseNetworkExtensionStore {
fun toExtensionStore(indexUrl: String): ExtensionStore
}
@@ -0,0 +1,110 @@
package mihon.data.extension.model
import android.annotation.SuppressLint
import kotlinx.serialization.Serializable
import kotlinx.serialization.json.JsonNames
import kotlinx.serialization.protobuf.ProtoNumber
import mihon.domain.extension.model.ExtensionStore
@SuppressLint("UnsafeOptInUsageError")
@Serializable
data class NetworkExtensionStore(
@ProtoNumber(1) val name: String,
@ProtoNumber(2) val badgeLabel: String,
@ProtoNumber(3) val signingKey: String,
@ProtoNumber(4) val contact: Contact,
@ProtoNumber(5) val extensions: List<Extension>,
) : BaseNetworkExtensionStore {
@Serializable
data class Contact(
@ProtoNumber(1) val website: String,
@ProtoNumber(2) val discord: String?,
)
@Serializable
data class Extension(
@ProtoNumber(1) val name: String,
@ProtoNumber(2) val packageName: String,
@ProtoNumber(3) val resources: Resources,
@ProtoNumber(4) val extensionLib: String,
@ProtoNumber(5) val versionCode: Long,
@ProtoNumber(6) val versionName: String,
@ProtoNumber(7) val sources: List<Source>,
)
@Serializable
data class Resources(
@ProtoNumber(1) val apkUrl: String,
@ProtoNumber(2) val iconUrl: String,
)
@Serializable
data class Source(
@ProtoNumber(1) val id: Long,
@ProtoNumber(2) val name: String,
@ProtoNumber(3) val language: String,
@ProtoNumber(4) val homeUrl: String = "",
@ProtoNumber(5) val mirrorUrls: List<String> = emptyList(),
@ProtoNumber(6) val contentRating: ContentRating = ContentRating.SAFE,
@ProtoNumber(7) val message: String? = null,
)
@Suppress("Unused")
enum class ContentRating {
@ProtoNumber(0)
@JsonNames("CONTENT_RATING_SAFE")
SAFE,
@ProtoNumber(1)
@JsonNames("CONTENT_RATING_SUGGESTIVE")
SUGGESTIVE,
@ProtoNumber(2)
@JsonNames("CONTENT_RATING_EROTICA")
EROTICA,
@ProtoNumber(3)
@JsonNames("CONTENT_RATING_PORNOGRAPHIC")
PORNOGRAPHIC,
}
override fun toExtensionStore(indexUrl: String): ExtensionStore {
return ExtensionStore(
indexUrl = indexUrl,
name = name,
badgeLabel = badgeLabel,
signingKey = signingKey,
contact = ExtensionStore.Contact(
website = contact.website,
discord = contact.discord,
),
isLegacy = false,
)
}
fun toAvailableExtensions(store: ExtensionStore): List<eu.kanade.tachiyomi.extension.model.Extension.Available> {
return extensions.map { extension ->
val lang = extension.sources.map { it.language }.toSet()
eu.kanade.tachiyomi.extension.model.Extension.Available(
name = extension.name,
pkgName = extension.packageName,
apkUrl = extension.resources.apkUrl,
iconUrl = extension.resources.iconUrl,
libVersion = extension.extensionLib.toDouble(),
versionCode = extension.versionCode,
versionName = extension.versionName,
lang = if (lang.size == 1) lang.first() else "all",
isNsfw = extension.sources.maxOfOrNull { it.contentRating } == ContentRating.PORNOGRAPHIC,
sources = extension.sources.map { source ->
eu.kanade.tachiyomi.extension.model.Extension.Available.Source(
id = source.id,
name = source.name,
lang = source.language,
baseUrl = source.homeUrl,
)
},
store = store,
)
}
}
}
@@ -0,0 +1,61 @@
package mihon.data.extension.model
import android.annotation.SuppressLint
import eu.kanade.tachiyomi.extension.model.Extension
import kotlinx.serialization.Serializable
import mihon.domain.extension.model.ExtensionStore
@SuppressLint("UnsafeOptInUsageError")
@Serializable
data class NetworkLegacyExtension(
val name: String,
val pkg: String,
val apk: String,
val lang: String,
val code: Long,
val version: String,
val nsfw: Int,
val sources: List<Source>?,
) {
@Serializable
data class Source(
val id: Long,
val lang: String,
val name: String,
val baseUrl: String,
)
fun toAvailableExtension(store: ExtensionStore, storeBaseUrl: String): Extension.Available {
return Extension.Available(
name = name.substringAfter("Tachiyomi: "),
pkgName = pkg,
apkUrl = "$storeBaseUrl/apk/$apk",
iconUrl = "$storeBaseUrl/icon/$pkg.png",
libVersion = version.substringBeforeLast('.').toDouble(),
versionCode = code,
versionName = version,
lang = lang,
isNsfw = nsfw == 1,
sources = if (sources.isNullOrEmpty()) {
listOf(
Extension.Available.Source(
id = 0,
name = name,
lang = lang,
baseUrl = "",
),
)
} else {
sources.map { source ->
Extension.Available.Source(
id = source.id,
name = source.name,
lang = source.lang,
baseUrl = source.baseUrl,
)
}
},
store = store,
)
}
}
@@ -0,0 +1,33 @@
package mihon.data.extension.model
import android.annotation.SuppressLint
import kotlinx.serialization.Serializable
import mihon.domain.extension.model.ExtensionStore
@SuppressLint("UnsafeOptInUsageError")
@Serializable
data class NetworkLegacyExtensionRepo(
val meta: Meta,
) : BaseNetworkExtensionStore {
@Serializable
data class Meta(
val name: String,
val shortName: String?,
val website: String,
val signingKeyFingerprint: String,
)
override fun toExtensionStore(indexUrl: String): ExtensionStore {
return ExtensionStore(
indexUrl = indexUrl,
name = meta.name,
badgeLabel = meta.shortName ?: meta.name,
signingKey = meta.signingKeyFingerprint,
contact = ExtensionStore.Contact(
website = meta.website,
discord = null,
),
isLegacy = true,
)
}
}
@@ -0,0 +1,124 @@
package mihon.data.extension.repository
import app.cash.sqldelight.async.coroutines.awaitAsList
import eu.kanade.tachiyomi.extension.model.Extension
import kotlinx.coroutines.async
import kotlinx.coroutines.awaitAll
import kotlinx.coroutines.flow.Flow
import kotlinx.coroutines.supervisorScope
import logcat.LogPriority
import mihon.data.extension.service.ExtensionStoreService
import mihon.domain.extension.model.ExtensionStore
import mihon.domain.extension.repository.ExtensionStoreRepository
import tachiyomi.core.common.util.system.logcat
import tachiyomi.data.Database
import tachiyomi.data.subscribeToList
import tachiyomi.data.subscribeToOne
class ExtensionStoreRepositoryImpl(
private val service: ExtensionStoreService,
private val database: Database,
) : ExtensionStoreRepository {
override suspend fun insert(indexUrl: String): Result<Unit> {
return service.fetch(indexUrl).mapCatching { upsert(it) }
}
override suspend fun insertFromPreference(indexUrl: String, name: String) {
database.extension_storeQueries.upsert(
indexUrl = indexUrl,
name = name,
badgeLabel = name,
signingKey = "NO_SIGNING_KEY",
contactWebsite = indexUrl,
contactDiscord = null,
isLegacy = false,
)
}
override suspend fun refreshAll() {
try {
database.extension_storeQueries.getAll().awaitAsList().forEach { store ->
service.fetch(store.index_url)
.mapCatching { upsert(it) }
.onFailure {
logcat(LogPriority.ERROR, it) {
"Failed to refresh extension store '${store.name} (${store.index_url})'"
}
}
}
} catch (e: Exception) {
logcat(LogPriority.ERROR, e)
}
}
private suspend fun upsert(store: ExtensionStore) {
database.extension_storeQueries.upsert(
indexUrl = store.indexUrl,
name = store.name,
badgeLabel = store.badgeLabel,
signingKey = store.signingKey,
contactWebsite = store.contact.website,
contactDiscord = store.contact.discord,
isLegacy = store.isLegacy,
)
}
override suspend fun fetchExtensions(): List<Extension.Available> {
return try {
supervisorScope {
database.extension_storeQueries.getAll(::extensionStoreMapper).awaitAsList().map { store ->
async {
service.getExtensions(store).onFailure {
this@ExtensionStoreRepositoryImpl.logcat(LogPriority.ERROR, it) {
"Failed to fetch extensions for store '${store.name} (${store.indexUrl})'"
}
}
}
}
.awaitAll()
.flatMap { it.getOrDefault(emptyList()) }
}
} catch (e: Exception) {
logcat(LogPriority.ERROR, e)
emptyList()
}
}
override suspend fun getAll(): List<ExtensionStore> {
return database.extension_storeQueries.getAll(::extensionStoreMapper).awaitAsList()
}
override fun getAllAsFlow(): Flow<List<ExtensionStore>> {
return database.extension_storeQueries.getAll(::extensionStoreMapper).subscribeToList()
}
override fun getCountAsFlow(): Flow<Long> {
return database.extension_storeQueries
.getCount()
.subscribeToOne()
}
override suspend fun remove(indexUrl: String) {
database.extension_storeQueries.delete(indexUrl)
}
private fun extensionStoreMapper(
indexUrl: String,
name: String,
badgeLabel: String,
signingKey: String,
contactWebsite: String,
contactDiscord: String?,
isLegacy: Boolean,
): ExtensionStore = ExtensionStore(
indexUrl = indexUrl,
name = name,
badgeLabel = badgeLabel,
signingKey = signingKey,
contact = ExtensionStore.Contact(
website = contactWebsite,
discord = contactDiscord,
),
isLegacy = isLegacy,
)
}
@@ -0,0 +1,97 @@
package mihon.data.extension.service
import eu.kanade.tachiyomi.extension.model.Extension
import eu.kanade.tachiyomi.network.GET
import eu.kanade.tachiyomi.network.NetworkHelper
import eu.kanade.tachiyomi.network.awaitSuccess
import kotlinx.serialization.decodeFromByteArray
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.okio.decodeFromBufferedSource
import kotlinx.serialization.protobuf.ProtoBuf
import logcat.LogPriority
import mihon.data.extension.model.NetworkExtensionStore
import mihon.data.extension.model.NetworkLegacyExtension
import mihon.data.extension.model.NetworkLegacyExtensionRepo
import mihon.domain.extension.model.ExtensionStore
import tachiyomi.core.common.util.system.logcat
import kotlin.coroutines.cancellation.CancellationException
class ExtensionStoreService(
private val network: NetworkHelper,
private val json: Json,
private val protoBuf: ProtoBuf,
) {
suspend fun fetch(indexUrl: String): Result<ExtensionStore> {
var updatedIndexUrl: String = indexUrl
return try {
val store = network.client.newCall(GET(indexUrl)).awaitSuccess().body.source().use { source ->
try {
protoBuf.decodeFromByteArray<NetworkExtensionStore>(source.peek().readByteArray())
} catch (e: IllegalArgumentException) {
logcat(LogPriority.ERROR, e) {
"Failed to add extension store '$updatedIndexUrl'"
}
try {
json.decodeFromBufferedSource<NetworkExtensionStore>(source.peek())
} catch (e: IllegalArgumentException) {
logcat(LogPriority.ERROR, e) {
"Failed to add extension store '$updatedIndexUrl'"
}
try {
json.decodeFromBufferedSource<NetworkLegacyExtensionRepo>(source.peek())
} catch (e: IllegalArgumentException) {
if (!indexUrl.endsWith("/index.min.json")) {
throw e
}
logcat(LogPriority.ERROR, e) {
"Failed to add extension store '$updatedIndexUrl'"
}
updatedIndexUrl = indexUrl.replace("/index.min.json", "/repo.json")
network.client.newCall(GET(updatedIndexUrl)).awaitSuccess().body.source().use {
json.decodeFromBufferedSource<NetworkLegacyExtensionRepo>(it)
}
}
}
}
.toExtensionStore(updatedIndexUrl)
}
Result.success(store)
} catch (e: CancellationException) {
throw e
} catch (e: Exception) {
logcat(LogPriority.ERROR, e) {
"Failed to add extension store '$updatedIndexUrl'"
}
Result.failure(e)
}
}
suspend fun getExtensions(store: ExtensionStore): Result<List<Extension.Available>> {
return try {
val extensions = if (!store.isLegacy) {
val response = network.client.newCall(GET(store.indexUrl)).awaitSuccess()
response.body.source().use { source ->
try {
protoBuf.decodeFromByteArray<NetworkExtensionStore>(source.peek().readByteArray())
.toAvailableExtensions(store)
} catch (_: IllegalArgumentException) {
json.decodeFromBufferedSource<NetworkExtensionStore>(source.peek())
.toAvailableExtensions(store)
}
}
} else {
val storeBaseUrl = store.indexUrl.removeSuffix("/repo.json")
val response = network.client.newCall(GET("$storeBaseUrl/index.min.json")).awaitSuccess()
response.body.source().use { source ->
json.decodeFromBufferedSource<List<NetworkLegacyExtension>>(source)
.map { it.toAvailableExtension(store, storeBaseUrl) }
}
}
Result.success(extensions)
} catch (e: CancellationException) {
throw e
} catch (e: Exception) {
Result.failure(e)
}
}
}
@@ -1,116 +0,0 @@
package mihon.data.repository
import android.database.SQLException
import app.cash.sqldelight.async.coroutines.awaitAsList
import app.cash.sqldelight.async.coroutines.awaitAsOneOrNull
import kotlinx.coroutines.flow.Flow
import kotlinx.coroutines.flow.map
import mihon.domain.extensionrepo.exception.SaveExtensionRepoException
import mihon.domain.extensionrepo.model.ExtensionRepo
import mihon.domain.extensionrepo.repository.ExtensionRepoRepository
import tachiyomi.data.Database
import tachiyomi.data.subscribeToList
import tachiyomi.data.subscribeToOne
class ExtensionRepoRepositoryImpl(
private val database: Database,
) : ExtensionRepoRepository {
override fun subscribeAll(): Flow<List<ExtensionRepo>> {
return database.extension_reposQueries
.findAll(::mapExtensionRepo)
.subscribeToList()
}
override suspend fun getAll(): List<ExtensionRepo> {
return database.extension_reposQueries
.findAll(::mapExtensionRepo)
.awaitAsList()
}
override suspend fun getRepo(baseUrl: String): ExtensionRepo? {
return database.extension_reposQueries
.findOne(baseUrl, ::mapExtensionRepo)
.awaitAsOneOrNull()
}
override suspend fun getRepoBySigningKeyFingerprint(fingerprint: String): ExtensionRepo? {
return database.extension_reposQueries
.findOneBySigningKeyFingerprint(fingerprint, ::mapExtensionRepo)
.awaitAsOneOrNull()
}
override fun getCount(): Flow<Int> {
return database.extension_reposQueries
.count()
.subscribeToOne()
.map { it.toInt() }
}
override suspend fun insertRepo(
baseUrl: String,
name: String,
shortName: String?,
website: String,
signingKeyFingerprint: String,
) {
try {
database.extension_reposQueries.insert(
baseUrl,
name,
shortName,
website,
signingKeyFingerprint,
)
} catch (ex: SQLException) {
throw SaveExtensionRepoException(ex)
}
}
override suspend fun upsertRepo(
baseUrl: String,
name: String,
shortName: String?,
website: String,
signingKeyFingerprint: String,
) {
try {
database.extension_reposQueries.upsert(
baseUrl,
name,
shortName,
website,
signingKeyFingerprint,
)
} catch (ex: SQLException) {
throw SaveExtensionRepoException(ex)
}
}
override suspend fun replaceRepo(newRepo: ExtensionRepo) {
database.extension_reposQueries.replace(
newRepo.baseUrl,
newRepo.name,
newRepo.shortName,
newRepo.website,
newRepo.signingKeyFingerprint,
)
}
override suspend fun deleteRepo(baseUrl: String) {
database.extension_reposQueries.delete(baseUrl)
}
private fun mapExtensionRepo(
baseUrl: String,
name: String,
shortName: String?,
website: String,
signingKeyFingerprint: String,
): ExtensionRepo = ExtensionRepo(
baseUrl = baseUrl,
name = name,
shortName = shortName,
website = website,
signingKeyFingerprint = signingKeyFingerprint,
)
}
@@ -1,57 +0,0 @@
CREATE TABLE extension_repos (
base_url TEXT NOT NULL PRIMARY KEY,
name TEXT NOT NULL,
short_name TEXT,
website TEXT NOT NULL,
signing_key_fingerprint TEXT UNIQUE NOT NULL
);
findOne:
SELECT *
FROM extension_repos
WHERE base_url = :base_url;
findOneBySigningKeyFingerprint:
SELECT *
FROM extension_repos
WHERE signing_key_fingerprint = :fingerprint;
findAll:
SELECT *
FROM extension_repos;
count:
SELECT COUNT(*)
FROM extension_repos;
insert:
INSERT INTO extension_repos(base_url, name, short_name, website, signing_key_fingerprint)
VALUES (:base_url, :name, :short_name, :website, :fingerprint);
upsert:
INSERT INTO extension_repos(base_url, name, short_name, website, signing_key_fingerprint)
VALUES (:base_url, :name, :short_name, :website, :fingerprint)
ON CONFLICT(base_url)
DO UPDATE
SET
name = :name,
short_name = :short_name,
website =: website,
signing_key_fingerprint = :fingerprint
WHERE base_url = base_url;
replace:
INSERT INTO extension_repos(base_url, name, short_name, website, signing_key_fingerprint)
VALUES (:base_url, :name, :short_name, :website, :fingerprint)
ON CONFLICT(signing_key_fingerprint)
DO UPDATE
SET
base_url = :base_url,
name = :name,
short_name = :short_name,
website =: website
WHERE signing_key_fingerprint = signing_key_fingerprint;
delete:
DELETE FROM extension_repos
WHERE base_url = :base_url;
@@ -0,0 +1,42 @@
import kotlin.Boolean;
CREATE TABLE extension_store(
index_url TEXT NOT NULL PRIMARY KEY,
name TEXT NOT NULL,
badge_label TEXT NOT NULL,
signing_key TEXT NOT NULL,
contact_website TEXT NOT NULL,
contact_discord TEXT,
is_legacy INTEGER AS Boolean NOT NULL
);
get:
SELECT *
FROM extension_store
WHERE index_url = :indexUrl;
getAll:
SELECT *
FROM extension_store;
getCount:
SELECT COUNT(*)
FROM extension_store;
upsert:
INSERT INTO extension_store(index_url, name, badge_label, signing_key, contact_website, contact_discord, is_legacy)
VALUES (:indexUrl, :name, :badgeLabel, :signingKey, :contactWebsite, :contactDiscord, :isLegacy)
ON CONFLICT(index_url)
DO UPDATE
SET
name = :name,
badge_label = :badgeLabel,
signing_key =: signingKey,
contact_website = :contactWebsite,
contact_discord = :contactDiscord,
is_legacy = :isLegacy
WHERE index_url = :indexUrl;
delete:
DELETE FROM extension_store
WHERE index_url = :indexUrl;
@@ -0,0 +1,16 @@
import kotlin.Boolean;
CREATE TABLE extension_store(
index_url TEXT NOT NULL PRIMARY KEY,
name TEXT NOT NULL,
badge_label TEXT NOT NULL,
signing_key TEXT NOT NULL,
contact_website TEXT NOT NULL,
contact_discord TEXT,
is_legacy INTEGER AS Boolean NOT NULL
);
INSERT INTO extension_store(index_url, name, badge_label, signing_key, contact_website, contact_discord, is_legacy)
SELECT base_url || '/repo.json', name, coalesce(short_name, name), signing_key_fingerprint, website, NULL, 1 FROM extension_repos;
DROP TABLE extension_repos;