Downloading extensions from Github Repo. (#1101)

Downloading extensions from Github Repo.
This commit is contained in:
Carlos
2018-02-05 16:50:56 -05:00
committed by inorichi
parent a71c805959
commit 854112095b
46 changed files with 2319 additions and 615 deletions
@@ -1,6 +1,5 @@
package eu.kanade.tachiyomi.source.online
import eu.kanade.tachiyomi.data.preference.PreferencesHelper
import eu.kanade.tachiyomi.network.GET
import eu.kanade.tachiyomi.network.NetworkHelper
import eu.kanade.tachiyomi.network.asObservableSuccess
@@ -28,10 +27,12 @@ abstract class HttpSource : CatalogueSource {
*/
protected val network: NetworkHelper by injectLazy()
/**
* Preferences helper.
*/
protected val preferences: PreferencesHelper by injectLazy()
// /**
// * Preferences that a source may need.
// */
// val preferences: SharedPreferences by lazy {
// Injekt.get<Application>().getSharedPreferences("source_$id", Context.MODE_PRIVATE)
// }
/**
* Base url of the website without the trailing slash, like: http://mysite.com
@@ -1,231 +0,0 @@
package eu.kanade.tachiyomi.source.online
import eu.kanade.tachiyomi.network.GET
import eu.kanade.tachiyomi.network.POST
import eu.kanade.tachiyomi.source.model.*
import eu.kanade.tachiyomi.util.asJsoup
import eu.kanade.tachiyomi.util.attrOrText
import okhttp3.Request
import okhttp3.Response
import org.jsoup.Jsoup
import org.jsoup.nodes.Element
import java.text.SimpleDateFormat
import java.util.*
class YamlHttpSource(mappings: Map<*, *>) : HttpSource() {
val map = YamlSourceNode(mappings)
override val name: String
get() = map.name
override val baseUrl = map.host.let {
if (it.endsWith("/")) it.dropLast(1) else it
}
override val lang = map.lang.toLowerCase()
override val supportsLatest = map.latestupdates != null
override val client = when (map.client) {
"cloudflare" -> network.cloudflareClient
else -> network.client
}
override val id = map.id.let {
(it as? Int ?: (lang.toUpperCase().hashCode() + 31 * it.hashCode()) and 0x7fffffff).toLong()
}
// Ugly, but needed after the changes
var popularNextPage: String? = null
var searchNextPage: String? = null
var latestNextPage: String? = null
override fun popularMangaRequest(page: Int): Request {
val url = if (page == 1) {
popularNextPage = null
map.popular.url
} else {
popularNextPage!!
}
return when (map.popular.method?.toLowerCase()) {
"post" -> POST(url, headers, map.popular.createForm())
else -> GET(url, headers)
}
}
override fun popularMangaParse(response: Response): MangasPage {
val document = response.asJsoup()
val mangas = document.select(map.popular.manga_css).map { element ->
SManga.create().apply {
title = element.text()
setUrlWithoutDomain(element.attr("href"))
}
}
popularNextPage = map.popular.next_url_css?.let { selector ->
document.select(selector).first()?.absUrl("href")
}
return MangasPage(mangas, popularNextPage != null)
}
override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
val url = if (page == 1) {
searchNextPage = null
map.search.url.replace("\$query", query)
} else {
searchNextPage!!
}
return when (map.search.method?.toLowerCase()) {
"post" -> POST(url, headers, map.search.createForm())
else -> GET(url, headers)
}
}
override fun searchMangaParse(response: Response): MangasPage {
val document = response.asJsoup()
val mangas = document.select(map.search.manga_css).map { element ->
SManga.create().apply {
title = element.text()
setUrlWithoutDomain(element.attr("href"))
}
}
searchNextPage = map.search.next_url_css?.let { selector ->
document.select(selector).first()?.absUrl("href")
}
return MangasPage(mangas, searchNextPage != null)
}
override fun latestUpdatesRequest(page: Int): Request {
val url = if (page == 1) {
latestNextPage = null
map.latestupdates!!.url
} else {
latestNextPage!!
}
return when (map.latestupdates!!.method?.toLowerCase()) {
"post" -> POST(url, headers, map.latestupdates.createForm())
else -> GET(url, headers)
}
}
override fun latestUpdatesParse(response: Response): MangasPage {
val document = response.asJsoup()
val mangas = document.select(map.latestupdates!!.manga_css).map { element ->
SManga.create().apply {
title = element.text()
setUrlWithoutDomain(element.attr("href"))
}
}
popularNextPage = map.latestupdates.next_url_css?.let { selector ->
document.select(selector).first()?.absUrl("href")
}
return MangasPage(mangas, popularNextPage != null)
}
override fun mangaDetailsParse(response: Response): SManga {
val document = response.asJsoup()
val manga = SManga.create()
with(map.manga) {
val pool = parts.get(document)
manga.author = author?.process(document, pool)
manga.artist = artist?.process(document, pool)
manga.description = summary?.process(document, pool)
manga.thumbnail_url = cover?.process(document, pool)
manga.genre = genres?.process(document, pool)
manga.status = status?.getStatus(document, pool) ?: SManga.UNKNOWN
}
return manga
}
override fun chapterListParse(response: Response): List<SChapter> {
val document = response.asJsoup()
val chapters = mutableListOf<SChapter>()
with(map.chapters) {
val pool = emptyMap<String, Element>()
val dateFormat = SimpleDateFormat(date?.format, Locale.ENGLISH)
for (element in document.select(chapter_css)) {
val chapter = SChapter.create()
element.select(title).first().let {
chapter.name = it.text()
chapter.setUrlWithoutDomain(it.attr("href"))
}
val dateElement = element.select(date?.select).first()
chapter.date_upload = date?.getDate(dateElement, pool, dateFormat)?.time ?: 0
chapters.add(chapter)
}
}
return chapters
}
override fun pageListParse(response: Response): List<Page> {
val body = response.body()!!.string()
val url = response.request().url().toString()
val pages = mutableListOf<Page>()
val document by lazy { Jsoup.parse(body, url) }
with(map.pages) {
// Capture a list of values where page urls will be resolved.
val capturedPages = if (pages_regex != null)
pages_regex!!.toRegex().findAll(body).map { it.value }.toList()
else if (pages_css != null)
document.select(pages_css).map { it.attrOrText(pages_attr!!) }
else
null
// For each captured value, obtain the url and create a new page.
capturedPages?.forEach { value ->
// If the captured value isn't an url, we have to use replaces with the chapter url.
val pageUrl = if (replace != null && replacement != null)
url.replace(replace!!.toRegex(), replacement!!.replace("\$value", value))
else
value
pages.add(Page(pages.size, pageUrl))
}
// Capture a list of images.
val capturedImages = if (image_regex != null)
image_regex!!.toRegex().findAll(body).map { it.groups[1]?.value }.toList()
else if (image_css != null)
document.select(image_css).map { it.absUrl(image_attr) }
else
null
// Assign the image url to each page
capturedImages?.forEachIndexed { i, url ->
val page = pages.getOrElse(i) { Page(i, "").apply { pages.add(this) } }
page.imageUrl = url
}
}
return pages
}
override fun imageUrlParse(response: Response): String {
val body = response.body()!!.string()
val url = response.request().url().toString()
with(map.pages) {
return if (image_regex != null)
image_regex!!.toRegex().find(body)!!.groups[1]!!.value
else if (image_css != null)
Jsoup.parse(body, url).select(image_css).first().absUrl(image_attr)
else
throw Exception("image_regex and image_css are null")
}
}
}
@@ -1,234 +0,0 @@
@file:Suppress("UNCHECKED_CAST")
package eu.kanade.tachiyomi.source.online
import eu.kanade.tachiyomi.source.model.SManga
import okhttp3.FormBody
import okhttp3.RequestBody
import org.jsoup.nodes.Document
import org.jsoup.nodes.Element
import java.text.ParseException
import java.text.SimpleDateFormat
import java.util.*
private fun toMap(map: Any?) = map as? Map<String, Any?>
class YamlSourceNode(uncheckedMap: Map<*, *>) {
val map = toMap(uncheckedMap)!!
val id: Any by map
val name: String by map
val host: String by map
val lang: String by map
val client: String?
get() = map["client"] as? String
val popular = PopularNode(toMap(map["popular"])!!)
val latestupdates = toMap(map["latest_updates"])?.let { LatestUpdatesNode(it) }
val search = SearchNode(toMap(map["search"])!!)
val manga = MangaNode(toMap(map["manga"])!!)
val chapters = ChaptersNode(toMap(map["chapters"])!!)
val pages = PagesNode(toMap(map["pages"])!!)
}
interface RequestableNode {
val map: Map<String, Any?>
val url: String
get() = map["url"] as String
val method: String?
get() = map["method"] as? String
val payload: Map<String, String>?
get() = map["payload"] as? Map<String, String>
fun createForm(): RequestBody {
return FormBody.Builder().apply {
payload?.let {
for ((key, value) in it) {
add(key, value)
}
}
}.build()
}
}
class PopularNode(override val map: Map<String, Any?>): RequestableNode {
val manga_css: String by map
val next_url_css: String?
get() = map["next_url_css"] as? String
}
class LatestUpdatesNode(override val map: Map<String, Any?>): RequestableNode {
val manga_css: String by map
val next_url_css: String?
get() = map["next_url_css"] as? String
}
class SearchNode(override val map: Map<String, Any?>): RequestableNode {
val manga_css: String by map
val next_url_css: String?
get() = map["next_url_css"] as? String
}
class MangaNode(private val map: Map<String, Any?>) {
val parts = CacheNode(toMap(map["parts"]) ?: emptyMap())
val artist = toMap(map["artist"])?.let { SelectableNode(it) }
val author = toMap(map["author"])?.let { SelectableNode(it) }
val summary = toMap(map["summary"])?.let { SelectableNode(it) }
val status = toMap(map["status"])?.let { StatusNode(it) }
val genres = toMap(map["genres"])?.let { SelectableNode(it) }
val cover = toMap(map["cover"])?.let { CoverNode(it) }
}
class ChaptersNode(private val map: Map<String, Any?>) {
val chapter_css: String by map
val title: String by map
val date = toMap(toMap(map["date"]))?.let { DateNode(it) }
}
class CacheNode(private val map: Map<String, Any?>) {
fun get(document: Document) = map.mapValues { document.select(it.value as String).first() }
}
open class SelectableNode(private val map: Map<String, Any?>) {
val select: String by map
val from: String?
get() = map["from"] as? String
open val attr: String?
get() = map["attr"] as? String
val capture: String?
get() = map["capture"] as? String
fun process(document: Element, cache: Map<String, Element>): String {
val parent = from?.let { cache[it] } ?: document
val node = parent.select(select).first()
var text = attr?.let { node.attr(it) } ?: node.text()
capture?.let {
text = Regex(it).find(text)?.groupValues?.get(1) ?: text
}
return text
}
}
class StatusNode(private val map: Map<String, Any?>) : SelectableNode(map) {
val complete: String?
get() = map["complete"] as? String
val ongoing: String?
get() = map["ongoing"] as? String
val licensed: String?
get() = map["licensed"] as? String
fun getStatus(document: Element, cache: Map<String, Element>): Int {
val text = process(document, cache)
complete?.let {
if (text.contains(it)) return SManga.COMPLETED
}
ongoing?.let {
if (text.contains(it)) return SManga.ONGOING
}
licensed?.let {
if (text.contains(it)) return SManga.LICENSED
}
return SManga.UNKNOWN
}
}
class CoverNode(private val map: Map<String, Any?>) : SelectableNode(map) {
override val attr: String?
get() = map["attr"] as? String ?: "src"
}
class DateNode(private val map: Map<String, Any?>) : SelectableNode(map) {
val format: String by map
fun getDate(document: Element, cache: Map<String, Element>, formatter: SimpleDateFormat): Date {
val text = process(document, cache)
try {
return formatter.parse(text)
} catch (exception: ParseException) {}
for (i in 0..7) {
(map["day$i"] as? List<String>)?.let {
it.find { it.toRegex().containsMatchIn(text) }?.let {
return Calendar.getInstance().apply { add(Calendar.DATE, -i) }.time
}
}
}
return Date(0)
}
}
class PagesNode(private val map: Map<String, Any?>) {
val pages_regex: String?
get() = map["pages_regex"] as? String
val pages_css: String?
get() = map["pages_css"] as? String
val pages_attr: String?
get() = map["pages_attr"] as? String ?: "value"
val replace: String?
get() = map["url_replace"] as? String
val replacement: String?
get() = map["url_replacement"] as? String
val image_regex: String?
get() = map["image_regex"] as? String
val image_css: String?
get() = map["image_css"] as? String
val image_attr: String
get() = map["image_attr"] as? String ?: "src"
}
@@ -1,6 +1,7 @@
package eu.kanade.tachiyomi.source.online.english
import android.text.Html
import eu.kanade.tachiyomi.data.preference.PreferencesHelper
import eu.kanade.tachiyomi.network.GET
import eu.kanade.tachiyomi.network.POST
import eu.kanade.tachiyomi.network.asObservable
@@ -9,14 +10,11 @@ import eu.kanade.tachiyomi.source.online.LoginSource
import eu.kanade.tachiyomi.source.online.ParsedHttpSource
import eu.kanade.tachiyomi.util.asJsoup
import eu.kanade.tachiyomi.util.selectText
import okhttp3.FormBody
import okhttp3.HttpUrl
import okhttp3.OkHttpClient
import okhttp3.Request
import okhttp3.Response
import okhttp3.*
import org.jsoup.nodes.Document
import org.jsoup.nodes.Element
import rx.Observable
import uy.kohesive.injekt.injectLazy
import java.net.URI
import java.text.ParseException
import java.text.SimpleDateFormat
@@ -25,6 +23,9 @@ import java.util.regex.Pattern
class Batoto : ParsedHttpSource(), LoginSource {
// TODO remove
private val preferences: PreferencesHelper by injectLazy()
override val id: Long = 1
override val name = "Batoto"