Move source and network outside data

2017-01-20 21:24:31 +01:00
parent a4c145c1ef
commit 706163e7a6
95 changed files with 4082 additions and 4082 deletions
@@ -0,0 +1,80 @@
+package eu.kanade.tachiyomi.network
+
+import com.squareup.duktape.Duktape
+import okhttp3.HttpUrl
+import okhttp3.Interceptor
+import okhttp3.Request
+import okhttp3.Response
+
+class CloudflareInterceptor(private val cookies: PersistentCookieStore) : Interceptor {
+
+    //language=RegExp
+    private val operationPattern = Regex("""setTimeout\(function\(\)\{\s+(var (?:\w,)+f.+?\r?\n[\s\S]+?a\.value =.+?)\r?\n""")
+    
+    //language=RegExp
+    private val passPattern = Regex("""name="pass" value="(.+?)"""")
+
+    //language=RegExp
+    private val challengePattern = Regex("""name="jschl_vc" value="(\w+)"""")
+
+    override fun intercept(chain: Interceptor.Chain): Response {
+        val response = chain.proceed(chain.request())
+
+        // Check if we already solved a challenge
+        if (response.code() != 503 &&
+                cookies.get(response.request().url()).any { it.name() == "cf_clearance" }) {
+            return response
+        }
+
+        // Check if Cloudflare anti-bot is on
+        if ("URL=/cdn-cgi/" in response.header("Refresh", "")
+                && response.header("Server", "") == "cloudflare-nginx") {
+            return chain.proceed(resolveChallenge(response))
+        }
+
+        return response
+    }
+
+    private fun resolveChallenge(response: Response): Request {
+        val duktape = Duktape.create()
+        try {
+            val originalRequest = response.request()
+            val domain = originalRequest.url().host()
+            val content = response.body().string()
+
+            // CloudFlare requires waiting 4 seconds before resolving the challenge
+            Thread.sleep(4000)
+
+            val operation = operationPattern.find(content)?.groups?.get(1)?.value
+            val challenge = challengePattern.find(content)?.groups?.get(1)?.value
+            val pass = passPattern.find(content)?.groups?.get(1)?.value
+
+            if (operation == null || challenge == null || pass == null) {
+                throw RuntimeException("Failed resolving Cloudflare challenge")
+            }
+
+            val js = operation
+                    //language=RegExp
+                    .replace(Regex("""a\.value =(.+?) \+.*"""), "$1")
+                    //language=RegExp
+                    .replace(Regex("""\s{3,}[a-z](?: = |\.).+"""), "")
+                    .replace("\n", "")
+
+            val result = (duktape.evaluate(js) as Double).toInt()
+
+            val answer = "${result + domain.length}"
+
+            val url = HttpUrl.parse("http://$domain/cdn-cgi/l/chk_jschl").newBuilder()
+                    .addQueryParameter("jschl_vc", challenge)
+                    .addQueryParameter("pass", pass)
+                    .addQueryParameter("jschl_answer", answer)
+                    .toString()
+
+            val referer = originalRequest.url().toString()
+            return GET(url, originalRequest.headers().newBuilder().add("Referer", referer).build())
+        } finally {
+            duktape.close()
+        }
+    }
+
+}
@@ -0,0 +1,38 @@
+package eu.kanade.tachiyomi.network
+
+import android.content.Context
+import okhttp3.Cache
+import okhttp3.OkHttpClient
+import java.io.File
+
+class NetworkHelper(context: Context) {
+
+    private val cacheDir = File(context.cacheDir, "network_cache")
+
+    private val cacheSize = 5L * 1024 * 1024 // 5 MiB
+
+    private val cookieManager = PersistentCookieJar(context)
+
+    val client = OkHttpClient.Builder()
+            .cookieJar(cookieManager)
+            .cache(Cache(cacheDir, cacheSize))
+            .build()
+
+    val forceCacheClient = client.newBuilder()
+            .addNetworkInterceptor { chain ->
+                val originalResponse = chain.proceed(chain.request())
+                originalResponse.newBuilder()
+                        .removeHeader("Pragma")
+                        .header("Cache-Control", "max-age=600")
+                        .build()
+            }
+            .build()
+
+    val cloudflareClient = client.newBuilder()
+            .addInterceptor(CloudflareInterceptor(cookies))
+            .build()
+
+    val cookies: PersistentCookieStore
+        get() = cookieManager.store
+
+}
@@ -0,0 +1,70 @@
+package eu.kanade.tachiyomi.network
+
+import okhttp3.Call
+import okhttp3.OkHttpClient
+import okhttp3.Request
+import okhttp3.Response
+import rx.Observable
+import rx.Producer
+import rx.Subscription
+import java.util.concurrent.atomic.AtomicBoolean
+
+fun Call.asObservable(): Observable<Response> {
+    return Observable.create { subscriber ->
+        // Since Call is a one-shot type, clone it for each new subscriber.
+        val call = clone()
+
+        // Wrap the call in a helper which handles both unsubscription and backpressure.
+        val requestArbiter = object : AtomicBoolean(), Producer, Subscription {
+            override fun request(n: Long) {
+                if (n == 0L || !compareAndSet(false, true)) return
+
+                try {
+                    val response = call.execute()
+                    if (!subscriber.isUnsubscribed) {
+                        subscriber.onNext(response)
+                        subscriber.onCompleted()
+                    }
+                } catch (error: Exception) {
+                    if (!subscriber.isUnsubscribed) {
+                        subscriber.onError(error)
+                    }
+                }
+            }
+
+            override fun unsubscribe() {
+                call.cancel()
+            }
+
+            override fun isUnsubscribed(): Boolean {
+                return call.isCanceled
+            }
+        }
+
+        subscriber.add(requestArbiter)
+        subscriber.setProducer(requestArbiter)
+    }
+}
+
+fun Call.asObservableSuccess(): Observable<Response> {
+    return asObservable().doOnNext { response ->
+        if (!response.isSuccessful) {
+            response.close()
+            throw Exception("HTTP error ${response.code()}")
+        }
+    }
+}
+
+fun OkHttpClient.newCallWithProgress(request: Request, listener: ProgressListener): Call {
+    val progressClient = newBuilder()
+            .cache(null)
+            .addNetworkInterceptor { chain ->
+                val originalResponse = chain.proceed(chain.request())
+                originalResponse.newBuilder()
+                        .body(ProgressResponseBody(originalResponse.body(), listener))
+                        .build()
+            }
+            .build()
+
+    return progressClient.newCall(request)
+}
@@ -0,0 +1,19 @@
+package eu.kanade.tachiyomi.network
+
+import android.content.Context
+import okhttp3.Cookie
+import okhttp3.CookieJar
+import okhttp3.HttpUrl
+
+class PersistentCookieJar(context: Context) : CookieJar {
+
+    val store = PersistentCookieStore(context)
+
+    override fun saveFromResponse(url: HttpUrl, cookies: List<Cookie>) {
+        store.addAll(url, cookies)
+    }
+
+    override fun loadForRequest(url: HttpUrl): List<Cookie> {
+        return store.get(url)
+    }
+}
@@ -0,0 +1,75 @@
+package eu.kanade.tachiyomi.network
+
+import android.content.Context
+import okhttp3.Cookie
+import okhttp3.HttpUrl
+import java.net.URI
+import java.util.concurrent.ConcurrentHashMap
+
+class PersistentCookieStore(context: Context) {
+
+    private val cookieMap = ConcurrentHashMap<String, List<Cookie>>()
+    private val prefs = context.getSharedPreferences("cookie_store", Context.MODE_PRIVATE)
+
+    init {
+        for ((key, value) in prefs.all) {
+            @Suppress("UNCHECKED_CAST")
+            val cookies = value as? Set<String>
+            if (cookies != null) {
+                try {
+                    val url = HttpUrl.parse("http://$key")
+                    val nonExpiredCookies = cookies.map { Cookie.parse(url, it) }
+                            .filter { !it.hasExpired() }
+                    cookieMap.put(key, nonExpiredCookies)
+                } catch (e: Exception) {
+                    // Ignore
+                }
+            }
+        }
+    }
+
+    fun addAll(url: HttpUrl, cookies: List<Cookie>) {
+        synchronized(this) {
+            val key = url.uri().host
+
+            // Append or replace the cookies for this domain.
+            val cookiesForDomain = cookieMap[key].orEmpty().toMutableList()
+            for (cookie in cookies) {
+                // Find a cookie with the same name. Replace it if found, otherwise add a new one.
+                val pos = cookiesForDomain.indexOfFirst { it.name() == cookie.name() }
+                if (pos == -1) {
+                    cookiesForDomain.add(cookie)
+                } else {
+                    cookiesForDomain[pos] = cookie
+                }
+            }
+            cookieMap.put(key, cookiesForDomain)
+
+            // Get cookies to be stored in disk
+            val newValues = cookiesForDomain.asSequence()
+                    .filter { it.persistent() && !it.hasExpired() }
+                    .map { it.toString() }
+                    .toSet()
+
+            prefs.edit().putStringSet(key, newValues).apply()
+        }
+    }
+
+    fun removeAll() {
+        synchronized(this) {
+            prefs.edit().clear().apply()
+            cookieMap.clear()
+        }
+    }
+
+    fun get(url: HttpUrl) = get(url.uri().host)
+
+    fun get(uri: URI) = get(uri.host)
+
+    private fun get(url: String): List<Cookie> {
+        return cookieMap[url].orEmpty().filter { !it.hasExpired() }
+    }
+
+    private fun Cookie.hasExpired() = System.currentTimeMillis() >= expiresAt()
+
+}
@@ -0,0 +1,5 @@
+package eu.kanade.tachiyomi.network
+
+interface ProgressListener {
+    fun update(bytesRead: Long, contentLength: Long, done: Boolean)
+}
@@ -0,0 +1,40 @@
+package eu.kanade.tachiyomi.network
+
+import okhttp3.MediaType
+import okhttp3.ResponseBody
+import okio.*
+import java.io.IOException
+
+class ProgressResponseBody(private val responseBody: ResponseBody, private val progressListener: ProgressListener) : ResponseBody() {
+
+    private val bufferedSource: BufferedSource by lazy {
+        Okio.buffer(source(responseBody.source()))
+    }
+
+    override fun contentType(): MediaType {
+        return responseBody.contentType()
+    }
+
+    override fun contentLength(): Long {
+        return responseBody.contentLength()
+    }
+
+    override fun source(): BufferedSource {
+        return bufferedSource
+    }
+
+    private fun source(source: Source): Source {
+        return object : ForwardingSource(source) {
+            internal var totalBytesRead = 0L
+
+            @Throws(IOException::class)
+            override fun read(sink: Buffer, byteCount: Long): Long {
+                val bytesRead = super.read(sink, byteCount)
+                // read() returns the number of bytes read, or -1 if this source is exhausted.
+                totalBytesRead += if (bytesRead != -1L) bytesRead else 0
+                progressListener.update(totalBytesRead, responseBody.contentLength(), bytesRead == -1L)
+                return bytesRead
+            }
+        }
+    }
+}
@@ -0,0 +1,32 @@
+package eu.kanade.tachiyomi.network
+
+import okhttp3.*
+import java.util.concurrent.TimeUnit.MINUTES
+
+private val DEFAULT_CACHE_CONTROL = CacheControl.Builder().maxAge(10, MINUTES).build()
+private val DEFAULT_HEADERS = Headers.Builder().build()
+private val DEFAULT_BODY: RequestBody = FormBody.Builder().build()
+
+fun GET(url: String,
+        headers: Headers = DEFAULT_HEADERS,
+        cache: CacheControl = DEFAULT_CACHE_CONTROL): Request {
+
+    return Request.Builder()
+            .url(url)
+            .headers(headers)
+            .cacheControl(cache)
+            .build()
+}
+
+fun POST(url: String,
+         headers: Headers = DEFAULT_HEADERS,
+         body: RequestBody = DEFAULT_BODY,
+         cache: CacheControl = DEFAULT_CACHE_CONTROL): Request {
+
+    return Request.Builder()
+            .url(url)
+            .post(body)
+            .headers(headers)
+            .cacheControl(cache)
+            .build()
+}