Unix line endings

This commit is contained in:
arkon
2020-02-29 13:03:29 -05:00
parent cae04656b9
commit c4dad1c20b
96 changed files with 9761 additions and 9737 deletions
@@ -1,154 +1,154 @@
package eu.kanade.tachiyomi.network
import android.annotation.SuppressLint
import android.content.Context
import android.os.Build
import android.os.Handler
import android.os.Looper
import android.webkit.WebResourceResponse
import android.webkit.WebSettings
import android.webkit.WebView
import eu.kanade.tachiyomi.util.WebViewClientCompat
import okhttp3.Interceptor
import okhttp3.Request
import okhttp3.Response
import java.io.IOException
import java.util.concurrent.CountDownLatch
import java.util.concurrent.TimeUnit
class CloudflareInterceptor(private val context: Context) : Interceptor {
private val serverCheck = arrayOf("cloudflare-nginx", "cloudflare")
private val handler = Handler(Looper.getMainLooper())
/**
* When this is called, it initializes the WebView if it wasn't already. We use this to avoid
* blocking the main thread too much. If used too often we could consider moving it to the
* Application class.
*/
private val initWebView by lazy {
if (Build.VERSION.SDK_INT >= 17) {
WebSettings.getDefaultUserAgent(context)
} else {
null
}
}
@Synchronized
override fun intercept(chain: Interceptor.Chain): Response {
initWebView
val response = chain.proceed(chain.request())
// Check if Cloudflare anti-bot is on
if (response.code() == 503 && response.header("Server") in serverCheck) {
try {
response.close()
val solutionRequest = resolveWithWebView(chain.request())
return chain.proceed(solutionRequest)
} catch (e: Exception) {
// Because OkHttp's enqueue only handles IOExceptions, wrap the exception so that
// we don't crash the entire app
throw IOException(e)
}
}
return response
}
private fun isChallengeSolutionUrl(url: String): Boolean {
return "chk_jschl" in url
}
@SuppressLint("SetJavaScriptEnabled")
private fun resolveWithWebView(request: Request): Request {
// We need to lock this thread until the WebView finds the challenge solution url, because
// OkHttp doesn't support asynchronous interceptors.
val latch = CountDownLatch(1)
var webView: WebView? = null
var solutionUrl: String? = null
var challengeFound = false
val origRequestUrl = request.url().toString()
val headers = request.headers().toMultimap().mapValues { it.value.getOrNull(0) ?: "" }
handler.post {
val view = WebView(context)
webView = view
view.settings.javaScriptEnabled = true
view.settings.userAgentString = request.header("User-Agent")
view.webViewClient = object : WebViewClientCompat() {
override fun shouldOverrideUrlCompat(view: WebView, url: String): Boolean {
if (isChallengeSolutionUrl(url)) {
solutionUrl = url
latch.countDown()
}
return solutionUrl != null
}
override fun shouldInterceptRequestCompat(
view: WebView,
url: String
): WebResourceResponse? {
if (solutionUrl != null) {
// Intercept any request when we have the solution.
return WebResourceResponse("text/plain", "UTF-8", null)
}
return null
}
override fun onPageFinished(view: WebView, url: String) {
// Http error codes are only received since M
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M &&
url == origRequestUrl && !challengeFound
) {
// The first request didn't return the challenge, abort.
latch.countDown()
}
}
override fun onReceivedErrorCompat(
view: WebView,
errorCode: Int,
description: String?,
failingUrl: String,
isMainFrame: Boolean
) {
if (isMainFrame) {
if (errorCode == 503) {
// Found the cloudflare challenge page.
challengeFound = true
} else {
// Unlock thread, the challenge wasn't found.
latch.countDown()
}
}
}
}
webView?.loadUrl(origRequestUrl, headers)
}
// Wait a reasonable amount of time to retrieve the solution. The minimum should be
// around 4 seconds but it can take more due to slow networks or server issues.
latch.await(12, TimeUnit.SECONDS)
handler.post {
webView?.stopLoading()
webView?.destroy()
}
val solution = solutionUrl ?: throw Exception("Challenge not found")
return Request.Builder().get()
.url(solution)
.headers(request.headers())
.addHeader("Referer", origRequestUrl)
.addHeader("Accept", "text/html,application/xhtml+xml,application/xml")
.addHeader("Accept-Language", "en")
.build()
}
}
package eu.kanade.tachiyomi.network
import android.annotation.SuppressLint
import android.content.Context
import android.os.Build
import android.os.Handler
import android.os.Looper
import android.webkit.WebResourceResponse
import android.webkit.WebSettings
import android.webkit.WebView
import eu.kanade.tachiyomi.util.WebViewClientCompat
import okhttp3.Interceptor
import okhttp3.Request
import okhttp3.Response
import java.io.IOException
import java.util.concurrent.CountDownLatch
import java.util.concurrent.TimeUnit
class CloudflareInterceptor(private val context: Context) : Interceptor {
private val serverCheck = arrayOf("cloudflare-nginx", "cloudflare")
private val handler = Handler(Looper.getMainLooper())
/**
* When this is called, it initializes the WebView if it wasn't already. We use this to avoid
* blocking the main thread too much. If used too often we could consider moving it to the
* Application class.
*/
private val initWebView by lazy {
if (Build.VERSION.SDK_INT >= 17) {
WebSettings.getDefaultUserAgent(context)
} else {
null
}
}
@Synchronized
override fun intercept(chain: Interceptor.Chain): Response {
initWebView
val response = chain.proceed(chain.request())
// Check if Cloudflare anti-bot is on
if (response.code() == 503 && response.header("Server") in serverCheck) {
try {
response.close()
val solutionRequest = resolveWithWebView(chain.request())
return chain.proceed(solutionRequest)
} catch (e: Exception) {
// Because OkHttp's enqueue only handles IOExceptions, wrap the exception so that
// we don't crash the entire app
throw IOException(e)
}
}
return response
}
private fun isChallengeSolutionUrl(url: String): Boolean {
return "chk_jschl" in url
}
@SuppressLint("SetJavaScriptEnabled")
private fun resolveWithWebView(request: Request): Request {
// We need to lock this thread until the WebView finds the challenge solution url, because
// OkHttp doesn't support asynchronous interceptors.
val latch = CountDownLatch(1)
var webView: WebView? = null
var solutionUrl: String? = null
var challengeFound = false
val origRequestUrl = request.url().toString()
val headers = request.headers().toMultimap().mapValues { it.value.getOrNull(0) ?: "" }
handler.post {
val view = WebView(context)
webView = view
view.settings.javaScriptEnabled = true
view.settings.userAgentString = request.header("User-Agent")
view.webViewClient = object : WebViewClientCompat() {
override fun shouldOverrideUrlCompat(view: WebView, url: String): Boolean {
if (isChallengeSolutionUrl(url)) {
solutionUrl = url
latch.countDown()
}
return solutionUrl != null
}
override fun shouldInterceptRequestCompat(
view: WebView,
url: String
): WebResourceResponse? {
if (solutionUrl != null) {
// Intercept any request when we have the solution.
return WebResourceResponse("text/plain", "UTF-8", null)
}
return null
}
override fun onPageFinished(view: WebView, url: String) {
// Http error codes are only received since M
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M &&
url == origRequestUrl && !challengeFound
) {
// The first request didn't return the challenge, abort.
latch.countDown()
}
}
override fun onReceivedErrorCompat(
view: WebView,
errorCode: Int,
description: String?,
failingUrl: String,
isMainFrame: Boolean
) {
if (isMainFrame) {
if (errorCode == 503) {
// Found the cloudflare challenge page.
challengeFound = true
} else {
// Unlock thread, the challenge wasn't found.
latch.countDown()
}
}
}
}
webView?.loadUrl(origRequestUrl, headers)
}
// Wait a reasonable amount of time to retrieve the solution. The minimum should be
// around 4 seconds but it can take more due to slow networks or server issues.
latch.await(12, TimeUnit.SECONDS)
handler.post {
webView?.stopLoading()
webView?.destroy()
}
val solution = solutionUrl ?: throw Exception("Challenge not found")
return Request.Builder().get()
.url(solution)
.headers(request.headers())
.addHeader("Referer", origRequestUrl)
.addHeader("Accept", "text/html,application/xhtml+xml,application/xml")
.addHeader("Accept-Language", "en")
.build()
}
}
@@ -1,117 +1,117 @@
package eu.kanade.tachiyomi.network
import android.content.Context
import android.os.Build
import okhttp3.*
import java.io.File
import java.io.IOException
import java.net.InetAddress
import java.net.Socket
import java.net.UnknownHostException
import java.security.KeyManagementException
import java.security.KeyStore
import java.security.NoSuchAlgorithmException
import javax.net.ssl.*
class NetworkHelper(context: Context) {
private val cacheDir = File(context.cacheDir, "network_cache")
private val cacheSize = 5L * 1024 * 1024 // 5 MiB
val cookieManager = AndroidCookieJar(context)
val client = OkHttpClient.Builder()
.cookieJar(cookieManager)
.cache(Cache(cacheDir, cacheSize))
.enableTLS12()
.build()
val cloudflareClient = client.newBuilder()
.addInterceptor(CloudflareInterceptor(context))
.build()
private fun OkHttpClient.Builder.enableTLS12(): OkHttpClient.Builder {
if (Build.VERSION.SDK_INT > Build.VERSION_CODES.KITKAT) {
return this
}
val trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm())
trustManagerFactory.init(null as KeyStore?)
val trustManagers = trustManagerFactory.trustManagers
if (trustManagers.size == 1 && trustManagers[0] is X509TrustManager) {
class TLSSocketFactory @Throws(KeyManagementException::class, NoSuchAlgorithmException::class)
constructor() : SSLSocketFactory() {
private val internalSSLSocketFactory: SSLSocketFactory
init {
val context = SSLContext.getInstance("TLS")
context.init(null, null, null)
internalSSLSocketFactory = context.socketFactory
}
override fun getDefaultCipherSuites(): Array<String> {
return internalSSLSocketFactory.defaultCipherSuites
}
override fun getSupportedCipherSuites(): Array<String> {
return internalSSLSocketFactory.supportedCipherSuites
}
@Throws(IOException::class)
override fun createSocket(): Socket? {
return enableTLSOnSocket(internalSSLSocketFactory.createSocket())
}
@Throws(IOException::class)
override fun createSocket(s: Socket, host: String, port: Int, autoClose: Boolean): Socket? {
return enableTLSOnSocket(internalSSLSocketFactory.createSocket(s, host, port, autoClose))
}
@Throws(IOException::class, UnknownHostException::class)
override fun createSocket(host: String, port: Int): Socket? {
return enableTLSOnSocket(internalSSLSocketFactory.createSocket(host, port))
}
@Throws(IOException::class, UnknownHostException::class)
override fun createSocket(host: String, port: Int, localHost: InetAddress, localPort: Int): Socket? {
return enableTLSOnSocket(internalSSLSocketFactory.createSocket(host, port, localHost, localPort))
}
@Throws(IOException::class)
override fun createSocket(host: InetAddress, port: Int): Socket? {
return enableTLSOnSocket(internalSSLSocketFactory.createSocket(host, port))
}
@Throws(IOException::class)
override fun createSocket(address: InetAddress, port: Int, localAddress: InetAddress, localPort: Int): Socket? {
return enableTLSOnSocket(internalSSLSocketFactory.createSocket(address, port, localAddress, localPort))
}
private fun enableTLSOnSocket(socket: Socket?): Socket? {
if (socket != null && socket is SSLSocket) {
socket.enabledProtocols = socket.supportedProtocols
}
return socket
}
}
sslSocketFactory(TLSSocketFactory(), trustManagers[0] as X509TrustManager)
}
val specCompat = ConnectionSpec.Builder(ConnectionSpec.MODERN_TLS)
.tlsVersions(TlsVersion.TLS_1_2, TlsVersion.TLS_1_1, TlsVersion.TLS_1_0)
.cipherSuites(
*ConnectionSpec.MODERN_TLS.cipherSuites().orEmpty().toTypedArray(),
CipherSuite.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
CipherSuite.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA
)
.build()
val specs = listOf(specCompat, ConnectionSpec.CLEARTEXT)
connectionSpecs(specs)
return this
}
}
package eu.kanade.tachiyomi.network
import android.content.Context
import android.os.Build
import okhttp3.*
import java.io.File
import java.io.IOException
import java.net.InetAddress
import java.net.Socket
import java.net.UnknownHostException
import java.security.KeyManagementException
import java.security.KeyStore
import java.security.NoSuchAlgorithmException
import javax.net.ssl.*
class NetworkHelper(context: Context) {
private val cacheDir = File(context.cacheDir, "network_cache")
private val cacheSize = 5L * 1024 * 1024 // 5 MiB
val cookieManager = AndroidCookieJar(context)
val client = OkHttpClient.Builder()
.cookieJar(cookieManager)
.cache(Cache(cacheDir, cacheSize))
.enableTLS12()
.build()
val cloudflareClient = client.newBuilder()
.addInterceptor(CloudflareInterceptor(context))
.build()
private fun OkHttpClient.Builder.enableTLS12(): OkHttpClient.Builder {
if (Build.VERSION.SDK_INT > Build.VERSION_CODES.KITKAT) {
return this
}
val trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm())
trustManagerFactory.init(null as KeyStore?)
val trustManagers = trustManagerFactory.trustManagers
if (trustManagers.size == 1 && trustManagers[0] is X509TrustManager) {
class TLSSocketFactory @Throws(KeyManagementException::class, NoSuchAlgorithmException::class)
constructor() : SSLSocketFactory() {
private val internalSSLSocketFactory: SSLSocketFactory
init {
val context = SSLContext.getInstance("TLS")
context.init(null, null, null)
internalSSLSocketFactory = context.socketFactory
}
override fun getDefaultCipherSuites(): Array<String> {
return internalSSLSocketFactory.defaultCipherSuites
}
override fun getSupportedCipherSuites(): Array<String> {
return internalSSLSocketFactory.supportedCipherSuites
}
@Throws(IOException::class)
override fun createSocket(): Socket? {
return enableTLSOnSocket(internalSSLSocketFactory.createSocket())
}
@Throws(IOException::class)
override fun createSocket(s: Socket, host: String, port: Int, autoClose: Boolean): Socket? {
return enableTLSOnSocket(internalSSLSocketFactory.createSocket(s, host, port, autoClose))
}
@Throws(IOException::class, UnknownHostException::class)
override fun createSocket(host: String, port: Int): Socket? {
return enableTLSOnSocket(internalSSLSocketFactory.createSocket(host, port))
}
@Throws(IOException::class, UnknownHostException::class)
override fun createSocket(host: String, port: Int, localHost: InetAddress, localPort: Int): Socket? {
return enableTLSOnSocket(internalSSLSocketFactory.createSocket(host, port, localHost, localPort))
}
@Throws(IOException::class)
override fun createSocket(host: InetAddress, port: Int): Socket? {
return enableTLSOnSocket(internalSSLSocketFactory.createSocket(host, port))
}
@Throws(IOException::class)
override fun createSocket(address: InetAddress, port: Int, localAddress: InetAddress, localPort: Int): Socket? {
return enableTLSOnSocket(internalSSLSocketFactory.createSocket(address, port, localAddress, localPort))
}
private fun enableTLSOnSocket(socket: Socket?): Socket? {
if (socket != null && socket is SSLSocket) {
socket.enabledProtocols = socket.supportedProtocols
}
return socket
}
}
sslSocketFactory(TLSSocketFactory(), trustManagers[0] as X509TrustManager)
}
val specCompat = ConnectionSpec.Builder(ConnectionSpec.MODERN_TLS)
.tlsVersions(TlsVersion.TLS_1_2, TlsVersion.TLS_1_1, TlsVersion.TLS_1_0)
.cipherSuites(
*ConnectionSpec.MODERN_TLS.cipherSuites().orEmpty().toTypedArray(),
CipherSuite.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
CipherSuite.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA
)
.build()
val specs = listOf(specCompat, ConnectionSpec.CLEARTEXT)
connectionSpecs(specs)
return this
}
}
@@ -1,70 +1,70 @@
package eu.kanade.tachiyomi.network
import okhttp3.Call
import okhttp3.OkHttpClient
import okhttp3.Request
import okhttp3.Response
import rx.Observable
import rx.Producer
import rx.Subscription
import java.util.concurrent.atomic.AtomicBoolean
fun Call.asObservable(): Observable<Response> {
return Observable.unsafeCreate { subscriber ->
// Since Call is a one-shot type, clone it for each new subscriber.
val call = clone()
// Wrap the call in a helper which handles both unsubscription and backpressure.
val requestArbiter = object : AtomicBoolean(), Producer, Subscription {
override fun request(n: Long) {
if (n == 0L || !compareAndSet(false, true)) return
try {
val response = call.execute()
if (!subscriber.isUnsubscribed) {
subscriber.onNext(response)
subscriber.onCompleted()
}
} catch (error: Exception) {
if (!subscriber.isUnsubscribed) {
subscriber.onError(error)
}
}
}
override fun unsubscribe() {
call.cancel()
}
override fun isUnsubscribed(): Boolean {
return call.isCanceled
}
}
subscriber.add(requestArbiter)
subscriber.setProducer(requestArbiter)
}
}
fun Call.asObservableSuccess(): Observable<Response> {
return asObservable().doOnNext { response ->
if (!response.isSuccessful) {
response.close()
throw Exception("HTTP error ${response.code()}")
}
}
}
fun OkHttpClient.newCallWithProgress(request: Request, listener: ProgressListener): Call {
val progressClient = newBuilder()
.cache(null)
.addNetworkInterceptor { chain ->
val originalResponse = chain.proceed(chain.request())
originalResponse.newBuilder()
.body(ProgressResponseBody(originalResponse.body()!!, listener))
.build()
}
.build()
return progressClient.newCall(request)
package eu.kanade.tachiyomi.network
import okhttp3.Call
import okhttp3.OkHttpClient
import okhttp3.Request
import okhttp3.Response
import rx.Observable
import rx.Producer
import rx.Subscription
import java.util.concurrent.atomic.AtomicBoolean
fun Call.asObservable(): Observable<Response> {
return Observable.unsafeCreate { subscriber ->
// Since Call is a one-shot type, clone it for each new subscriber.
val call = clone()
// Wrap the call in a helper which handles both unsubscription and backpressure.
val requestArbiter = object : AtomicBoolean(), Producer, Subscription {
override fun request(n: Long) {
if (n == 0L || !compareAndSet(false, true)) return
try {
val response = call.execute()
if (!subscriber.isUnsubscribed) {
subscriber.onNext(response)
subscriber.onCompleted()
}
} catch (error: Exception) {
if (!subscriber.isUnsubscribed) {
subscriber.onError(error)
}
}
}
override fun unsubscribe() {
call.cancel()
}
override fun isUnsubscribed(): Boolean {
return call.isCanceled
}
}
subscriber.add(requestArbiter)
subscriber.setProducer(requestArbiter)
}
}
fun Call.asObservableSuccess(): Observable<Response> {
return asObservable().doOnNext { response ->
if (!response.isSuccessful) {
response.close()
throw Exception("HTTP error ${response.code()}")
}
}
}
fun OkHttpClient.newCallWithProgress(request: Request, listener: ProgressListener): Call {
val progressClient = newBuilder()
.cache(null)
.addNetworkInterceptor { chain ->
val originalResponse = chain.proceed(chain.request())
originalResponse.newBuilder()
.body(ProgressResponseBody(originalResponse.body()!!, listener))
.build()
}
.build()
return progressClient.newCall(request)
}
@@ -1,5 +1,5 @@
package eu.kanade.tachiyomi.network
interface ProgressListener {
fun update(bytesRead: Long, contentLength: Long, done: Boolean)
package eu.kanade.tachiyomi.network
interface ProgressListener {
fun update(bytesRead: Long, contentLength: Long, done: Boolean)
}
@@ -1,40 +1,40 @@
package eu.kanade.tachiyomi.network
import okhttp3.MediaType
import okhttp3.ResponseBody
import okio.*
import java.io.IOException
class ProgressResponseBody(private val responseBody: ResponseBody, private val progressListener: ProgressListener) : ResponseBody() {
private val bufferedSource: BufferedSource by lazy {
Okio.buffer(source(responseBody.source()))
}
override fun contentType(): MediaType {
return responseBody.contentType()!!
}
override fun contentLength(): Long {
return responseBody.contentLength()
}
override fun source(): BufferedSource {
return bufferedSource
}
private fun source(source: Source): Source {
return object : ForwardingSource(source) {
internal var totalBytesRead = 0L
@Throws(IOException::class)
override fun read(sink: Buffer, byteCount: Long): Long {
val bytesRead = super.read(sink, byteCount)
// read() returns the number of bytes read, or -1 if this source is exhausted.
totalBytesRead += if (bytesRead != -1L) bytesRead else 0
progressListener.update(totalBytesRead, responseBody.contentLength(), bytesRead == -1L)
return bytesRead
}
}
}
package eu.kanade.tachiyomi.network
import okhttp3.MediaType
import okhttp3.ResponseBody
import okio.*
import java.io.IOException
class ProgressResponseBody(private val responseBody: ResponseBody, private val progressListener: ProgressListener) : ResponseBody() {
private val bufferedSource: BufferedSource by lazy {
Okio.buffer(source(responseBody.source()))
}
override fun contentType(): MediaType {
return responseBody.contentType()!!
}
override fun contentLength(): Long {
return responseBody.contentLength()
}
override fun source(): BufferedSource {
return bufferedSource
}
private fun source(source: Source): Source {
return object : ForwardingSource(source) {
internal var totalBytesRead = 0L
@Throws(IOException::class)
override fun read(sink: Buffer, byteCount: Long): Long {
val bytesRead = super.read(sink, byteCount)
// read() returns the number of bytes read, or -1 if this source is exhausted.
totalBytesRead += if (bytesRead != -1L) bytesRead else 0
progressListener.update(totalBytesRead, responseBody.contentLength(), bytesRead == -1L)
return bytesRead
}
}
}
}
@@ -1,32 +1,32 @@
package eu.kanade.tachiyomi.network
import okhttp3.*
import java.util.concurrent.TimeUnit.MINUTES
private val DEFAULT_CACHE_CONTROL = CacheControl.Builder().maxAge(10, MINUTES).build()
private val DEFAULT_HEADERS = Headers.Builder().build()
private val DEFAULT_BODY: RequestBody = FormBody.Builder().build()
fun GET(url: String,
headers: Headers = DEFAULT_HEADERS,
cache: CacheControl = DEFAULT_CACHE_CONTROL): Request {
return Request.Builder()
.url(url)
.headers(headers)
.cacheControl(cache)
.build()
}
fun POST(url: String,
headers: Headers = DEFAULT_HEADERS,
body: RequestBody = DEFAULT_BODY,
cache: CacheControl = DEFAULT_CACHE_CONTROL): Request {
return Request.Builder()
.url(url)
.post(body)
.headers(headers)
.cacheControl(cache)
.build()
}
package eu.kanade.tachiyomi.network
import okhttp3.*
import java.util.concurrent.TimeUnit.MINUTES
private val DEFAULT_CACHE_CONTROL = CacheControl.Builder().maxAge(10, MINUTES).build()
private val DEFAULT_HEADERS = Headers.Builder().build()
private val DEFAULT_BODY: RequestBody = FormBody.Builder().build()
fun GET(url: String,
headers: Headers = DEFAULT_HEADERS,
cache: CacheControl = DEFAULT_CACHE_CONTROL): Request {
return Request.Builder()
.url(url)
.headers(headers)
.cacheControl(cache)
.build()
}
fun POST(url: String,
headers: Headers = DEFAULT_HEADERS,
body: RequestBody = DEFAULT_BODY,
cache: CacheControl = DEFAULT_CACHE_CONTROL): Request {
return Request.Builder()
.url(url)
.post(body)
.headers(headers)
.cacheControl(cache)
.build()
}