From f1f6a2b341ea554eb98ae797594b930e7d0f2f71 Mon Sep 17 00:00:00 2001 From: inorichi Date: Mon, 1 Apr 2019 17:20:13 +0200 Subject: [PATCH] Test solving Cloudflare's challenge with WebView --- .../data/track/myanimelist/MyAnimeList.kt | 12 +- .../tachiyomi/network/AndroidCookieJar.kt | 63 +++++++ .../network/CloudflareInterceptor.kt | 168 ++++++++++-------- .../kanade/tachiyomi/network/NetworkHelper.kt | 19 +- .../tachiyomi/network/PersistentCookieJar.kt | 19 -- .../network/PersistentCookieStore.kt | 78 -------- .../ui/setting/SettingsAdvancedController.kt | 2 +- .../tachiyomi/util/WebViewClientCompat.kt | 83 +++++++++ 8 files changed, 250 insertions(+), 194 deletions(-) create mode 100644 app/src/main/java/eu/kanade/tachiyomi/network/AndroidCookieJar.kt delete mode 100644 app/src/main/java/eu/kanade/tachiyomi/network/PersistentCookieJar.kt delete mode 100644 app/src/main/java/eu/kanade/tachiyomi/network/PersistentCookieStore.kt create mode 100644 app/src/main/java/eu/kanade/tachiyomi/util/WebViewClientCompat.kt diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/track/myanimelist/MyAnimeList.kt b/app/src/main/java/eu/kanade/tachiyomi/data/track/myanimelist/MyAnimeList.kt index f871db2b2..a83e8b9ff 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/track/myanimelist/MyAnimeList.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/track/myanimelist/MyAnimeList.kt @@ -7,9 +7,9 @@ import eu.kanade.tachiyomi.data.database.models.Track import eu.kanade.tachiyomi.data.preference.getOrDefault import eu.kanade.tachiyomi.data.track.TrackService import eu.kanade.tachiyomi.data.track.model.TrackSearch +import okhttp3.HttpUrl import rx.Completable import rx.Observable -import java.net.URI class Myanimelist(private val context: Context, id: Int) : TrackService(id) { @@ -114,23 +114,23 @@ class Myanimelist(private val context: Context, id: Int) : TrackService(id) { override fun logout() { super.logout() preferences.trackToken(this).delete() - networkService.cookies.remove(URI(BASE_URL)) + networkService.cookieManager.remove(HttpUrl.parse(BASE_URL)!!) } override val isLogged: Boolean get() = !getUsername().isEmpty() && !getPassword().isEmpty() && - checkCookies(URI(BASE_URL)) && + checkCookies() && !getCSRF().isEmpty() private fun getCSRF(): String = preferences.trackToken(this).getOrDefault() private fun saveCSRF(csrf: String) = preferences.trackToken(this).set(csrf) - private fun checkCookies(uri: URI): Boolean { + private fun checkCookies(): Boolean { var ckCount = 0 - - for (ck in networkService.cookies.get(uri)) { + val url = HttpUrl.parse(BASE_URL)!! + for (ck in networkService.cookieManager.get(url)) { if (ck.name() == USER_SESSION_COOKIE || ck.name() == LOGGED_IN_COOKIE) ckCount++ } diff --git a/app/src/main/java/eu/kanade/tachiyomi/network/AndroidCookieJar.kt b/app/src/main/java/eu/kanade/tachiyomi/network/AndroidCookieJar.kt new file mode 100644 index 000000000..8430b9cd0 --- /dev/null +++ b/app/src/main/java/eu/kanade/tachiyomi/network/AndroidCookieJar.kt @@ -0,0 +1,63 @@ +package eu.kanade.tachiyomi.network + +import android.content.Context +import android.os.Build +import android.webkit.CookieManager +import android.webkit.CookieSyncManager +import okhttp3.Cookie +import okhttp3.CookieJar +import okhttp3.HttpUrl + +class AndroidCookieJar(context: Context) : CookieJar { + + private val manager = CookieManager.getInstance() + + private val syncManager by lazy { CookieSyncManager.createInstance(context) } + + override fun saveFromResponse(url: HttpUrl, cookies: MutableList) { + val urlString = url.toString() + + for (cookie in cookies) { + manager.setCookie(urlString, cookie.toString()) + } + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.LOLLIPOP) { + syncManager.sync() + } + } + + override fun loadForRequest(url: HttpUrl): List { + return get(url) + } + + fun get(url: HttpUrl): List { + val cookies = manager.getCookie(url.toString()) + + return if (cookies != null && !cookies.isEmpty()) { + cookies.split(";").mapNotNull { Cookie.parse(url, it) } + } else { + emptyList() + } + } + + fun remove(url: HttpUrl) { + val cookies = manager.getCookie(url.toString()) ?: return + val domain = ".${url.host()}" + cookies.split(";") + .map { it.substringBefore("=") } + .onEach { manager.setCookie(domain, "$it=;Max-Age=-1") } + + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.LOLLIPOP) { + syncManager.sync() + } + } + + fun removeAll() { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { + manager.removeAllCookies {} + } else { + manager.removeAllCookie() + syncManager.sync() + } + } + +} diff --git a/app/src/main/java/eu/kanade/tachiyomi/network/CloudflareInterceptor.kt b/app/src/main/java/eu/kanade/tachiyomi/network/CloudflareInterceptor.kt index 641cdd960..9159cd995 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/network/CloudflareInterceptor.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/network/CloudflareInterceptor.kt @@ -1,31 +1,32 @@ package eu.kanade.tachiyomi.network -import com.squareup.duktape.Duktape -import okhttp3.* +import android.annotation.SuppressLint +import android.content.Context +import android.os.Handler +import android.os.HandlerThread +import android.webkit.WebResourceResponse +import android.webkit.WebView +import eu.kanade.tachiyomi.util.WebViewClientCompat +import okhttp3.Interceptor +import okhttp3.Request +import okhttp3.Response +import timber.log.Timber import java.io.IOException +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit -class CloudflareInterceptor : Interceptor { - - private val operationPattern = Regex("""setTimeout\(function\(\)\{\s+(var (?:\w,)+f.+?\r?\n[\s\S]+?a\.value =.+?)\r?\n""") - - private val passPattern = Regex("""name="pass" value="(.+?)"""") - - private val challengePattern = Regex("""name="jschl_vc" value="(\w+)"""") - - private val sPattern = Regex("""name="s" value="([^"]+)""") - - private val kPattern = Regex("""k\s+=\s+'([^']+)';""") +class CloudflareInterceptor(private val context: Context) : Interceptor { private val serverCheck = arrayOf("cloudflare-nginx", "cloudflare") - private interface IBase64 { - fun decode(input: String): String - } - - private val b64: IBase64 = object : IBase64 { - override fun decode(input: String): String { - return okio.ByteString.decodeBase64(input)!!.utf8() + private val handler by lazy { + val thread = HandlerThread("WebViewThread").apply { + uncaughtExceptionHandler = Thread.UncaughtExceptionHandler { _, e -> + Timber.e(e) + } + start() } + Handler(thread.looper) } @Synchronized @@ -34,8 +35,14 @@ class CloudflareInterceptor : Interceptor { // Check if Cloudflare anti-bot is on if (response.code() == 503 && response.header("Server") in serverCheck) { - return try { - chain.proceed(resolveChallenge(response)) + try { + response.close() + if (resolveWithWebView(chain.request())) { + // Retry original request + return chain.proceed(chain.request()) + } else { + throw Exception("Failed resolving Cloudflare challenge") + } } catch (e: Exception) { // Because OkHttp's enqueue only handles IOExceptions, wrap the exception so that // we don't crash the entire app @@ -46,65 +53,76 @@ class CloudflareInterceptor : Interceptor { return response } - private fun resolveChallenge(response: Response): Request { - Duktape.create().use { duktape -> - val originalRequest = response.request() - val url = originalRequest.url() - val domain = url.host() - val content = response.body()!!.string() + private fun isChallengeResolverUrl(url: String): Boolean { + return "chk_jschl" in url + } - // CloudFlare requires waiting 4 seconds before resolving the challenge - Thread.sleep(4000) + @SuppressLint("SetJavaScriptEnabled") + private fun resolveWithWebView(request: Request): Boolean { + val latch = CountDownLatch(1) - val operation = operationPattern.find(content)?.groups?.get(1)?.value - val challenge = challengePattern.find(content)?.groups?.get(1)?.value - val pass = passPattern.find(content)?.groups?.get(1)?.value - val s = sPattern.find(content)?.groups?.get(1)?.value + var result = false + var isResolvingChallenge = false - // If `k` is null, it uses old methods. - val k = kPattern.find(content)?.groups?.get(1)?.value ?: "" - val innerHTMLValue = Regex("""(.*)""") - .find(content)?.groups?.get(3)?.value ?: "" + val requestUrl = request.url().toString() + val headers = request.headers().toMultimap().mapValues { it.value.getOrNull(0) ?: "" } - if (operation == null || challenge == null || pass == null || s == null) { - throw Exception("Failed resolving Cloudflare challenge") + handler.post { + val view = WebView(context) + view.settings.javaScriptEnabled = true + view.settings.userAgentString = request.header("User-Agent") + view.webViewClient = object : WebViewClientCompat() { + + override fun shouldInterceptRequestCompat( + view: WebView, + url: String + ): WebResourceResponse? { + val isChallengeResolverUrl = isChallengeResolverUrl(url) + if (requestUrl != url && !isChallengeResolverUrl) { + return WebResourceResponse("text/plain", "UTF-8", null) + } + + if (isChallengeResolverUrl) { + isResolvingChallenge = true + } + return null + } + + override fun onPageFinished(view: WebView, url: String) { + super.onPageFinished(view, url) + if (isResolvingChallenge && url == requestUrl) { + setResultAndFinish(true) + } + } + + override fun onReceivedErrorCompat( + view: WebView, + errorCode: Int, + description: String?, + failingUrl: String, + isMainFrame: Boolean + ) { + if ((errorCode != 503 && requestUrl == failingUrl) || + isChallengeResolverUrl(failingUrl) + ) { + setResultAndFinish(false) + } + } + + private fun setResultAndFinish(resolved: Boolean) { + result = resolved + latch.countDown() + view.stopLoading() + view.destroy() + } } - // Export native Base64 decode function to js object. - duktape.set("b64", IBase64::class.java, b64) - - // Return simulated innerHTML when call DOM. - val simulatedDocumentJS = """var document = { getElementById: function (x) { return { innerHTML: "$innerHTMLValue" }; } }""" - - val js = operation - .replace(Regex("""a\.value = (.+\.toFixed\(10\);).+"""), "$1") - .replace(Regex("""\s{3,}[a-z](?: = |\.).+"""), "") - .replace("t.length", "${domain.length}") - .replace("\n", "") - - val result = duktape.evaluate("""$simulatedDocumentJS;$ATOB_JS;var t="$domain";$js""") as String - - val cloudflareUrl = HttpUrl.parse("${url.scheme()}://$domain/cdn-cgi/l/chk_jschl")!! - .newBuilder() - .addQueryParameter("jschl_vc", challenge) - .addQueryParameter("pass", pass) - .addQueryParameter("s", s) - .addQueryParameter("jschl_answer", result) - .toString() - - val cloudflareHeaders = originalRequest.headers() - .newBuilder() - .add("Referer", url.toString()) - .add("Accept", "text/html,application/xhtml+xml,application/xml") - .add("Accept-Language", "en") - .build() - - return GET(cloudflareUrl, cloudflareHeaders, cache = CacheControl.Builder().build()) + view.loadUrl(requestUrl, headers) } + + latch.await(12, TimeUnit.SECONDS) + + return result } - companion object { - // atob() is browser API, Using Android's own function. (java.util.Base64 can't be used because of min API level) - private const val ATOB_JS = """var atob = function (input) { return b64.decode(input) }""" - } -} \ No newline at end of file +} diff --git a/app/src/main/java/eu/kanade/tachiyomi/network/NetworkHelper.kt b/app/src/main/java/eu/kanade/tachiyomi/network/NetworkHelper.kt index 5e9389483..275dca17d 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/network/NetworkHelper.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/network/NetworkHelper.kt @@ -2,11 +2,7 @@ package eu.kanade.tachiyomi.network import android.content.Context import android.os.Build -import okhttp3.Cache -import okhttp3.CipherSuite -import okhttp3.ConnectionSpec -import okhttp3.OkHttpClient -import okhttp3.TlsVersion +import okhttp3.* import java.io.File import java.io.IOException import java.net.InetAddress @@ -15,11 +11,7 @@ import java.net.UnknownHostException import java.security.KeyManagementException import java.security.KeyStore import java.security.NoSuchAlgorithmException -import javax.net.ssl.SSLContext -import javax.net.ssl.SSLSocket -import javax.net.ssl.SSLSocketFactory -import javax.net.ssl.TrustManagerFactory -import javax.net.ssl.X509TrustManager +import javax.net.ssl.* class NetworkHelper(context: Context) { @@ -27,7 +19,7 @@ class NetworkHelper(context: Context) { private val cacheSize = 5L * 1024 * 1024 // 5 MiB - private val cookieManager = PersistentCookieJar(context) + val cookieManager = AndroidCookieJar(context) val client = OkHttpClient.Builder() .cookieJar(cookieManager) @@ -36,12 +28,9 @@ class NetworkHelper(context: Context) { .build() val cloudflareClient = client.newBuilder() - .addInterceptor(CloudflareInterceptor()) + .addInterceptor(CloudflareInterceptor(context)) .build() - val cookies: PersistentCookieStore - get() = cookieManager.store - private fun OkHttpClient.Builder.enableTLS12(): OkHttpClient.Builder { if (Build.VERSION.SDK_INT > Build.VERSION_CODES.KITKAT) { return this diff --git a/app/src/main/java/eu/kanade/tachiyomi/network/PersistentCookieJar.kt b/app/src/main/java/eu/kanade/tachiyomi/network/PersistentCookieJar.kt deleted file mode 100644 index fda979978..000000000 --- a/app/src/main/java/eu/kanade/tachiyomi/network/PersistentCookieJar.kt +++ /dev/null @@ -1,19 +0,0 @@ -package eu.kanade.tachiyomi.network - -import android.content.Context -import okhttp3.Cookie -import okhttp3.CookieJar -import okhttp3.HttpUrl - -class PersistentCookieJar(context: Context) : CookieJar { - - val store = PersistentCookieStore(context) - - override fun saveFromResponse(url: HttpUrl, cookies: List) { - store.addAll(url, cookies) - } - - override fun loadForRequest(url: HttpUrl): List { - return store.get(url) - } -} \ No newline at end of file diff --git a/app/src/main/java/eu/kanade/tachiyomi/network/PersistentCookieStore.kt b/app/src/main/java/eu/kanade/tachiyomi/network/PersistentCookieStore.kt deleted file mode 100644 index ca854bb72..000000000 --- a/app/src/main/java/eu/kanade/tachiyomi/network/PersistentCookieStore.kt +++ /dev/null @@ -1,78 +0,0 @@ -package eu.kanade.tachiyomi.network - -import android.content.Context -import okhttp3.Cookie -import okhttp3.HttpUrl -import java.net.URI -import java.util.concurrent.ConcurrentHashMap - -class PersistentCookieStore(context: Context) { - - private val cookieMap = ConcurrentHashMap>() - private val prefs = context.getSharedPreferences("cookie_store", Context.MODE_PRIVATE) - - init { - for ((key, value) in prefs.all) { - @Suppress("UNCHECKED_CAST") - val cookies = value as? Set - if (cookies != null) { - try { - val url = HttpUrl.parse("http://$key") ?: continue - val nonExpiredCookies = cookies.mapNotNull { Cookie.parse(url, it) } - .filter { !it.hasExpired() } - cookieMap.put(key, nonExpiredCookies) - } catch (e: Exception) { - // Ignore - } - } - } - } - - @Synchronized - fun addAll(url: HttpUrl, cookies: List) { - val key = url.uri().host - - // Append or replace the cookies for this domain. - val cookiesForDomain = cookieMap[key].orEmpty().toMutableList() - for (cookie in cookies) { - // Find a cookie with the same name. Replace it if found, otherwise add a new one. - val pos = cookiesForDomain.indexOfFirst { it.name() == cookie.name() } - if (pos == -1) { - cookiesForDomain.add(cookie) - } else { - cookiesForDomain[pos] = cookie - } - } - cookieMap.put(key, cookiesForDomain) - - // Get cookies to be stored in disk - val newValues = cookiesForDomain.asSequence() - .filter { it.persistent() && !it.hasExpired() } - .map(Cookie::toString) - .toSet() - - prefs.edit().putStringSet(key, newValues).apply() - } - - @Synchronized - fun removeAll() { - prefs.edit().clear().apply() - cookieMap.clear() - } - - fun remove(uri: URI) { - prefs.edit().remove(uri.host).apply() - cookieMap.remove(uri.host) - } - - fun get(url: HttpUrl) = get(url.uri().host) - - fun get(uri: URI) = get(uri.host) - - private fun get(url: String): List { - return cookieMap[url].orEmpty().filter { !it.hasExpired() } - } - - private fun Cookie.hasExpired() = System.currentTimeMillis() >= expiresAt() - -} \ No newline at end of file diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/setting/SettingsAdvancedController.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/setting/SettingsAdvancedController.kt index 07f69edde..3ad78247a 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/setting/SettingsAdvancedController.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/setting/SettingsAdvancedController.kt @@ -43,7 +43,7 @@ class SettingsAdvancedController : SettingsController() { titleRes = R.string.pref_clear_cookies onClick { - network.cookies.removeAll() + network.cookieManager.removeAll() activity?.toast(R.string.cookies_cleared) } } diff --git a/app/src/main/java/eu/kanade/tachiyomi/util/WebViewClientCompat.kt b/app/src/main/java/eu/kanade/tachiyomi/util/WebViewClientCompat.kt new file mode 100644 index 000000000..977dca5e6 --- /dev/null +++ b/app/src/main/java/eu/kanade/tachiyomi/util/WebViewClientCompat.kt @@ -0,0 +1,83 @@ +package eu.kanade.tachiyomi.util + +import android.annotation.TargetApi +import android.os.Build +import android.webkit.* + +@Suppress("OverridingDeprecatedMember") +abstract class WebViewClientCompat : WebViewClient() { + + open fun shouldOverrideUrlCompat(view: WebView, url: String): Boolean { + return false + } + + open fun shouldInterceptRequestCompat(view: WebView, url: String): WebResourceResponse? { + return null + } + + open fun onReceivedErrorCompat( + view: WebView, + errorCode: Int, + description: String?, + failingUrl: String, + isMainFrame: Boolean) { + + } + + @TargetApi(Build.VERSION_CODES.N) + final override fun shouldOverrideUrlLoading( + view: WebView, + request: WebResourceRequest + ): Boolean { + return shouldOverrideUrlCompat(view, request.url.toString()) + } + + final override fun shouldOverrideUrlLoading(view: WebView, url: String): Boolean { + return shouldOverrideUrlCompat(view, url) + } + + @TargetApi(Build.VERSION_CODES.LOLLIPOP) + final override fun shouldInterceptRequest( + view: WebView, + request: WebResourceRequest + ): WebResourceResponse? { + return shouldInterceptRequestCompat(view, request.url.toString()) + } + + final override fun shouldInterceptRequest( + view: WebView, + url: String + ): WebResourceResponse? { + return shouldInterceptRequestCompat(view, url) + } + + @TargetApi(Build.VERSION_CODES.M) + final override fun onReceivedError( + view: WebView, + request: WebResourceRequest, + error: WebResourceError + ) { + onReceivedErrorCompat(view, error.errorCode, error.description?.toString(), + request.url.toString(), request.isForMainFrame) + } + + final override fun onReceivedError( + view: WebView, + errorCode: Int, + description: String?, + failingUrl: String + ) { + onReceivedErrorCompat(view, errorCode, description, failingUrl, failingUrl == view.url) + } + + @TargetApi(Build.VERSION_CODES.M) + final override fun onReceivedHttpError( + view: WebView, + request: WebResourceRequest, + error: WebResourceResponse + ) { + onReceivedErrorCompat(view, error.statusCode, error.reasonPhrase, request.url + .toString(), request.isForMainFrame) + } + +}