From c3645048c86b91d4788c78db4a0dd5c9627e6449 Mon Sep 17 00:00:00 2001 From: Samfun75 <38332931+Samfun75@users.noreply.github.com> Date: Wed, 1 Feb 2023 11:05:43 +0300 Subject: [PATCH] ScrapingWars: Use one vrf webview for everything and fix extraction (#1227) * ScrapingWars: Use one vrf webview for everything and fix embed extraction * Update some options --- src/en/nineanime/build.gradle | 2 +- .../en/nineanime/JsInterceptor.kt | 52 +++---- .../en/nineanime/JsVizInterceptor.kt | 140 ------------------ .../en/nineanime/JsVrfInterceptor.kt | 101 ++++++------- .../animeextension/en/nineanime/NineAnime.kt | 39 +++-- 5 files changed, 87 insertions(+), 247 deletions(-) delete mode 100644 src/en/nineanime/src/eu/kanade/tachiyomi/animeextension/en/nineanime/JsVizInterceptor.kt diff --git a/src/en/nineanime/build.gradle b/src/en/nineanime/build.gradle index 7e81a1b7c..911bd02ca 100644 --- a/src/en/nineanime/build.gradle +++ b/src/en/nineanime/build.gradle @@ -5,7 +5,7 @@ ext { extName = '9anime' pkgNameSuffix = 'en.nineanime' extClass = '.NineAnime' - extVersionCode = 27 + extVersionCode = 28 libVersion = '13' } diff --git a/src/en/nineanime/src/eu/kanade/tachiyomi/animeextension/en/nineanime/JsInterceptor.kt b/src/en/nineanime/src/eu/kanade/tachiyomi/animeextension/en/nineanime/JsInterceptor.kt index 8732d37b2..297d24f3b 100644 --- a/src/en/nineanime/src/eu/kanade/tachiyomi/animeextension/en/nineanime/JsInterceptor.kt +++ b/src/en/nineanime/src/eu/kanade/tachiyomi/animeextension/en/nineanime/JsInterceptor.kt @@ -9,6 +9,7 @@ import android.webkit.WebResourceRequest import android.webkit.WebResourceResponse import android.webkit.WebView import android.webkit.WebViewClient +import android.widget.Toast import eu.kanade.tachiyomi.network.GET import okhttp3.Headers import okhttp3.Interceptor @@ -24,18 +25,19 @@ class JsInterceptor(private val lang: String) : Interceptor { private val context = Injekt.get() private val handler by lazy { Handler(Looper.getMainLooper()) } - class JsObject(private val latch: CountDownLatch, var payload: String = "") { + class JsObject(var payload: String = "") { @JavascriptInterface fun passPayload(passedPayload: String) { payload = passedPayload - latch.countDown() } } override fun intercept(chain: Interceptor.Chain): Response { val originalRequest = chain.request() - - val newRequest = resolveWithWebView(originalRequest) ?: throw Exception("Please reload Episode List") + handler.post { + context.let { Toast.makeText(it, "This might take a while, Don't close me", Toast.LENGTH_LONG).show() } + } + val newRequest = resolveWithWebView(originalRequest) ?: throw Exception("Someting went wrong or took too long") return chain.proceed(newRequest) } @@ -49,20 +51,22 @@ class JsInterceptor(private val lang: String) : Interceptor { val origRequestUrl = request.url.toString() - val jsinterface = JsObject(latch) + val jsinterface = JsObject() // JavaSrcipt gets the Dub or Sub link of vidstream val jsScript = """ (function(){ - let hoster = document.querySelector('div[data-type="$lang"] ul li[data-sv-id="41"]'); - let event = document.createEvent('HTMLEvents'); - event.initEvent('click',true,true); - hoster.dispatchEvent(event); + let jqclk = jQuery.Event('click'); + jqclk.isTrusted = true; + jqclk.originalEvent = { + isTrusted: true + }; + ${'$'}('div[data-type="$lang"] ul li[data-sv-id="41"]').trigger(jqclk); let intervalId = setInterval(() => { let element = document.querySelector("#player iframe"); if (element) { clearInterval(intervalId); - window.android.passPayload(element.src) + window.android.passPayload(element.src); } }, 500); })(); @@ -72,8 +76,6 @@ class JsInterceptor(private val lang: String) : Interceptor { var newRequest: Request? = null - var head = "" - handler.post { val webview = WebView(context) webView = webview @@ -87,8 +89,17 @@ class JsInterceptor(private val lang: String) : Interceptor { webview.addJavascriptInterface(jsinterface, "android") webview.webViewClient = object : WebViewClient() { override fun shouldInterceptRequest(view: WebView?, request: WebResourceRequest?): WebResourceResponse? { - if (request?.url.toString().contains("https://vidstream.pro/embed")) { - head = request?.url.toString() + if (!request?.url.toString().contains("vidstream") && + !request?.url.toString().contains("vizcloud") + ) return null + + if (request?.url.toString().contains("/simple/")) { + newRequest = GET( + request?.url.toString(), + Headers.headersOf("referer", "/orp.maertsdiv//:sptth".reversed()) + ) + latch.countDown() + return null } return super.shouldInterceptRequest(view, request) } @@ -100,24 +111,13 @@ class JsInterceptor(private val lang: String) : Interceptor { } } - latch.await(12, TimeUnit.SECONDS) + latch.await(30, TimeUnit.SECONDS) handler.post { webView?.stopLoading() webView?.destroy() webView = null } - newRequest = GET( - request.url.toString(), - headers = Headers.headersOf( - "url", - if (jsinterface.payload.isNullOrEmpty() || (!jsinterface.payload.contains("https://vidstream.pro/embed"))) { - head - } else { - jsinterface.payload - } - ) - ) return newRequest } } diff --git a/src/en/nineanime/src/eu/kanade/tachiyomi/animeextension/en/nineanime/JsVizInterceptor.kt b/src/en/nineanime/src/eu/kanade/tachiyomi/animeextension/en/nineanime/JsVizInterceptor.kt deleted file mode 100644 index e220b75c6..000000000 --- a/src/en/nineanime/src/eu/kanade/tachiyomi/animeextension/en/nineanime/JsVizInterceptor.kt +++ /dev/null @@ -1,140 +0,0 @@ -package eu.kanade.tachiyomi.animeextension.en.nineanime - -import android.annotation.SuppressLint -import android.app.Application -import android.os.Handler -import android.os.Looper -import android.webkit.JavascriptInterface -import android.webkit.WebResourceRequest -import android.webkit.WebResourceResponse -import android.webkit.WebView -import android.webkit.WebViewClient -import eu.kanade.tachiyomi.network.GET -import okhttp3.Headers -import okhttp3.Interceptor -import okhttp3.Request -import okhttp3.Response -import uy.kohesive.injekt.Injekt -import uy.kohesive.injekt.api.get -import java.util.concurrent.CountDownLatch -import java.util.concurrent.TimeUnit - -class JsVizInterceptor(private val embedLink: String) : Interceptor { - - private val context = Injekt.get() - private val handler by lazy { Handler(Looper.getMainLooper()) } - - class JsObject(private val latch: CountDownLatch, var payload: String = "") { - @JavascriptInterface - fun passPayload(passedPayload: String) { - payload = passedPayload - latch.countDown() - } - } - - override fun intercept(chain: Interceptor.Chain): Response { - val originalRequest = chain.request() - - val newRequest = resolveWithWebView(originalRequest) ?: throw Exception("Please reload Episode List") - - return chain.proceed(newRequest) - } - - @SuppressLint("SetJavaScriptEnabled") - private fun resolveWithWebView(request: Request): Request? { - - val latch = CountDownLatch(1) - - var webView: WebView? = null - - val origRequestUrl = request.url.toString() - - val jsinterface = JsObject(latch) - - // JavaSrcipt creates Iframe on vidstream page to bypass iframe-cors and gets the sourceUrl - val jsScript = """ - (function(){ - const html = ''; - document.body.innerHTML += html; - const iframe = document.querySelector('iframe'); - - const originalOpen = iframe.contentWindow.XMLHttpRequest.prototype.open; - iframe.contentWindow.XMLHttpRequest.prototype.open = function(method, url, async) { - if (!url.includes("ping") && !url.includes("/assets/") && !url.includes("thumbnails") && !url.includes("jpg") && !url.includes("m3u8") && !url.includes("simplewebanalysis")) { - if (url == null) { - const entries = iframe.contentWindow.performance.getEntries(); - entries.forEach((entry) => { - if (entry.initiatorType.includes("xmlhttprequest")) { - if (!entry.name.includes("/ping/") && !entry.name.includes("/assets/") && !entry.name.includes("thumbnails")) { - window.android.passPayload(entry.name); - } - } - }); - } else { - window.android.passPayload("https://" + document.domain + "/" + url); - } - } - originalOpen.apply(this, arguments); - } - })(); - """ - - val headers = request.headers.toMultimap().mapValues { it.value.getOrNull(0) ?: "" }.toMutableMap() - - var newRequest: Request? = null - - var head = "" - - handler.post { - val webview = WebView(context) - webView = webview - with(webview.settings) { - javaScriptEnabled = true - domStorageEnabled = true - databaseEnabled = true - useWideViewPort = false - loadWithOverviewMode = false - userAgentString = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:106.0) Gecko/20100101 Firefox/106.0" - webview.addJavascriptInterface(jsinterface, "android") - webview.webViewClient = object : WebViewClient() { - override fun shouldInterceptRequest(view: WebView?, request: WebResourceRequest?): WebResourceResponse? { - if (request?.url.toString().contains("https://vidstream.pro/")) { - if (request?.url.toString().contains("/embed/") || request?.url.toString().contains("/ping/") || request?.url.toString().contains("favicon.ico") || - request?.url.toString().contains("/assets/") || request?.url.toString().contains("/players/") - ) { - return null - } else { - head = request?.url.toString() - } - } - return super.shouldInterceptRequest(view, request) - } - override fun onPageFinished(view: WebView?, url: String?) { - view?.evaluateJavascript(jsScript) {} - } - } - webView?.loadUrl(origRequestUrl, headers) - } - } - - latch.await(12, TimeUnit.SECONDS) - - handler.post { - webView?.stopLoading() - webView?.destroy() - webView = null - } - newRequest = GET( - request.url.toString(), - headers = Headers.headersOf( - "url", - if (jsinterface.payload.isNullOrEmpty() || (!jsinterface.payload.contains("https://vidstream.pro"))) { - head - } else { - jsinterface.payload - } - ) - ) - return newRequest - } -} diff --git a/src/en/nineanime/src/eu/kanade/tachiyomi/animeextension/en/nineanime/JsVrfInterceptor.kt b/src/en/nineanime/src/eu/kanade/tachiyomi/animeextension/en/nineanime/JsVrfInterceptor.kt index 96ef4981a..3052c2bb6 100644 --- a/src/en/nineanime/src/eu/kanade/tachiyomi/animeextension/en/nineanime/JsVrfInterceptor.kt +++ b/src/en/nineanime/src/eu/kanade/tachiyomi/animeextension/en/nineanime/JsVrfInterceptor.kt @@ -4,74 +4,42 @@ import android.annotation.SuppressLint import android.app.Application import android.os.Handler import android.os.Looper -import android.webkit.JavascriptInterface import android.webkit.WebResourceRequest import android.webkit.WebView import android.webkit.WebViewClient -import eu.kanade.tachiyomi.network.GET -import okhttp3.Headers -import okhttp3.Interceptor -import okhttp3.Request -import okhttp3.Response import uy.kohesive.injekt.Injekt import uy.kohesive.injekt.api.get import java.util.concurrent.CountDownLatch import java.util.concurrent.TimeUnit -class JsVrfInterceptor(private val query: String, private val baseUrl: String) : Interceptor { +class JsVrfInterceptor(private val baseUrl: String) { private val context = Injekt.get() private val handler by lazy { Handler(Looper.getMainLooper()) } + private val vrfWebView = createWebView() - class JsObject(private val latch: CountDownLatch, var payload: String = "") { - @JavascriptInterface - fun passPayload(passedPayload: String) { - payload = passedPayload - latch.countDown() + fun wake() = "" + + fun getVrf(query: String): String { + val jscript = getJs(query) + val cdl = CountDownLatch(1) + var vrf = "" + handler.post { + vrfWebView?.evaluateJavascript(jscript) { + vrf = it?.removeSurrounding("\"") ?: "" + cdl.countDown() + } } - } - - override fun intercept(chain: Interceptor.Chain): Response { - val originalRequest = chain.request() - - val newRequest = resolveWithWebView(originalRequest) ?: throw Exception("Please reload Episode List") - - return chain.proceed(newRequest) + cdl.await(12, TimeUnit.SECONDS) + if (vrf.isBlank()) throw Exception("vrf could not be retrieved") + return vrf } @SuppressLint("SetJavaScriptEnabled") - private fun resolveWithWebView(request: Request): Request? { - + private fun createWebView(): WebView? { val latch = CountDownLatch(1) - var webView: WebView? = null - val origRequestUrl = request.url.toString() - - val jsinterface = JsObject(latch) - - // JavaScript uses search of 9Anime to convert IDs & Querys to the VRF-Key - val jsScript = """ - (function() { - document.querySelector("form.filters input.form-control").value = '$query'; - let inputElemente = document.querySelector('form.filters input.form-control'); - let e = document.createEvent('HTMLEvents'); - e.initEvent('keyup', true, true); - inputElemente.dispatchEvent(e); - let intervalId = setInterval(() => { - let element = document.querySelector('form.filters input[type="hidden"]').value; - if (element) { - clearInterval(intervalId); - window.android.passPayload(element) - } - }, 100); - })(); - """ - - val headers = request.headers.toMultimap().mapValues { it.value.getOrNull(0) ?: "" }.toMutableMap() - - var newRequest: Request? = null - handler.post { val webview = WebView(context) webView = webview @@ -83,8 +51,6 @@ class JsVrfInterceptor(private val query: String, private val baseUrl: String) : loadWithOverviewMode = false userAgentString = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:106.0) Gecko/20100101 Firefox/106.0" - webview.addJavascriptInterface(jsinterface, "android") - webview.webViewClient = object : WebViewClient() { override fun shouldOverrideUrlLoading(view: WebView?, request: WebResourceRequest?): Boolean { if (request?.url.toString().contains("$baseUrl/filter")) { @@ -95,21 +61,40 @@ class JsVrfInterceptor(private val query: String, private val baseUrl: String) : } } override fun onPageFinished(view: WebView?, url: String?) { - view?.evaluateJavascript(jsScript) {} + latch.countDown() } } - webView?.loadUrl(origRequestUrl, headers) + webView?.loadUrl("$baseUrl/filter") } } - latch.await(12, TimeUnit.SECONDS) + latch.await() handler.post { webView?.stopLoading() - webView?.destroy() - webView = null } - newRequest = GET(request.url.toString(), headers = Headers.headersOf("url", jsinterface.payload, "user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:106.0) Gecko/20100101 Firefox/106.0")) - return newRequest + return webView + } + + private fun getJs(query: String): String { + return """ + (function() { + document.querySelector("form.filters input.form-control").value = '$query'; + let inputElemente = document.querySelector('form.filters input.form-control'); + let e = document.createEvent('HTMLEvents'); + e.initEvent('keyup', true, true); + inputElemente.dispatchEvent(e); + let val = ""; + while (val == "") { + let element = document.querySelector('form.filters input[type="hidden"]').value; + if (element) { + val = element; + break; + } + } + document.querySelector("form.filters input.form-control").value = ''; + return val; + })(); + """.trimIndent() } } diff --git a/src/en/nineanime/src/eu/kanade/tachiyomi/animeextension/en/nineanime/NineAnime.kt b/src/en/nineanime/src/eu/kanade/tachiyomi/animeextension/en/nineanime/NineAnime.kt index 06a759e1f..94153651f 100644 --- a/src/en/nineanime/src/eu/kanade/tachiyomi/animeextension/en/nineanime/NineAnime.kt +++ b/src/en/nineanime/src/eu/kanade/tachiyomi/animeextension/en/nineanime/NineAnime.kt @@ -21,8 +21,6 @@ import kotlinx.coroutines.runBlocking import kotlinx.serialization.decodeFromString import kotlinx.serialization.json.Json import kotlinx.serialization.json.JsonObject -import kotlinx.serialization.json.jsonArray -import kotlinx.serialization.json.jsonObject import kotlinx.serialization.json.jsonPrimitive import okhttp3.Headers import okhttp3.OkHttpClient @@ -50,6 +48,8 @@ class NineAnime : ConfigurableAnimeSource, ParsedAnimeHttpSource() { private val json: Json by injectLazy() + private val vrfInterceptor by lazy { JsVrfInterceptor(baseUrl) } + private val preferences: SharedPreferences by lazy { Injekt.get().getSharedPreferences("source_$id", 0x0000) } @@ -60,7 +60,11 @@ class NineAnime : ConfigurableAnimeSource, ParsedAnimeHttpSource() { override fun popularAnimeSelector(): String = "div.ani.items > div" - override fun popularAnimeRequest(page: Int): Request = GET("$baseUrl/filter?sort=trending&page=$page") + override fun popularAnimeRequest(page: Int): Request { + // make the vrf webview available beforehand. please find another solution for this :) + vrfInterceptor.wake() + return GET("$baseUrl/filter?sort=trending&page=$page") + } override fun popularAnimeFromElement(element: Element) = SAnime.create().apply { setUrlWithoutDomain(element.select("a.name").attr("href").substringBefore("?")) @@ -72,8 +76,7 @@ class NineAnime : ConfigurableAnimeSource, ParsedAnimeHttpSource() { override fun episodeListRequest(anime: SAnime): Request { val id = client.newCall(GET(baseUrl + anime.url)).execute().asJsoup().selectFirst("div[data-id]").attr("data-id") - val jsVrfInterceptor = client.newBuilder().addInterceptor(JsVrfInterceptor(id, baseUrl)).build() - val vrf = jsVrfInterceptor.newCall(GET("$baseUrl/filter")).execute().request.header("url").toString() + val vrf = vrfInterceptor.getVrf(id) return GET("$baseUrl/ajax/episode/list/$id?vrf=${java.net.URLEncoder.encode(vrf, "utf-8")}", headers = Headers.headersOf("url", anime.url)) } @@ -129,8 +132,7 @@ class NineAnime : ConfigurableAnimeSource, ParsedAnimeHttpSource() { override fun videoListRequest(episode: SEpisode): Request { val ids = episode.url.substringAfter("list/").substringBefore("?vrf") - val jsVrfInterceptor = client.newBuilder().addInterceptor(JsVrfInterceptor(ids, baseUrl)).build() - val vrf = jsVrfInterceptor.newCall(GET("$baseUrl/filter")).execute().request.header("url").toString() + val vrf = vrfInterceptor.getVrf(ids) val url = "/ajax/server/list/$ids?vrf=${java.net.URLEncoder.encode(vrf, "utf-8")}" val epurl = episode.url.substringAfter("epurl=") return GET(baseUrl + url, headers = Headers.headersOf("url", epurl)) @@ -155,18 +157,8 @@ class NineAnime : ConfigurableAnimeSource, ParsedAnimeHttpSource() { private fun extractVideo(lang: String, epurl: String): List