ScrapingWars: Use one vrf webview for everything and fix extraction (#1227)

* ScrapingWars: Use one vrf webview for everything and fix embed extraction

* Update some options

src/en/nineanime/build.gradle

@@ -5,7 +5,7 @@ ext {
     extName = '9anime'
     pkgNameSuffix = 'en.nineanime'
     extClass = '.NineAnime'
-    extVersionCode = 27
+    extVersionCode = 28
     libVersion = '13'
 }
 

src/en/nineanime/src/eu/kanade/tachiyomi/animeextension/en/nineanime/JsInterceptor.kt

@@ -9,6 +9,7 @@ import android.webkit.WebResourceRequest
 import android.webkit.WebResourceResponse
 import android.webkit.WebView
 import android.webkit.WebViewClient
+import android.widget.Toast
 import eu.kanade.tachiyomi.network.GET
 import okhttp3.Headers
 import okhttp3.Interceptor
@@ -24,18 +25,19 @@ class JsInterceptor(private val lang: String) : Interceptor {
     private val context = Injekt.get<Application>()
     private val handler by lazy { Handler(Looper.getMainLooper()) }
 
-    class JsObject(private val latch: CountDownLatch, var payload: String = "") {
+    class JsObject(var payload: String = "") {
         @JavascriptInterface
         fun passPayload(passedPayload: String) {
             payload = passedPayload
-            latch.countDown()
         }
     }
 
     override fun intercept(chain: Interceptor.Chain): Response {
         val originalRequest = chain.request()
-
+        handler.post {
-        val newRequest = resolveWithWebView(originalRequest) ?: throw Exception("Please reload Episode List")
+            context.let { Toast.makeText(it, "This might take a while, Don't close me", Toast.LENGTH_LONG).show() }
+        }
+        val newRequest = resolveWithWebView(originalRequest) ?: throw Exception("Someting went wrong or took too long")
 
         return chain.proceed(newRequest)
     }
@@ -49,20 +51,22 @@ class JsInterceptor(private val lang: String) : Interceptor {
 
         val origRequestUrl = request.url.toString()
 
-        val jsinterface = JsObject(latch)
+        val jsinterface = JsObject()
 
         // JavaSrcipt gets the Dub or Sub link of vidstream
         val jsScript = """
             (function(){
-                let hoster = document.querySelector('div[data-type="$lang"] ul li[data-sv-id="41"]');
+                let jqclk = jQuery.Event('click');
-                let event = document.createEvent('HTMLEvents');
+                jqclk.isTrusted = true;
-                event.initEvent('click',true,true);
+                jqclk.originalEvent = {
-                hoster.dispatchEvent(event);
+                  isTrusted: true
+                };
+                ${'$'}('div[data-type="$lang"] ul li[data-sv-id="41"]').trigger(jqclk);
                 let intervalId = setInterval(() => {
                     let element = document.querySelector("#player iframe");
                     if (element) {
                         clearInterval(intervalId);
-                        window.android.passPayload(element.src)
+                        window.android.passPayload(element.src);
                     }
                 }, 500);
             })();
@@ -72,8 +76,6 @@ class JsInterceptor(private val lang: String) : Interceptor {
 
         var newRequest: Request? = null
 
-        var head = ""
-
         handler.post {
             val webview = WebView(context)
             webView = webview
@@ -87,8 +89,17 @@ class JsInterceptor(private val lang: String) : Interceptor {
                 webview.addJavascriptInterface(jsinterface, "android")
                 webview.webViewClient = object : WebViewClient() {
                     override fun shouldInterceptRequest(view: WebView?, request: WebResourceRequest?): WebResourceResponse? {
-                        if (request?.url.toString().contains("https://vidstream.pro/embed")) {
+                        if (!request?.url.toString().contains("vidstream") &&
-                            head = request?.url.toString()
+                            !request?.url.toString().contains("vizcloud")
+                        ) return null
+
+                        if (request?.url.toString().contains("/simple/")) {
+                            newRequest = GET(
+                                request?.url.toString(),
+                                Headers.headersOf("referer", "/orp.maertsdiv//:sptth".reversed())
+                            )
+                            latch.countDown()
+                            return null
                         }
                         return super.shouldInterceptRequest(view, request)
                     }
@@ -100,24 +111,13 @@ class JsInterceptor(private val lang: String) : Interceptor {
             }
         }
 
-        latch.await(12, TimeUnit.SECONDS)
+        latch.await(30, TimeUnit.SECONDS)
 
         handler.post {
             webView?.stopLoading()
             webView?.destroy()
             webView = null
         }
-        newRequest = GET(
-            request.url.toString(),
-            headers = Headers.headersOf(
-                "url",
-                if (jsinterface.payload.isNullOrEmpty() || (!jsinterface.payload.contains("https://vidstream.pro/embed"))) {
-                    head
-                } else {
-                    jsinterface.payload
-                }
-            )
-        )
         return newRequest
     }
 }

src/en/nineanime/src/eu/kanade/tachiyomi/animeextension/en/nineanime/JsVizInterceptor.kt

@@ -1,140 +0,0 @@
-package eu.kanade.tachiyomi.animeextension.en.nineanime
-
-import android.annotation.SuppressLint
-import android.app.Application
-import android.os.Handler
-import android.os.Looper
-import android.webkit.JavascriptInterface
-import android.webkit.WebResourceRequest
-import android.webkit.WebResourceResponse
-import android.webkit.WebView
-import android.webkit.WebViewClient
-import eu.kanade.tachiyomi.network.GET
-import okhttp3.Headers
-import okhttp3.Interceptor
-import okhttp3.Request
-import okhttp3.Response
-import uy.kohesive.injekt.Injekt
-import uy.kohesive.injekt.api.get
-import java.util.concurrent.CountDownLatch
-import java.util.concurrent.TimeUnit
-
-class JsVizInterceptor(private val embedLink: String) : Interceptor {
-
-    private val context = Injekt.get<Application>()
-    private val handler by lazy { Handler(Looper.getMainLooper()) }
-
-    class JsObject(private val latch: CountDownLatch, var payload: String = "") {
-        @JavascriptInterface
-        fun passPayload(passedPayload: String) {
-            payload = passedPayload
-            latch.countDown()
-        }
-    }
-
-    override fun intercept(chain: Interceptor.Chain): Response {
-        val originalRequest = chain.request()
-
-        val newRequest = resolveWithWebView(originalRequest) ?: throw Exception("Please reload Episode List")
-
-        return chain.proceed(newRequest)
-    }
-
-    @SuppressLint("SetJavaScriptEnabled")
-    private fun resolveWithWebView(request: Request): Request? {
-
-        val latch = CountDownLatch(1)
-
-        var webView: WebView? = null
-
-        val origRequestUrl = request.url.toString()
-
-        val jsinterface = JsObject(latch)
-
-        // JavaSrcipt creates Iframe on vidstream page to bypass iframe-cors and gets the sourceUrl
-        val jsScript = """
-            (function(){
-                const html = '<iframe src="$embedLink" allow="autoplay; fullscreen" allowfullscreen="yes" scrolling="no" style="width: 100%; height: 100%; overflow: hidden;" frameborder="no" onload="handleIframeLoad()"></iframe>';
-                document.body.innerHTML += html;
-                const iframe = document.querySelector('iframe');
-
-                const originalOpen = iframe.contentWindow.XMLHttpRequest.prototype.open;
-                iframe.contentWindow.XMLHttpRequest.prototype.open = function(method, url, async) {
-                    if (!url.includes("ping") && !url.includes("/assets/") && !url.includes("thumbnails") && !url.includes("jpg") && !url.includes("m3u8") && !url.includes("simplewebanalysis")) {
-                        if (url == null) {
-                            const entries = iframe.contentWindow.performance.getEntries();
-                            entries.forEach((entry) => {
-                                if (entry.initiatorType.includes("xmlhttprequest")) {
-                                    if (!entry.name.includes("/ping/") && !entry.name.includes("/assets/") && !entry.name.includes("thumbnails")) {
-                                        window.android.passPayload(entry.name);
-                                    }
-                                }
-                            });
-                        } else {
-                            window.android.passPayload("https://" + document.domain + "/" + url);
-                        }
-                    }
-                    originalOpen.apply(this, arguments);
-                }
-            })();
-        """
-
-        val headers = request.headers.toMultimap().mapValues { it.value.getOrNull(0) ?: "" }.toMutableMap()
-
-        var newRequest: Request? = null
-
-        var head = ""
-
-        handler.post {
-            val webview = WebView(context)
-            webView = webview
-            with(webview.settings) {
-                javaScriptEnabled = true
-                domStorageEnabled = true
-                databaseEnabled = true
-                useWideViewPort = false
-                loadWithOverviewMode = false
-                userAgentString = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:106.0) Gecko/20100101 Firefox/106.0"
-                webview.addJavascriptInterface(jsinterface, "android")
-                webview.webViewClient = object : WebViewClient() {
-                    override fun shouldInterceptRequest(view: WebView?, request: WebResourceRequest?): WebResourceResponse? {
-                        if (request?.url.toString().contains("https://vidstream.pro/")) {
-                            if (request?.url.toString().contains("/embed/") || request?.url.toString().contains("/ping/") || request?.url.toString().contains("favicon.ico") ||
-                                request?.url.toString().contains("/assets/") || request?.url.toString().contains("/players/")
-                            ) {
-                                return null
-                            } else {
-                                head = request?.url.toString()
-                            }
-                        }
-                        return super.shouldInterceptRequest(view, request)
-                    }
-                    override fun onPageFinished(view: WebView?, url: String?) {
-                        view?.evaluateJavascript(jsScript) {}
-                    }
-                }
-                webView?.loadUrl(origRequestUrl, headers)
-            }
-        }
-
-        latch.await(12, TimeUnit.SECONDS)
-
-        handler.post {
-            webView?.stopLoading()
-            webView?.destroy()
-            webView = null
-        }
-        newRequest = GET(
-            request.url.toString(),
-            headers = Headers.headersOf(
-                "url",
-                if (jsinterface.payload.isNullOrEmpty() || (!jsinterface.payload.contains("https://vidstream.pro"))) {
-                    head
-                } else {
-                    jsinterface.payload
-                }
-            )
-        )
-        return newRequest
-    }
-}

src/en/nineanime/src/eu/kanade/tachiyomi/animeextension/en/nineanime/JsVrfInterceptor.kt

@@ -4,74 +4,42 @@ import android.annotation.SuppressLint
 import android.app.Application
 import android.os.Handler
 import android.os.Looper
-import android.webkit.JavascriptInterface
 import android.webkit.WebResourceRequest
 import android.webkit.WebView
 import android.webkit.WebViewClient
-import eu.kanade.tachiyomi.network.GET
-import okhttp3.Headers
-import okhttp3.Interceptor
-import okhttp3.Request
-import okhttp3.Response
 import uy.kohesive.injekt.Injekt
 import uy.kohesive.injekt.api.get
 import java.util.concurrent.CountDownLatch
 import java.util.concurrent.TimeUnit
 
-class JsVrfInterceptor(private val query: String, private val baseUrl: String) : Interceptor {
+class JsVrfInterceptor(private val baseUrl: String) {
 
     private val context = Injekt.get<Application>()
     private val handler by lazy { Handler(Looper.getMainLooper()) }
+    private val vrfWebView = createWebView()
 
-    class JsObject(private val latch: CountDownLatch, var payload: String = "") {
+    fun wake() = ""
-        @JavascriptInterface
+
-        fun passPayload(passedPayload: String) {
+    fun getVrf(query: String): String {
-            payload = passedPayload
+        val jscript = getJs(query)
-            latch.countDown()
+        val cdl = CountDownLatch(1)
+        var vrf = ""
+        handler.post {
+            vrfWebView?.evaluateJavascript(jscript) {
+                vrf = it?.removeSurrounding("\"") ?: ""
+                cdl.countDown()
             }
         }
-
+        cdl.await(12, TimeUnit.SECONDS)
-    override fun intercept(chain: Interceptor.Chain): Response {
+        if (vrf.isBlank()) throw Exception("vrf could not be retrieved")
-        val originalRequest = chain.request()
+        return vrf
-
-        val newRequest = resolveWithWebView(originalRequest) ?: throw Exception("Please reload Episode List")
-
-        return chain.proceed(newRequest)
     }
 
     @SuppressLint("SetJavaScriptEnabled")
-    private fun resolveWithWebView(request: Request): Request? {
+    private fun createWebView(): WebView? {
-
         val latch = CountDownLatch(1)
-
         var webView: WebView? = null
 
-        val origRequestUrl = request.url.toString()
-
-        val jsinterface = JsObject(latch)
-
-        // JavaScript uses search of 9Anime to convert IDs & Querys to the VRF-Key
-        val jsScript = """
-            (function() {
-               document.querySelector("form.filters input.form-control").value = '$query';
-               let inputElemente = document.querySelector('form.filters input.form-control');
-               let e = document.createEvent('HTMLEvents');
-               e.initEvent('keyup', true, true);
-               inputElemente.dispatchEvent(e);
-               let intervalId = setInterval(() => {
-                    let element = document.querySelector('form.filters input[type="hidden"]').value;
-                    if (element) {
-                        clearInterval(intervalId);
-                        window.android.passPayload(element)
-                    }
-                }, 100);
-            })();
-        """
-
-        val headers = request.headers.toMultimap().mapValues { it.value.getOrNull(0) ?: "" }.toMutableMap()
-
-        var newRequest: Request? = null
-
         handler.post {
             val webview = WebView(context)
             webView = webview
@@ -83,8 +51,6 @@ class JsVrfInterceptor(private val query: String, private val baseUrl: String) :
                 loadWithOverviewMode = false
                 userAgentString = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:106.0) Gecko/20100101 Firefox/106.0"
 
-                webview.addJavascriptInterface(jsinterface, "android")
-
                 webview.webViewClient = object : WebViewClient() {
                     override fun shouldOverrideUrlLoading(view: WebView?, request: WebResourceRequest?): Boolean {
                         if (request?.url.toString().contains("$baseUrl/filter")) {
@@ -95,21 +61,40 @@ class JsVrfInterceptor(private val query: String, private val baseUrl: String) :
                         }
                     }
                     override fun onPageFinished(view: WebView?, url: String?) {
-                        view?.evaluateJavascript(jsScript) {}
+                        latch.countDown()
                     }
                 }
-                webView?.loadUrl(origRequestUrl, headers)
+                webView?.loadUrl("$baseUrl/filter")
             }
         }
 
-        latch.await(12, TimeUnit.SECONDS)
+        latch.await()
 
         handler.post {
             webView?.stopLoading()
-            webView?.destroy()
-            webView = null
         }
-        newRequest = GET(request.url.toString(), headers = Headers.headersOf("url", jsinterface.payload, "user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:106.0) Gecko/20100101 Firefox/106.0"))
+        return webView
-        return newRequest
+    }
+
+    private fun getJs(query: String): String {
+        return """
+            (function() {
+                  document.querySelector("form.filters input.form-control").value = '$query';
+                  let inputElemente = document.querySelector('form.filters input.form-control');
+                  let e = document.createEvent('HTMLEvents');
+                  e.initEvent('keyup', true, true);
+                  inputElemente.dispatchEvent(e);
+                  let val = "";
+                  while (val == "") {
+                    let element = document.querySelector('form.filters input[type="hidden"]').value;
+                    if (element) {
+                      val = element;
+                      break;
+                    }
+                  }
+                  document.querySelector("form.filters input.form-control").value = '';
+                  return val;
+            })();
+        """.trimIndent()
     }
 }

src/en/nineanime/src/eu/kanade/tachiyomi/animeextension/en/nineanime/NineAnime.kt

@@ -21,8 +21,6 @@ import kotlinx.coroutines.runBlocking
 import kotlinx.serialization.decodeFromString
 import kotlinx.serialization.json.Json
 import kotlinx.serialization.json.JsonObject
-import kotlinx.serialization.json.jsonArray
-import kotlinx.serialization.json.jsonObject
 import kotlinx.serialization.json.jsonPrimitive
 import okhttp3.Headers
 import okhttp3.OkHttpClient
@@ -50,6 +48,8 @@ class NineAnime : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
 
     private val json: Json by injectLazy()
 
+    private val vrfInterceptor by lazy { JsVrfInterceptor(baseUrl) }
+
     private val preferences: SharedPreferences by lazy {
         Injekt.get<Application>().getSharedPreferences("source_$id", 0x0000)
     }
@@ -60,7 +60,11 @@ class NineAnime : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
 
     override fun popularAnimeSelector(): String = "div.ani.items > div"
 
-    override fun popularAnimeRequest(page: Int): Request = GET("$baseUrl/filter?sort=trending&page=$page")
+    override fun popularAnimeRequest(page: Int): Request {
+        // make the vrf webview available beforehand. please find another solution for this :)
+        vrfInterceptor.wake()
+        return GET("$baseUrl/filter?sort=trending&page=$page")
+    }
 
     override fun popularAnimeFromElement(element: Element) = SAnime.create().apply {
         setUrlWithoutDomain(element.select("a.name").attr("href").substringBefore("?"))
@@ -72,8 +76,7 @@ class NineAnime : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
 
     override fun episodeListRequest(anime: SAnime): Request {
         val id = client.newCall(GET(baseUrl + anime.url)).execute().asJsoup().selectFirst("div[data-id]").attr("data-id")
-        val jsVrfInterceptor = client.newBuilder().addInterceptor(JsVrfInterceptor(id, baseUrl)).build()
+        val vrf = vrfInterceptor.getVrf(id)
-        val vrf = jsVrfInterceptor.newCall(GET("$baseUrl/filter")).execute().request.header("url").toString()
         return GET("$baseUrl/ajax/episode/list/$id?vrf=${java.net.URLEncoder.encode(vrf, "utf-8")}", headers = Headers.headersOf("url", anime.url))
     }
 
@@ -129,8 +132,7 @@ class NineAnime : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
 
     override fun videoListRequest(episode: SEpisode): Request {
         val ids = episode.url.substringAfter("list/").substringBefore("?vrf")
-        val jsVrfInterceptor = client.newBuilder().addInterceptor(JsVrfInterceptor(ids, baseUrl)).build()
+        val vrf = vrfInterceptor.getVrf(ids)
-        val vrf = jsVrfInterceptor.newCall(GET("$baseUrl/filter")).execute().request.header("url").toString()
         val url = "/ajax/server/list/$ids?vrf=${java.net.URLEncoder.encode(vrf, "utf-8")}"
         val epurl = episode.url.substringAfter("epurl=")
         return GET(baseUrl + url, headers = Headers.headersOf("url", epurl))
@@ -155,18 +157,8 @@ class NineAnime : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
 
     private fun extractVideo(lang: String, epurl: String): List<Video> {
         val jsInterceptor = client.newBuilder().addInterceptor(JsInterceptor(lang.lowercase())).build()
-        val embedLink = jsInterceptor.newCall(GET("$baseUrl$epurl")).execute().request.header("url").toString()
+        val result = jsInterceptor.newCall(GET("$baseUrl$epurl")).execute()
-        val jsVizInterceptor = client.newBuilder().addInterceptor(JsVizInterceptor(embedLink)).build()
+        val masterUrl = result.request.url.toString()
-        val sourceUrl = jsVizInterceptor.newCall(GET(embedLink, headers = Headers.headersOf("Referer", "$baseUrl/"))).execute().request.header("url").toString()
-        val referer = Headers.headersOf("referer", embedLink)
-        val sourceObject = json.decodeFromString<JsonObject>(
-            client.newCall(GET(sourceUrl, referer))
-                .execute().body!!.string()
-        )
-        val mediaSources = sourceObject["data"]!!.jsonObject["media"]!!.jsonObject["sources"]!!.jsonArray
-        val masterUrls = mediaSources.map { it.jsonObject["file"]!!.jsonPrimitive.content }
-        val masterUrl = masterUrls.find { it.contains("/simple/") } ?: masterUrls.first()
-        val result = client.newCall(GET(masterUrl)).execute()
         val masterPlaylist = result.body!!.string()
         return masterPlaylist.substringAfter("#EXT-X-STREAM-INF:")
             .split("#EXT-X-STREAM-INF:").map {
@@ -234,8 +226,7 @@ class NineAnime : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
     override fun searchAnimeRequest(page: Int, query: String, filters: AnimeFilterList): Request = throw Exception("Not used")
 
     private fun searchAnimeRequest(page: Int, query: String, filters: NineAnimeFilters.FilterSearchParams): Request {
-        val jsVrfInterceptor = client.newBuilder().addInterceptor(JsVrfInterceptor(query, baseUrl)).build()
+        val vrf = vrfInterceptor.getVrf(query)
-        val vrf = jsVrfInterceptor.newCall(GET("$baseUrl/filter")).execute().request.header("url").toString()
 
         var url = "$baseUrl/filter?keyword=$query"
 
@@ -286,7 +277,11 @@ class NineAnime : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
 
     override fun latestUpdatesFromElement(element: Element) = popularAnimeFromElement(element)
 
-    override fun latestUpdatesRequest(page: Int): Request = GET("$baseUrl/filter?sort=recently_updated&page=$page")
+    override fun latestUpdatesRequest(page: Int): Request {
+        // make the vrf webview available beforehand. please find another solution for this :)
+        vrfInterceptor.wake()
+        return GET("$baseUrl/filter?sort=recently_updated&page=$page")
+    }
 
     override fun latestUpdatesSelector(): String = popularAnimeSelector()