INSANEMODE/aniyomi-extensions-backup
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ScrapingWars: Use one vrf webview for everything and fix extraction (#1227)

Browse Source 
* ScrapingWars: Use one vrf webview for everything and fix embed extraction

* Update some options
This commit is contained in:
Samfun75
2023-02-01 11:05:43 +03:00
committed by GitHub
parent 6c9b6a641b
commit c3645048c8
5 changed files with 87 additions and 247 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/en/nineanime/build.gradle
View File

@ -5,7 +5,7 @@ ext {
extName = '9anime'
pkgNameSuffix = 'en.nineanime'
extClass = '.NineAnime'
extVersionCode = 27
extVersionCode = 28
libVersion = '13'
}

52
src/en/nineanime/src/eu/kanade/tachiyomi/animeextension/en/nineanime/JsInterceptor.kt
View File

@ -9,6 +9,7 @@ import android.webkit.WebResourceRequest
import android.webkit.WebResourceResponse
import android.webkit.WebView
import android.webkit.WebViewClient
import android.widget.Toast
import eu.kanade.tachiyomi.network.GET
import okhttp3.Headers
import okhttp3.Interceptor
@ -24,18 +25,19 @@ class JsInterceptor(private val lang: String) : Interceptor {
private val context = Injekt.get<Application>()
private val handler by lazy { Handler(Looper.getMainLooper()) }
class JsObject(private val latch: CountDownLatch, var payload: String = "") {
class JsObject(var payload: String = "") {
@JavascriptInterface
fun passPayload(passedPayload: String) {
payload = passedPayload
latch.countDown()
}
}
override fun intercept(chain: Interceptor.Chain): Response {
val originalRequest = chain.request()
val newRequest = resolveWithWebView(originalRequest) ?: throw Exception("Please reload Episode List")
handler.post {
context.let { Toast.makeText(it, "This might take a while, Don't close me", Toast.LENGTH_LONG).show() }
}
val newRequest = resolveWithWebView(originalRequest) ?: throw Exception("Someting went wrong or took too long")
return chain.proceed(newRequest)
}
@ -49,20 +51,22 @@ class JsInterceptor(private val lang: String) : Interceptor {
val origRequestUrl = request.url.toString()
val jsinterface = JsObject(latch)
val jsinterface = JsObject()
// JavaSrcipt gets the Dub or Sub link of vidstream
val jsScript = """
(function(){
let hoster = document.querySelector('div[data-type="$lang"] ul li[data-sv-id="41"]');
let event = document.createEvent('HTMLEvents');
event.initEvent('click',true,true);
hoster.dispatchEvent(event);
let jqclk = jQuery.Event('click');
jqclk.isTrusted = true;
jqclk.originalEvent = {
isTrusted: true
};
${'$'}('div[data-type="$lang"] ul li[data-sv-id="41"]').trigger(jqclk);
let intervalId = setInterval(() => {
let element = document.querySelector("#player iframe");
if (element) {
clearInterval(intervalId);
window.android.passPayload(element.src)
window.android.passPayload(element.src);
}
}, 500);
})();
@ -72,8 +76,6 @@ class JsInterceptor(private val lang: String) : Interceptor {
var newRequest: Request? = null
var head = ""
handler.post {
val webview = WebView(context)
webView = webview
@ -87,8 +89,17 @@ class JsInterceptor(private val lang: String) : Interceptor {
webview.addJavascriptInterface(jsinterface, "android")
webview.webViewClient = object : WebViewClient() {
override fun shouldInterceptRequest(view: WebView?, request: WebResourceRequest?): WebResourceResponse? {
if (request?.url.toString().contains("https://vidstream.pro/embed")) {
head = request?.url.toString()
if (!request?.url.toString().contains("vidstream") &&
!request?.url.toString().contains("vizcloud")
) return null
if (request?.url.toString().contains("/simple/")) {
newRequest = GET(
request?.url.toString(),
Headers.headersOf("referer", "/orp.maertsdiv//:sptth".reversed())
)
latch.countDown()
return null
}
return super.shouldInterceptRequest(view, request)
}
@ -100,24 +111,13 @@ class JsInterceptor(private val lang: String) : Interceptor {
}
}
latch.await(12, TimeUnit.SECONDS)
latch.await(30, TimeUnit.SECONDS)
handler.post {
webView?.stopLoading()
webView?.destroy()
webView = null
}
newRequest = GET(
request.url.toString(),
headers = Headers.headersOf(
"url",
if (jsinterface.payload.isNullOrEmpty() || (!jsinterface.payload.contains("https://vidstream.pro/embed"))) {
head
} else {
jsinterface.payload
}
)
)
return newRequest
}
}

140
src/en/nineanime/src/eu/kanade/tachiyomi/animeextension/en/nineanime/JsVizInterceptor.kt
View File

@ -1,140 +0,0 @@
package eu.kanade.tachiyomi.animeextension.en.nineanime
import android.annotation.SuppressLint
import android.app.Application
import android.os.Handler
import android.os.Looper
import android.webkit.JavascriptInterface
import android.webkit.WebResourceRequest
import android.webkit.WebResourceResponse
import android.webkit.WebView
import android.webkit.WebViewClient
import eu.kanade.tachiyomi.network.GET
import okhttp3.Headers
import okhttp3.Interceptor
import okhttp3.Request
import okhttp3.Response
import uy.kohesive.injekt.Injekt
import uy.kohesive.injekt.api.get
import java.util.concurrent.CountDownLatch
import java.util.concurrent.TimeUnit
class JsVizInterceptor(private val embedLink: String) : Interceptor {
private val context = Injekt.get<Application>()
private val handler by lazy { Handler(Looper.getMainLooper()) }
class JsObject(private val latch: CountDownLatch, var payload: String = "") {
@JavascriptInterface
fun passPayload(passedPayload: String) {
payload = passedPayload
latch.countDown()
}
}
override fun intercept(chain: Interceptor.Chain): Response {
val originalRequest = chain.request()
val newRequest = resolveWithWebView(originalRequest) ?: throw Exception("Please reload Episode List")
return chain.proceed(newRequest)
}
@SuppressLint("SetJavaScriptEnabled")
private fun resolveWithWebView(request: Request): Request? {
val latch = CountDownLatch(1)
var webView: WebView? = null
val origRequestUrl = request.url.toString()
val jsinterface = JsObject(latch)
// JavaSrcipt creates Iframe on vidstream page to bypass iframe-cors and gets the sourceUrl
val jsScript = """
(function(){
const html = '<iframe src="$embedLink" allow="autoplay; fullscreen" allowfullscreen="yes" scrolling="no" style="width: 100%; height: 100%; overflow: hidden;" frameborder="no" onload="handleIframeLoad()"></iframe>';
document.body.innerHTML += html;
const iframe = document.querySelector('iframe');
const originalOpen = iframe.contentWindow.XMLHttpRequest.prototype.open;
iframe.contentWindow.XMLHttpRequest.prototype.open = function(method, url, async) {
if (!url.includes("ping") && !url.includes("/assets/") && !url.includes("thumbnails") && !url.includes("jpg") && !url.includes("m3u8") && !url.includes("simplewebanalysis")) {
if (url == null) {
const entries = iframe.contentWindow.performance.getEntries();
entries.forEach((entry) => {
if (entry.initiatorType.includes("xmlhttprequest")) {
if (!entry.name.includes("/ping/") && !entry.name.includes("/assets/") && !entry.name.includes("thumbnails")) {
window.android.passPayload(entry.name);
}
}
});
} else {
window.android.passPayload("https://" + document.domain + "/" + url);
}
}
originalOpen.apply(this, arguments);
}
})();
"""
val headers = request.headers.toMultimap().mapValues { it.value.getOrNull(0) ?: "" }.toMutableMap()
var newRequest: Request? = null
var head = ""
handler.post {
val webview = WebView(context)
webView = webview
with(webview.settings) {
javaScriptEnabled = true
domStorageEnabled = true
databaseEnabled = true
useWideViewPort = false
loadWithOverviewMode = false
userAgentString = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:106.0) Gecko/20100101 Firefox/106.0"
webview.addJavascriptInterface(jsinterface, "android")
webview.webViewClient = object : WebViewClient() {
override fun shouldInterceptRequest(view: WebView?, request: WebResourceRequest?): WebResourceResponse? {
if (request?.url.toString().contains("https://vidstream.pro/")) {
if (request?.url.toString().contains("/embed/") || request?.url.toString().contains("/ping/") || request?.url.toString().contains("favicon.ico") ||
request?.url.toString().contains("/assets/") || request?.url.toString().contains("/players/")
) {
return null
} else {
head = request?.url.toString()
}
}
return super.shouldInterceptRequest(view, request)
}
override fun onPageFinished(view: WebView?, url: String?) {
view?.evaluateJavascript(jsScript) {}
}
}
webView?.loadUrl(origRequestUrl, headers)
}
}
latch.await(12, TimeUnit.SECONDS)
handler.post {
webView?.stopLoading()
webView?.destroy()
webView = null
}
newRequest = GET(
request.url.toString(),
headers = Headers.headersOf(
"url",
if (jsinterface.payload.isNullOrEmpty() || (!jsinterface.payload.contains("https://vidstream.pro"))) {
head
} else {
jsinterface.payload
}
)
)
return newRequest
}
}

99
src/en/nineanime/src/eu/kanade/tachiyomi/animeextension/en/nineanime/JsVrfInterceptor.kt
View File

@ -4,74 +4,42 @@ import android.annotation.SuppressLint
import android.app.Application
import android.os.Handler
import android.os.Looper
import android.webkit.JavascriptInterface
import android.webkit.WebResourceRequest
import android.webkit.WebView
import android.webkit.WebViewClient
import eu.kanade.tachiyomi.network.GET
import okhttp3.Headers
import okhttp3.Interceptor
import okhttp3.Request
import okhttp3.Response
import uy.kohesive.injekt.Injekt
import uy.kohesive.injekt.api.get
import java.util.concurrent.CountDownLatch
import java.util.concurrent.TimeUnit
class JsVrfInterceptor(private val query: String, private val baseUrl: String) : Interceptor {
class JsVrfInterceptor(private val baseUrl: String) {
private val context = Injekt.get<Application>()
private val handler by lazy { Handler(Looper.getMainLooper()) }
private val vrfWebView = createWebView()
class JsObject(private val latch: CountDownLatch, var payload: String = "") {
@JavascriptInterface
fun passPayload(passedPayload: String) {
payload = passedPayload
latch.countDown()
fun wake() = ""
fun getVrf(query: String): String {
val jscript = getJs(query)
val cdl = CountDownLatch(1)
var vrf = ""
handler.post {
vrfWebView?.evaluateJavascript(jscript) {
vrf = it?.removeSurrounding("\"") ?: ""
cdl.countDown()
}
}
override fun intercept(chain: Interceptor.Chain): Response {
val originalRequest = chain.request()
val newRequest = resolveWithWebView(originalRequest) ?: throw Exception("Please reload Episode List")
return chain.proceed(newRequest)
cdl.await(12, TimeUnit.SECONDS)
if (vrf.isBlank()) throw Exception("vrf could not be retrieved")
return vrf
}
@SuppressLint("SetJavaScriptEnabled")
private fun resolveWithWebView(request: Request): Request? {
private fun createWebView(): WebView? {
val latch = CountDownLatch(1)
var webView: WebView? = null
val origRequestUrl = request.url.toString()
val jsinterface = JsObject(latch)
// JavaScript uses search of 9Anime to convert IDs & Querys to the VRF-Key
val jsScript = """
(function() {
document.querySelector("form.filters input.form-control").value = '$query';
let inputElemente = document.querySelector('form.filters input.form-control');
let e = document.createEvent('HTMLEvents');
e.initEvent('keyup', true, true);
inputElemente.dispatchEvent(e);
let intervalId = setInterval(() => {
let element = document.querySelector('form.filters input[type="hidden"]').value;
if (element) {
clearInterval(intervalId);
window.android.passPayload(element)
}
}, 100);
})();
"""
val headers = request.headers.toMultimap().mapValues { it.value.getOrNull(0) ?: "" }.toMutableMap()
var newRequest: Request? = null
handler.post {
val webview = WebView(context)
webView = webview
@ -83,8 +51,6 @@ class JsVrfInterceptor(private val query: String, private val baseUrl: String) :
loadWithOverviewMode = false
userAgentString = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:106.0) Gecko/20100101 Firefox/106.0"
webview.addJavascriptInterface(jsinterface, "android")
webview.webViewClient = object : WebViewClient() {
override fun shouldOverrideUrlLoading(view: WebView?, request: WebResourceRequest?): Boolean {
if (request?.url.toString().contains("$baseUrl/filter")) {
@ -95,21 +61,40 @@ class JsVrfInterceptor(private val query: String, private val baseUrl: String) :
}
}
override fun onPageFinished(view: WebView?, url: String?) {
view?.evaluateJavascript(jsScript) {}
latch.countDown()
}
}
webView?.loadUrl(origRequestUrl, headers)
webView?.loadUrl("$baseUrl/filter")
}
}
latch.await(12, TimeUnit.SECONDS)
latch.await()
handler.post {
webView?.stopLoading()
webView?.destroy()
webView = null
}
newRequest = GET(request.url.toString(), headers = Headers.headersOf("url", jsinterface.payload, "user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:106.0) Gecko/20100101 Firefox/106.0"))
return newRequest
return webView
}
private fun getJs(query: String): String {
return """
(function() {
document.querySelector("form.filters input.form-control").value = '$query';
let inputElemente = document.querySelector('form.filters input.form-control');
let e = document.createEvent('HTMLEvents');
e.initEvent('keyup', true, true);
inputElemente.dispatchEvent(e);
let val = "";
while (val == "") {
let element = document.querySelector('form.filters input[type="hidden"]').value;
if (element) {
val = element;
break;
}
}
document.querySelector("form.filters input.form-control").value = '';
return val;
})();
""".trimIndent()
}
}

39
src/en/nineanime/src/eu/kanade/tachiyomi/animeextension/en/nineanime/NineAnime.kt
View File

@ -21,8 +21,6 @@ import kotlinx.coroutines.runBlocking
import kotlinx.serialization.decodeFromString
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.jsonArray
import kotlinx.serialization.json.jsonObject
import kotlinx.serialization.json.jsonPrimitive
import okhttp3.Headers
import okhttp3.OkHttpClient
@ -50,6 +48,8 @@ class NineAnime : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
private val json: Json by injectLazy()
private val vrfInterceptor by lazy { JsVrfInterceptor(baseUrl) }
private val preferences: SharedPreferences by lazy {
Injekt.get<Application>().getSharedPreferences("source_$id", 0x0000)
}
@ -60,7 +60,11 @@ class NineAnime : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
override fun popularAnimeSelector(): String = "div.ani.items > div"
override fun popularAnimeRequest(page: Int): Request = GET("$baseUrl/filter?sort=trending&page=$page")
override fun popularAnimeRequest(page: Int): Request {
// make the vrf webview available beforehand. please find another solution for this :)
vrfInterceptor.wake()
return GET("$baseUrl/filter?sort=trending&page=$page")
}
override fun popularAnimeFromElement(element: Element) = SAnime.create().apply {
setUrlWithoutDomain(element.select("a.name").attr("href").substringBefore("?"))
@ -72,8 +76,7 @@ class NineAnime : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
override fun episodeListRequest(anime: SAnime): Request {
val id = client.newCall(GET(baseUrl + anime.url)).execute().asJsoup().selectFirst("div[data-id]").attr("data-id")
val jsVrfInterceptor = client.newBuilder().addInterceptor(JsVrfInterceptor(id, baseUrl)).build()
val vrf = jsVrfInterceptor.newCall(GET("$baseUrl/filter")).execute().request.header("url").toString()
val vrf = vrfInterceptor.getVrf(id)
return GET("$baseUrl/ajax/episode/list/$id?vrf=${java.net.URLEncoder.encode(vrf, "utf-8")}", headers = Headers.headersOf("url", anime.url))
}
@ -129,8 +132,7 @@ class NineAnime : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
override fun videoListRequest(episode: SEpisode): Request {
val ids = episode.url.substringAfter("list/").substringBefore("?vrf")
val jsVrfInterceptor = client.newBuilder().addInterceptor(JsVrfInterceptor(ids, baseUrl)).build()
val vrf = jsVrfInterceptor.newCall(GET("$baseUrl/filter")).execute().request.header("url").toString()
val vrf = vrfInterceptor.getVrf(ids)
val url = "/ajax/server/list/$ids?vrf=${java.net.URLEncoder.encode(vrf, "utf-8")}"
val epurl = episode.url.substringAfter("epurl=")
return GET(baseUrl + url, headers = Headers.headersOf("url", epurl))
@ -155,18 +157,8 @@ class NineAnime : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
private fun extractVideo(lang: String, epurl: String): List<Video> {
val jsInterceptor = client.newBuilder().addInterceptor(JsInterceptor(lang.lowercase())).build()
val embedLink = jsInterceptor.newCall(GET("$baseUrl$epurl")).execute().request.header("url").toString()
val jsVizInterceptor = client.newBuilder().addInterceptor(JsVizInterceptor(embedLink)).build()
val sourceUrl = jsVizInterceptor.newCall(GET(embedLink, headers = Headers.headersOf("Referer", "$baseUrl/"))).execute().request.header("url").toString()
val referer = Headers.headersOf("referer", embedLink)
val sourceObject = json.decodeFromString<JsonObject>(
client.newCall(GET(sourceUrl, referer))
.execute().body!!.string()
)
val mediaSources = sourceObject["data"]!!.jsonObject["media"]!!.jsonObject["sources"]!!.jsonArray
val masterUrls = mediaSources.map { it.jsonObject["file"]!!.jsonPrimitive.content }
val masterUrl = masterUrls.find { it.contains("/simple/") } ?: masterUrls.first()
val result = client.newCall(GET(masterUrl)).execute()
val result = jsInterceptor.newCall(GET("$baseUrl$epurl")).execute()
val masterUrl = result.request.url.toString()
val masterPlaylist = result.body!!.string()
return masterPlaylist.substringAfter("#EXT-X-STREAM-INF:")
.split("#EXT-X-STREAM-INF:").map {
@ -234,8 +226,7 @@ class NineAnime : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
override fun searchAnimeRequest(page: Int, query: String, filters: AnimeFilterList): Request = throw Exception("Not used")
private fun searchAnimeRequest(page: Int, query: String, filters: NineAnimeFilters.FilterSearchParams): Request {
val jsVrfInterceptor = client.newBuilder().addInterceptor(JsVrfInterceptor(query, baseUrl)).build()
val vrf = jsVrfInterceptor.newCall(GET("$baseUrl/filter")).execute().request.header("url").toString()
val vrf = vrfInterceptor.getVrf(query)
var url = "$baseUrl/filter?keyword=$query"
@ -286,7 +277,11 @@ class NineAnime : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
override fun latestUpdatesFromElement(element: Element) = popularAnimeFromElement(element)
override fun latestUpdatesRequest(page: Int): Request = GET("$baseUrl/filter?sort=recently_updated&page=$page")
override fun latestUpdatesRequest(page: Int): Request {
// make the vrf webview available beforehand. please find another solution for this :)
vrfInterceptor.wake()
return GET("$baseUrl/filter?sort=recently_updated&page=$page")
}
override fun latestUpdatesSelector(): String = popularAnimeSelector()