feat(de/animebase): Add VidGuard extractor + implement search filters (#2244)

src/de/animebase/build.gradle

@@ -7,8 +7,9 @@ ext {
     extName = 'Anime-Base'
     pkgNameSuffix = 'de.animebase'
     extClass = '.AnimeBase'
-    extVersionCode = 14
+    extVersionCode = 15
     libVersion = '13'
+    containsNsfw = true
 }
 
 dependencies {

src/de/animebase/src/eu/kanade/tachiyomi/animeextension/de/animebase/AnimeBase.kt

@@ -4,8 +4,10 @@ import android.app.Application
 import androidx.preference.ListPreference
 import androidx.preference.PreferenceScreen
 import eu.kanade.tachiyomi.animeextension.de.animebase.extractors.UnpackerExtractor
+import eu.kanade.tachiyomi.animeextension.de.animebase.extractors.VidGuardExtractor
 import eu.kanade.tachiyomi.animesource.ConfigurableAnimeSource
 import eu.kanade.tachiyomi.animesource.model.AnimeFilterList
+import eu.kanade.tachiyomi.animesource.model.AnimesPage
 import eu.kanade.tachiyomi.animesource.model.SAnime
 import eu.kanade.tachiyomi.animesource.model.SEpisode
 import eu.kanade.tachiyomi.animesource.model.Video
@@ -67,6 +69,8 @@ class AnimeBase : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
     override fun latestUpdatesNextPageSelector() = null
 
     // =============================== Search ===============================
+    override fun getFilterList() = AnimeBaseFilters.FILTER_LIST
+
     private val searchToken by lazy {
         client.newCall(GET("$baseUrl/searching", headers)).execute()
             .use { it.asJsoup() }
@@ -75,20 +79,49 @@ class AnimeBase : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
     }
 
     override fun searchAnimeRequest(page: Int, query: String, filters: AnimeFilterList): Request {
-        val body = FormBody.Builder()
-            .add("_token", searchToken)
-            .add("_token", searchToken)
-            .add("name_serie", query)
-            .add("jahr", "")
-            .build()
-        return POST("$baseUrl/searching", headers, body)
+        val params = AnimeBaseFilters.getSearchParameters(filters)
+
+        return when {
+            params.list.isEmpty() -> {
+                val body = FormBody.Builder()
+                    .add("_token", searchToken)
+                    .add("_token", searchToken)
+                    .add("name_serie", query)
+                    .add("jahr", params.year.toIntOrNull()?.toString() ?: "")
+                    .apply {
+                        params.languages.forEach { add("dubsub[]", it) }
+                        params.genres.forEach { add("genre[]", it) }
+                    }.build()
+                POST("$baseUrl/searching", headers, body)
+            }
+
+            else -> {
+                GET("$baseUrl/${params.list}${params.letter}?page=$page", headers)
+            }
+        }
     }
 
-    override fun searchAnimeSelector(): String = "div.col-lg-9.col-md-8 div.box-body a"
+    override fun searchAnimeParse(response: Response): AnimesPage {
+        val doc = response.use { it.asJsoup() }
+
+        return when {
+            doc.location().contains("/searching") -> {
+                val animes = doc.select(searchAnimeSelector()).map(::searchAnimeFromElement)
+                AnimesPage(animes, false)
+            }
+            else -> { // pages like filmlist or animelist
+                val animes = doc.select(popularAnimeSelector()).map(::popularAnimeFromElement)
+                val hasNext = doc.selectFirst(searchAnimeNextPageSelector()) != null
+                AnimesPage(animes, hasNext)
+            }
+        }
+    }
+
+    override fun searchAnimeSelector() = "div.col-lg-9.col-md-8 div.box-body > a"
 
     override fun searchAnimeFromElement(element: Element) = popularAnimeFromElement(element)
 
-    override fun searchAnimeNextPageSelector() = null
+    override fun searchAnimeNextPageSelector() = "ul.pagination li > a[rel=next]"
 
     // =========================== Anime Details ============================
     override fun animeDetailsParse(document: Document) = SAnime.create().apply {
@@ -154,6 +187,7 @@ class AnimeBase : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
             "Voe.SX" to "https://voe.sx/e/",
             "Lulustream" to "https://lulustream.com/e/",
             "VTube" to "https://vtbe.to/embed-",
+            "VidGuard" to "https://vembed.net/e/",
         )
     }
 
@@ -182,13 +216,14 @@ class AnimeBase : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
                                 video.audioTracks,
                             )
                         }
-                }.getOrElse { emptyList() }
+                }.onFailure { it.printStackTrace() }.getOrElse { emptyList() }
             }.flatten().ifEmpty { throw Exception("No videos xDDDDDD") }
     }
 
     private val streamWishExtractor by lazy { StreamWishExtractor(client, headers) }
     private val voeExtractor by lazy { VoeExtractor(client) }
     private val unpackerExtractor by lazy { UnpackerExtractor(client, headers) }
+    private val vidguardExtractor by lazy { VidGuardExtractor(client) }
 
     private fun getVideosFromHoster(hoster: String, urlpart: String): List<Video> {
         val url = hosterSettings.get(hoster)!! + urlpart
@@ -196,6 +231,7 @@ class AnimeBase : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
             "Streamwish" -> streamWishExtractor.videosFromUrl(url)
             "Voe.SX" -> voeExtractor.videoFromUrl(url)?.let(::listOf)
             "VTube", "Lulustream" -> unpackerExtractor.videosFromUrl(url, hoster)
+            "VidGuard" -> vidguardExtractor.videosFromUrl(url)
             else -> null
         } ?: emptyList()
     }

src/de/animebase/src/eu/kanade/tachiyomi/animeextension/de/animebase/AnimeBaseFilters.kt

@@ -0,0 +1,285 @@
+package eu.kanade.tachiyomi.animeextension.de.animebase
+
+import eu.kanade.tachiyomi.animesource.model.AnimeFilter
+import eu.kanade.tachiyomi.animesource.model.AnimeFilterList
+
+object AnimeBaseFilters {
+    open class QueryPartFilter(
+        displayName: String,
+        val vals: Array<Pair<String, String>>,
+    ) : AnimeFilter.Select<String>(
+        displayName,
+        vals.map { it.first }.toTypedArray(),
+    ) {
+        fun toQueryPart() = vals[state].second
+    }
+
+    private inline fun <reified R> AnimeFilterList.getFirst(): R {
+        return first { it is R } as R
+    }
+
+    private inline fun <reified R> AnimeFilterList.asQueryPart(): String {
+        return (getFirst<R>() as QueryPartFilter).toQueryPart()
+    }
+
+    open class CheckBoxFilterList(name: String, val pairs: Array<Pair<String, String>>) :
+        AnimeFilter.Group<AnimeFilter.CheckBox>(name, pairs.map { CheckBoxVal(it.first, false) })
+
+    private class CheckBoxVal(name: String, state: Boolean = false) : AnimeFilter.CheckBox(name, state)
+
+    private inline fun <reified R> AnimeFilterList.parseCheckbox(
+        options: Array<Pair<String, String>>,
+    ): List<String> {
+        return (getFirst<R>() as CheckBoxFilterList).state
+            .filter { it.state }
+            .map { checkbox -> options.find { it.first == checkbox.name }!!.second }
+            .filter(String::isNotBlank)
+    }
+
+    class YearFilter : AnimeFilter.Text("Erscheinungsjahr")
+
+    class LanguagesFilter : CheckBoxFilterList("Sprache", AnimeBaseFiltersData.LANGUAGES)
+    class GenresFilter : CheckBoxFilterList("Genre", AnimeBaseFiltersData.GENRES)
+
+    class ListFilter : QueryPartFilter("Liste der Konten", AnimeBaseFiltersData.LISTS)
+    class LetterFilter : QueryPartFilter("Schreiben", AnimeBaseFiltersData.LETTERS)
+
+    val FILTER_LIST get() = AnimeFilterList(
+        YearFilter(),
+        LanguagesFilter(),
+        GenresFilter(),
+        AnimeFilter.Separator(),
+        // >imagine using deepL
+        AnimeFilter.Header("Die untenstehenden Filter ignorieren die textsuche!"),
+        ListFilter(),
+        LetterFilter(),
+    )
+
+    data class FilterSearchParams(
+        val year: String = "",
+        val languages: List<String> = emptyList(),
+        val genres: List<String> = emptyList(),
+        val list: String = "",
+        val letter: String = "",
+    )
+
+    internal fun getSearchParameters(filters: AnimeFilterList): FilterSearchParams {
+        if (filters.isEmpty()) return FilterSearchParams()
+
+        return FilterSearchParams(
+            filters.getFirst<YearFilter>().state,
+            filters.parseCheckbox<LanguagesFilter>(AnimeBaseFiltersData.LANGUAGES),
+            filters.parseCheckbox<GenresFilter>(AnimeBaseFiltersData.GENRES),
+            filters.asQueryPart<ListFilter>(),
+            filters.asQueryPart<LetterFilter>(),
+        )
+    }
+
+    private object AnimeBaseFiltersData {
+        val LANGUAGES = arrayOf(
+            Pair("German Sub", "0"), // Literally Jmir
+            Pair("German Dub", "1"),
+            Pair("English Sub", "2"), // Average bri'ish
+            Pair("English Dub", "3"),
+        )
+
+        val GENRES = arrayOf(
+            Pair("Abenteuer", "1"),
+            Pair("Abenteuerkomödie", "261"),
+            Pair("Action", "2"),
+            Pair("Actiondrama", "3"),
+            Pair("Actionkomödie", "4"),
+            Pair("Adeliger", "258"),
+            Pair("Airing", "59"),
+            Pair("Alltagsdrama", "6"),
+            Pair("Alltagsleben", "7"),
+            Pair("Ältere Frau, jüngerer Mann", "210"),
+            Pair("Älterer Mann, jüngere Frau", "222"),
+            Pair("Alternative Welt", "53"),
+            Pair("Altes Asien", "187"),
+            Pair("Animation", "193"),
+            Pair("Anime & Film", "209"),
+            Pair("Anthologie", "260"),
+            Pair("Auftragsmörder / Attentäter", "265"),
+            Pair("Außerirdische", "204"),
+            Pair("Badminton", "259"),
+            Pair("Band", "121"),
+            Pair("Baseball", "234"),
+            Pair("Basketball", "239"),
+            Pair("Bionische Kräfte", "57"),
+            Pair("Boxen", "218"),
+            Pair("Boys Love", "226"),
+            Pair("Büroangestellter", "248"),
+            Pair("CG-Anime", "81"),
+            Pair("Charakterschwache Heldin", "102"),
+            Pair("Charakterschwacher Held", "101"),
+            Pair("Charakterstarke Heldin", "100"),
+            Pair("Charakterstarker Held", "88"),
+            Pair("Cyberpunk", "60"),
+            Pair("Cyborg", "109"),
+            Pair("Dämon", "58"),
+            Pair("Delinquent", "114"),
+            Pair("Denk- und Glücksspiele", "227"),
+            Pair("Detektiv", "91"),
+            Pair("Dialogwitz", "93"),
+            Pair("Dieb", "245"),
+            Pair("Diva", "112"),
+            Pair("Donghua", "257"),
+            Pair("Drache", "263"),
+            Pair("Drama", "8"),
+            Pair("Dunkle Fantasy", "90"),
+            Pair("Ecchi", "9"),
+            Pair("Elf", "89"),
+            Pair("Endzeit", "61"),
+            Pair("Epische Fantasy", "95"),
+            Pair("Episodisch", "92"),
+            Pair("Erotik", "186"),
+            Pair("Erwachsen", "70"),
+            Pair("Erwachsenwerden", "125"),
+            Pair("Essenszubereitung", "206"),
+            Pair("Familie", "63"),
+            Pair("Fantasy", "11"),
+            Pair("Fee", "264"),
+            Pair("Fighting-Shounen", "12"),
+            Pair("Football", "241"),
+            Pair("Frühe Neuzeit", "113"),
+            Pair("Fußball", "220"),
+            Pair("Gaming – Kartenspiele", "250"),
+            Pair("Ganbatte", "13"),
+            Pair("Gedächtnisverlust", "115"),
+            Pair("Gegenwart", "46"),
+            Pair("Geist", "75"),
+            Pair("Geistergeschichten", "14"),
+            Pair("Gender Bender", "216"),
+            Pair("Genie", "116"),
+            Pair("Girls Love", "201"),
+            Pair("Grundschule", "103"),
+            Pair("Harem", "15"),
+            Pair("Hentai", "16"),
+            Pair("Hexe", "97"),
+            Pair("Himmlische Wesen", "105"),
+            Pair("Historisch", "49"),
+            Pair("Horror", "17"),
+            Pair("Host-Club", "247"),
+            Pair("Idol", "122"),
+            Pair("In einem Raumschiff", "208"),
+            Pair("Independent Anime", "251"),
+            Pair("Industrialisierung", "230"),
+            Pair("Isekai", "120"),
+            Pair("Kami", "98"),
+            Pair("Kampfkunst", "246"),
+            Pair("Kampfsport", "79"),
+            Pair("Kemonomimi", "106"),
+            Pair("Kinder", "41"),
+            Pair("Kindergarten", "243"),
+            Pair("Klubs", "189"),
+            Pair("Kodomo", "40"),
+            Pair("Komödie", "18"),
+            Pair("Kopfgeldjäger", "211"),
+            Pair("Krieg", "68"),
+            Pair("Krimi", "19"),
+            Pair("Liebesdrama", "20"),
+            Pair("Mafia", "127"),
+            Pair("Magical Girl", "21"),
+            Pair("Magie", "52"),
+            Pair("Maid", "244"),
+            Pair("Malerei", "231"),
+            Pair("Manga & Doujinshi", "217"),
+            Pair("Mannschaftssport", "262"),
+            Pair("Martial Arts", "64"),
+            Pair("Mecha", "22"),
+            Pair("Mediziner", "238"),
+            Pair("Mediziner", "254"),
+            Pair("Meiji-Ära", "242"),
+            Pair("Militär", "62"),
+            Pair("Mittelalter", "76"),
+            Pair("Mittelschule", "190"),
+            Pair("Moe", "43"),
+            Pair("Monster", "54"),
+            Pair("Musik", "69"),
+            Pair("Mystery", "23"),
+            Pair("Ninja", "55"),
+            Pair("Nonsense-Komödie", "24"),
+            Pair("Oberschule", "83"),
+            Pair("Otaku", "215"),
+            Pair("Parodie", "94"),
+            Pair("Pirat", "252"),
+            Pair("Polizist", "84"),
+            Pair("PSI-Kräfte", "78"),
+            Pair("Psychodrama", "25"),
+            Pair("Real Robots", "212"),
+            Pair("Rennsport", "207"),
+            Pair("Ritter", "50"),
+            Pair("Roboter ", "73"),
+            Pair("Roboter & Android", "110"),
+            Pair("Romantische Komödie", "26"),
+            Pair("Romanze", "27"),
+            Pair("Samurai", "47"),
+            Pair("Satire", "232"),
+            Pair("Schule", "119"),
+            Pair("Schusswaffen", "82"),
+            Pair("Schwerter & Co", "51"),
+            Pair("Schwimmen", "223"),
+            Pair("Scifi", "28"),
+            Pair("Seinen", "39"),
+            Pair("Sentimentales Drama", "29"),
+            Pair("Shounen", "37"),
+            Pair("Slapstick", "56"),
+            Pair("Slice of Life", "5"),
+            Pair("Solosänger", "219"),
+            Pair("Space Opera", "253"),
+            Pair("Splatter", "36"),
+            Pair("Sport", "30"),
+            Pair("Stoische Heldin", "123"),
+            Pair("Stoischer Held", "85"),
+            Pair("Super Robots", "203"),
+            Pair("Super-Power", "71"),
+            Pair("Superhelden", "256"),
+            Pair("Supernatural", "225"),
+            Pair("Tanzen", "249"),
+            Pair("Tennis", "233"),
+            Pair("Theater", "224"),
+            Pair("Thriller", "31"),
+            Pair("Tiermensch", "111"),
+            Pair("Tomboy", "104"),
+            Pair("Tragödie", "86"),
+            Pair("Tsundere", "107"),
+            Pair("Überlebenskampf", "117"),
+            Pair("Übermäßige Gewaltdarstellung", "34"),
+            Pair("Unbestimmt", "205"),
+            Pair("Universität", "214"),
+            Pair("Vampir", "35"),
+            Pair("Verworrene Handlung", "126"),
+            Pair("Virtuelle Welt", "108"),
+            Pair("Volleyball", "191"),
+            Pair("Volljährig", "67"),
+            Pair("Wassersport", "266"),
+            Pair("Weiblich", "45"),
+            Pair("Weltkriege", "128"),
+            Pair("Weltraum", "74"),
+            Pair("Widerwillige Heldin", "124"),
+            Pair("Widerwilliger Held", "188"),
+            Pair("Yandere", "213"),
+            Pair("Yaoi", "32"),
+            Pair("Youkai", "99"),
+            Pair("Yuri", "33"),
+            Pair("Zeichentrick", "77"),
+            Pair("Zeichentrick", "255"),
+            Pair("Zeitgenössische Fantasy", "80"),
+            Pair("Zeitsprung", "240"),
+            Pair("Zombie", "87"),
+        )
+
+        val LISTS = arrayOf(
+            Pair("Keine", ""),
+            Pair("Anime", "animelist"),
+            Pair("Film", "filmlist"),
+            Pair("Hentai", "hentailist"),
+            Pair("Sonstiges", "misclist"),
+        )
+
+        val LETTERS = arrayOf(Pair("Jede", "")) + ('A'..'Z').map {
+            Pair(it.toString(), "/$it")
+        }.toTypedArray()
+    }
+}

src/de/animebase/src/eu/kanade/tachiyomi/animeextension/de/animebase/extractors/VidGuardExtractor.kt

@@ -0,0 +1,124 @@
+package eu.kanade.tachiyomi.animeextension.de.animebase.extractors
+
+import android.app.Application
+import android.os.Handler
+import android.os.Looper
+import android.util.Base64
+import android.webkit.JavascriptInterface
+import android.webkit.WebSettings
+import android.webkit.WebView
+import android.webkit.WebViewClient
+import eu.kanade.tachiyomi.animesource.model.Video
+import eu.kanade.tachiyomi.network.GET
+import eu.kanade.tachiyomi.util.asJsoup
+import okhttp3.Headers
+import okhttp3.HttpUrl.Companion.toHttpUrl
+import okhttp3.OkHttpClient
+import uy.kohesive.injekt.injectLazy
+import java.util.concurrent.CountDownLatch
+import java.util.concurrent.TimeUnit
+
+class VidGuardExtractor(private val client: OkHttpClient) {
+    private val context: Application by injectLazy()
+    private val handler by lazy { Handler(Looper.getMainLooper()) }
+
+    class JsObject(private val latch: CountDownLatch) {
+        var payload: String = ""
+
+        @JavascriptInterface
+        fun passPayload(passedPayload: String) {
+            payload = passedPayload
+            latch.countDown()
+        }
+    }
+
+    fun videosFromUrl(url: String): List<Video> {
+        val doc = client.newCall(GET(url)).execute().use { it.asJsoup() }
+        val scriptUrl = doc.selectFirst("script[src*=ad/plugin]")
+            ?.absUrl("src")
+            ?: return emptyList()
+
+        val headers = Headers.headersOf("Referer", url)
+        val script = client.newCall(GET(scriptUrl, headers)).execute()
+            .use { it.body.string() }
+
+        val sources = getSourcesFromScript(script, url)
+            .takeIf { it.isNotBlank() && it != "undefined" }
+            ?: return emptyList()
+
+        return sources.substringAfter("stream:[").substringBefore("}]")
+            .split('{')
+            .drop(1)
+            .mapNotNull { line ->
+                val resolution = line.substringAfter("Label\":\"").substringBefore('"')
+                val videoUrl = line.substringAfter("URL\":\"").substringBefore('"')
+                    .takeIf(String::isNotBlank)
+                    ?.let(::fixUrl)
+                    ?: return@mapNotNull null
+                Video(videoUrl, "VidGuard - $resolution", videoUrl, headers)
+            }
+    }
+
+    private fun getSourcesFromScript(script: String, url: String): String {
+        val latch = CountDownLatch(1)
+
+        var webView: WebView? = null
+
+        val jsinterface = JsObject(latch)
+
+        handler.post {
+            val webview = WebView(context)
+            webView = webview
+            with(webview.settings) {
+                javaScriptEnabled = true
+                domStorageEnabled = true
+                databaseEnabled = true
+                useWideViewPort = false
+                loadWithOverviewMode = false
+                cacheMode = WebSettings.LOAD_NO_CACHE
+            }
+
+            webview.addJavascriptInterface(jsinterface, "android")
+            webview.webViewClient = object : WebViewClient() {
+                override fun onPageFinished(view: WebView?, url: String?) {
+                    view?.clearCache(true)
+                    view?.clearFormData()
+                    view?.evaluateJavascript(script) {}
+                    view?.evaluateJavascript("window.android.passPayload(JSON.stringify(window.svg))") {}
+                }
+            }
+
+            webview.loadDataWithBaseURL(url, "<html></html>", "text/html", "UTF-8", null)
+        }
+
+        latch.await(5, TimeUnit.SECONDS)
+
+        handler.post {
+            webView?.stopLoading()
+            webView?.destroy()
+            webView = null
+        }
+
+        return jsinterface.payload
+    }
+
+    private fun fixUrl(url: String): String {
+        val httpUrl = url.toHttpUrl()
+        val originalSign = httpUrl.queryParameter("sig")!!
+        val newSign = originalSign.chunked(2).joinToString("") {
+            Char(it.toInt(16) xor 2).toString()
+        }
+            .let { String(Base64.decode(it, Base64.DEFAULT)) }
+            .substring(5)
+            .chunked(2)
+            .reversed()
+            .joinToString("")
+            .substring(5)
+
+        return httpUrl.newBuilder()
+            .removeAllQueryParameters("sig")
+            .addQueryParameter("sig", newSign)
+            .build()
+            .toString()
+    }
+}