refactor: Move some extractor to libraries for datalife engine multisrc (#2246)

lib/streamdav-extractor/build.gradle.kts

@@ -0,0 +1,17 @@
+plugins {
+    id("com.android.library")
+    kotlin("android")
+}
+
+android {
+    compileSdk = AndroidConfig.compileSdk
+    namespace = "eu.kanade.tachiyomi.lib.streamdavextractor"
+
+    defaultConfig {
+        minSdk = AndroidConfig.minSdk
+    }
+}
+
+dependencies {
+    compileOnly(libs.bundles.common)
+}

lib/streamdav-extractor/src/main/java/eu/kanade/tachiyomi/lib/streamdavextractor/StreamDavExtractor.kt

@@ -0,0 +1,17 @@
+package eu.kanade.tachiyomi.lib.streamdavextractor
+
+import eu.kanade.tachiyomi.animesource.model.Video
+import eu.kanade.tachiyomi.network.GET
+import eu.kanade.tachiyomi.util.asJsoup
+import okhttp3.OkHttpClient
+
+class StreamDavExtractor(private val client: OkHttpClient) {
+    fun videosFromUrl(url: String, prefix: String = ""): List<Video> {
+        val document = client.newCall(GET(url)).execute().use { it.asJsoup() }
+        return document.select("source").map {
+            val videoUrl = it.attr("src")
+            val quality = it.attr("label")
+            Video(url, "${prefix}StreamDav - ($quality)", videoUrl)
+        }
+    }
+}

lib/streamhidevid-extractor/build.gradle.kts

@@ -0,0 +1,20 @@
+plugins {
+    id("com.android.library")
+    kotlin("android")
+}
+
+android {
+    compileSdk = AndroidConfig.compileSdk
+    namespace = "eu.kanade.tachiyomi.lib.streamhidevidextractor"
+
+    defaultConfig {
+        minSdk = AndroidConfig.minSdk
+    }
+}
+
+dependencies {
+    compileOnly(libs.bundles.common)
+    implementation("dev.datlag.jsunpacker:jsunpacker:1.0.1") {
+        exclude(group = "org.jetbrains.kotlin", module = "kotlin-stdlib-jdk8")
+    }
+}

lib/streamhidevid-extractor/src/main/java/eu/kanade/tachiyomi/lib/streamhidevidextractor/StreamHideVidExtractor.kt

@@ -0,0 +1,36 @@
+package eu.kanade.tachiyomi.lib.streamhidevidextractor
+
+import dev.datlag.jsunpacker.JsUnpacker
+import eu.kanade.tachiyomi.animesource.model.Track
+import eu.kanade.tachiyomi.animesource.model.Video
+import eu.kanade.tachiyomi.network.GET
+import okhttp3.OkHttpClient
+
+class StreamHideVidExtractor(private val client: OkHttpClient) {
+    // from nineanime / ask4movie FilemoonExtractor
+    private val subtitleRegex = Regex("""#EXT-X-MEDIA:TYPE=SUBTITLES.*?NAME="(.*?)".*?URI="(.*?)"""")
+
+    fun videosFromUrl(url: String, prefix: String = ""): List<Video> {
+        val page = client.newCall(GET(url)).execute().body.string()
+        val unpacked = JsUnpacker.unpackAndCombine(page) ?: return emptyList()
+        val playlistUrl = unpacked.substringAfter("sources:")
+            .substringAfter("file:\"") // StreamHide
+            .substringAfter("src:\"") // StreamVid
+            .substringBefore('"')
+
+        val playlistData = client.newCall(GET(playlistUrl)).execute().body.string()
+
+        val subs = subtitleRegex.findAll(playlistData).map {
+            val urlPart = it.groupValues[2]
+            val subUrl = when {
+                !urlPart.startsWith("https:") ->
+                    playlistUrl.substringBeforeLast("/") + "/$urlPart"
+                else -> urlPart
+            }
+            Track(subUrl, it.groupValues[1])
+        }.toList()
+
+        // The playlist usually only have one video quality.
+        return listOf(Video(playlistUrl, "${prefix}StreamHideVid", playlistUrl, subtitleTracks = subs))
+    }
+}

lib/streamhub-extractor/build.gradle.kts

@@ -0,0 +1,18 @@
+plugins {
+    id("com.android.library")
+    kotlin("android")
+}
+
+android {
+    compileSdk = AndroidConfig.compileSdk
+    namespace = "eu.kanade.tachiyomi.lib.streamhubextractor"
+
+    defaultConfig {
+        minSdk = AndroidConfig.minSdk
+    }
+}
+
+dependencies {
+    implementation(project(":lib-playlist-utils"))
+    compileOnly(libs.bundles.common)
+}

lib/streamhub-extractor/src/main/java/eu/kanade/tachiyomi/lib/streamhubextractor/StreamHubExtractor.kt

@@ -0,0 +1,18 @@
+package eu.kanade.tachiyomi.lib.streamhubextractor
+
+import eu.kanade.tachiyomi.animesource.model.Video
+import eu.kanade.tachiyomi.lib.playlistutils.PlaylistUtils
+import eu.kanade.tachiyomi.network.GET
+import okhttp3.OkHttpClient
+
+class StreamHubExtractor(private val client: OkHttpClient) {
+
+    fun videosFromUrl(url: String, prefix: String = ""): List<Video> {
+        val document = client.newCall(GET(url)).execute().use { it.body.string() }
+        val id = Regex("urlset\\|(.*?)\\|").find(document)?.groupValues?.get(1)
+        val sub = Regex("width\\|(.*?)\\|").find(document)?.groupValues?.get(1)
+        val masterUrl = "https://${sub}.streamhub.ink/hls/,${id},.urlset/master.m3u8"
+        return PlaylistUtils(client).extractFromHls(masterUrl, videoNameGen = { "${prefix}StreamHub - (${it})" })
+    }
+
+}

lib/streamvid-extractor/build.gradle.kts

@@ -0,0 +1,21 @@
+plugins {
+    id("com.android.library")
+    kotlin("android")
+}
+
+android {
+    compileSdk = AndroidConfig.compileSdk
+    namespace = "eu.kanade.tachiyomi.lib.streamvidextractor"
+
+    defaultConfig {
+        minSdk = AndroidConfig.minSdk
+    }
+}
+
+dependencies {
+    implementation(project(":lib-playlist-utils"))
+    implementation("dev.datlag.jsunpacker:jsunpacker:1.0.1") {
+        exclude(group = "org.jetbrains.kotlin", module = "kotlin-stdlib-jdk8")
+    }
+    compileOnly(libs.bundles.common)
+}

lib/streamvid-extractor/src/main/java/eu/kanade/tachiyomi/lib/streamvidextractor/StreamVidExtractor.kt

@@ -0,0 +1,22 @@
+package eu.kanade.tachiyomi.lib.streamvidextractor
+
+import dev.datlag.jsunpacker.JsUnpacker
+import eu.kanade.tachiyomi.animesource.model.Video
+import eu.kanade.tachiyomi.lib.playlistutils.PlaylistUtils
+import eu.kanade.tachiyomi.network.GET
+import eu.kanade.tachiyomi.util.asJsoup
+import okhttp3.OkHttpClient
+
+class StreamVidExtractor(private val client: OkHttpClient) {
+    fun videosFromUrl(url: String, prefix: String = ""): List<Video> {
+        return runCatching {
+            val doc = client.newCall(GET(url)).execute().asJsoup()
+
+            val script = doc.selectFirst("script:containsData(eval):containsData(p,a,c,k,e,d)")?.data()
+                ?.let(JsUnpacker::unpackAndCombine)
+                ?: return emptyList()
+            val masterUrl = script.substringAfter("sources:[{src:\"").substringBefore("\",")
+            PlaylistUtils(client).extractFromHls(masterUrl, videoNameGen = { "${prefix}StreamVid - (${it}p)" })
+        }.getOrElse { emptyList() }
+    }
+}

lib/upstream-extractor/build.gradle.kts

@@ -0,0 +1,21 @@
+plugins {
+    id("com.android.library")
+    kotlin("android")
+}
+
+android {
+    compileSdk = AndroidConfig.compileSdk
+    namespace = "eu.kanade.tachiyomi.lib.upstreamextractor"
+
+    defaultConfig {
+        minSdk = AndroidConfig.minSdk
+    }
+}
+
+dependencies {
+    compileOnly(libs.bundles.common)
+    implementation(project(":lib-playlist-utils"))
+    implementation("dev.datlag.jsunpacker:jsunpacker:1.0.1") {
+        exclude(group = "org.jetbrains.kotlin", module = "kotlin-stdlib-jdk8")
+    }
+}

lib/upstream-extractor/src/main/java/eu/kanade/tachiyomi/lib/upstreamextractor/UpstreamExtractor.kt

@@ -0,0 +1,19 @@
+package eu.kanade.tachiyomi.lib.upstreamextractor
+
+import dev.datlag.jsunpacker.JsUnpacker
+import eu.kanade.tachiyomi.animesource.model.Video
+import eu.kanade.tachiyomi.lib.playlistutils.PlaylistUtils
+import eu.kanade.tachiyomi.network.GET
+import eu.kanade.tachiyomi.util.asJsoup
+import okhttp3.Headers
+import okhttp3.OkHttpClient
+import okhttp3.internal.commonEmptyHeaders
+
+class UpstreamExtractor(private val client: OkHttpClient) {
+    fun videosFromUrl(url: String, prefix: String = ""): List<Video> =
+        runCatching {
+            val jsE = client.newCall(GET(url)).execute().asJsoup().selectFirst("script:containsData(eval)")!!.data()
+            val masterUrl = JsUnpacker.unpackAndCombine(jsE)!!.substringAfter("{file:\"").substringBefore("\"}")
+            PlaylistUtils(client).extractFromHls(masterUrl, videoNameGen = { "${prefix}Upstream - $it" } )
+        }.getOrDefault(emptyList())
+}

lib/vido-extractor/build.gradle.kts

@@ -0,0 +1,21 @@
+plugins {
+    id("com.android.library")
+    kotlin("android")
+}
+
+android {
+    compileSdk = AndroidConfig.compileSdk
+    namespace = "eu.kanade.tachiyomi.lib.vidoextractor"
+
+    defaultConfig {
+        minSdk = AndroidConfig.minSdk
+    }
+}
+
+dependencies {
+    implementation(project(":lib-playlist-utils"))
+    implementation("dev.datlag.jsunpacker:jsunpacker:1.0.1") {
+        exclude(group = "org.jetbrains.kotlin", module = "kotlin-stdlib-jdk8")
+    }
+    compileOnly(libs.bundles.common)
+}

lib/vido-extractor/src/main/java/eu/kanade/tachiyomi/lib/vidoextractor/VidoExtractor.kt

@@ -0,0 +1,17 @@
+package eu.kanade.tachiyomi.lib.vidoextractor
+
+import eu.kanade.tachiyomi.animesource.model.Video
+import eu.kanade.tachiyomi.lib.playlistutils.PlaylistUtils
+import eu.kanade.tachiyomi.network.GET
+import okhttp3.OkHttpClient
+
+class VidoExtractor(private val client: OkHttpClient) {
+
+    fun videosFromUrl(url: String, prefix: String = ""): List<Video> {
+        val document = client.newCall(GET(url)).execute().use { it.body.string() }
+        val id = Regex("master\\|(.*?)\\|").find(document)?.groupValues?.get(1)
+        val masterUrl = "https://pink.vido.lol/hls/${id}/master.m3u8"
+        return PlaylistUtils(client).extractFromHls(masterUrl, videoNameGen = { "${prefix}Vido - (${it})" })
+    }
+
+}

multisrc/overrides/datalifeengine/frenchanime/additional.gradle

@@ -0,0 +1,12 @@
+dependencies {
+    implementation(project(':lib-dood-extractor'))
+    implementation(project(':lib-vido-extractor'))
+    implementation(project(':lib-uqload-extractor'))
+    implementation(project(':lib-vudeo-extractor'))
+    implementation(project(':lib-streamhidevid-extractor'))
+    implementation(project(':lib-upstream-extractor'))
+    implementation(project(':lib-streamvid-extractor'))
+    implementation(project(':lib-sibnet-extractor'))
+    implementation(project(':lib-okru-extractor'))
+    implementation(project(':lib-streamhub-extractor'))
+}

multisrc/overrides/datalifeengine/frenchanime/src/FrenchAnime.kt

@@ -0,0 +1,119 @@
+package eu.kanade.tachiyomi.animeextension.fr.frenchanime
+
+import eu.kanade.tachiyomi.animesource.model.SEpisode
+import eu.kanade.tachiyomi.animesource.model.Video
+import eu.kanade.tachiyomi.lib.doodextractor.DoodExtractor
+import eu.kanade.tachiyomi.lib.okruextractor.OkruExtractor
+import eu.kanade.tachiyomi.lib.sibnetextractor.SibnetExtractor
+import eu.kanade.tachiyomi.lib.streamhidevidextractor.StreamHideVidExtractor
+import eu.kanade.tachiyomi.lib.streamhubextractor.StreamHubExtractor
+import eu.kanade.tachiyomi.lib.streamvidextractor.StreamVidExtractor
+import eu.kanade.tachiyomi.lib.upstreamextractor.UpstreamExtractor
+import eu.kanade.tachiyomi.lib.uqloadextractor.UqloadExtractor
+import eu.kanade.tachiyomi.lib.vidoextractor.VidoExtractor
+import eu.kanade.tachiyomi.lib.vudeoextractor.VudeoExtractor
+import eu.kanade.tachiyomi.multisrc.datalifeengine.DataLifeEngine
+import eu.kanade.tachiyomi.network.GET
+import eu.kanade.tachiyomi.util.asJsoup
+import okhttp3.Request
+import okhttp3.Response
+import org.jsoup.nodes.Document
+import org.jsoup.nodes.Element
+import rx.Observable
+import java.lang.Exception
+
+class FrenchAnime : DataLifeEngine(
+    "French Anime",
+    "https://french-anime.com",
+    "fr",
+) {
+
+    override val categories = arrayOf(
+        Pair("<Sélectionner>", ""),
+        Pair("Animes VF", "/animes-vf/"),
+        Pair("Animes VOSTFR", "/animes-vostfr/"),
+        Pair("Films VF et VOSTFR", "/films-vf-vostfr/"),
+    )
+
+    override val genres = arrayOf(
+        Pair("<Sélectionner>", ""),
+        Pair("Action", "/genre/action/"),
+        Pair("Aventure", "/genre/aventure/"),
+        Pair("Arts martiaux", "/genre/arts-martiaux/"),
+        Pair("Combat", "/genre/combat/"),
+        Pair("Comédie", "/genre/comedie/"),
+        Pair("Drame", "/genre/drame/"),
+        Pair("Epouvante", "/genre/epouvante/"),
+        Pair("Fantastique", "/genre/fantastique/"),
+        Pair("Fantasy", "/genre/fantasy/"),
+        Pair("Mystère", "/genre/mystere/"),
+        Pair("Romance", "/genre/romance/"),
+        Pair("Shonen", "/genre/shonen/"),
+        Pair("Surnaturel", "/genre/surnaturel/"),
+        Pair("Sci-Fi", "/genre/sci-fi/"),
+        Pair("School life", "/genre/school-life/"),
+        Pair("Ninja", "/genre/ninja/"),
+        Pair("Seinen", "/genre/seinen/"),
+        Pair("Horreur", "/genre/horreur/"),
+        Pair("Tranche de vie", "/genre/tranchedevie/"),
+        Pair("Psychologique", "/genre/psychologique/"),
+    )
+
+    // ============================== Popular ===============================
+
+    override fun popularAnimeRequest(page: Int): Request = GET("$baseUrl/animes-vostfr/page/$page/")
+
+    // ============================== Episodes ==============================
+
+    override fun episodeListParse(response: Response): List<SEpisode> {
+        val document = response.asJsoup()
+        val episodeList = mutableListOf<SEpisode>()
+
+        val epsData = document.selectFirst("div.eps")?.text() ?: return emptyList()
+        epsData.split(" ").filter { it.isNotBlank() }.forEach {
+            val data = it.split("!", limit = 2)
+            val episode = SEpisode.create()
+            episode.episode_number = data[0].toFloatOrNull() ?: 0F
+            episode.name = "Episode ${data[0]}"
+            episode.url = data[1]
+            episodeList.add(episode)
+        }
+
+        return episodeList.reversed()
+    }
+
+    override fun episodeListSelector(): String = throw Exception("not used")
+
+    override fun episodeFromElement(element: Element): SEpisode = throw Exception("not used")
+
+    // ============================ Video Links =============================
+
+    override fun fetchVideoList(episode: SEpisode): Observable<List<Video>> {
+        val list = episode.url.split(",").filter { it.isNotBlank() }.parallelCatchingFlatMap {
+            with(it) {
+                when {
+                    contains("dood") -> DoodExtractor(client).videosFromUrl(this)
+                    contains("upstream") -> UpstreamExtractor(client).videosFromUrl(this)
+                    contains("vudeo") -> VudeoExtractor(client).videosFromUrl(this)
+                    contains("uqload") -> UqloadExtractor(client).videosFromUrl(this)
+                    contains("guccihide") ||
+                        contains("streamhide") -> StreamHideVidExtractor(client).videosFromUrl(this)
+                    contains("streamvid") -> StreamVidExtractor(client).videosFromUrl(this)
+                    contains("vido") -> VidoExtractor(client).videosFromUrl(this)
+                    contains("sibnet") -> SibnetExtractor(client).videosFromUrl(this)
+                    contains("ok.ru") -> OkruExtractor(client).videosFromUrl(this)
+                    contains("streamhub.gg") -> StreamHubExtractor(client).videosFromUrl(this)
+                    else -> emptyList()
+                }
+            }
+        }.sort()
+        if (list.isEmpty()) throw Exception("no player found")
+        return Observable.just(list)
+    }
+
+    override fun videoFromElement(element: Element): Video = throw Exception("Not Used")
+
+    override fun videoListSelector(): String = throw Exception("Not Used")
+
+    override fun videoUrlParse(document: Document): String = throw Exception("Not Used")
+}

multisrc/overrides/datalifeengine/wiflix/additional.gradle

@@ -0,0 +1,11 @@
+dependencies {
+    implementation(project(':lib-dood-extractor'))
+    implementation(project(':lib-vido-extractor'))
+    implementation(project(':lib-uqload-extractor'))
+    implementation(project(':lib-streamdav-extractor'))
+    // ? implementation(project(':lib-waaw1-extractor'))
+    implementation(project(':lib-vudeo-extractor'))
+    implementation(project(':lib-streamhidevid-extractor'))
+    implementation(project(':lib-upstream-extractor'))
+    implementation(project(':lib-voe-extractor'))
+}

multisrc/overrides/datalifeengine/wiflix/src/Wiflix.kt

@@ -0,0 +1,104 @@
+package eu.kanade.tachiyomi.animeextension.fr.wiflix
+
+import eu.kanade.tachiyomi.animesource.model.SEpisode
+import eu.kanade.tachiyomi.animesource.model.Video
+import eu.kanade.tachiyomi.lib.doodextractor.DoodExtractor
+import eu.kanade.tachiyomi.lib.streamdavextractor.StreamDavExtractor
+import eu.kanade.tachiyomi.lib.streamhidevidextractor.StreamHideVidExtractor
+import eu.kanade.tachiyomi.lib.upstreamextractor.UpstreamExtractor
+import eu.kanade.tachiyomi.lib.uqloadextractor.UqloadExtractor
+import eu.kanade.tachiyomi.lib.vidoextractor.VidoExtractor
+import eu.kanade.tachiyomi.lib.voeextractor.VoeExtractor
+import eu.kanade.tachiyomi.lib.vudeoextractor.VudeoExtractor
+import eu.kanade.tachiyomi.multisrc.datalifeengine.DataLifeEngine
+import eu.kanade.tachiyomi.network.GET
+import okhttp3.Request
+import okhttp3.Response
+import org.jsoup.nodes.Document
+import org.jsoup.nodes.Element
+import rx.Observable
+
+class Wiflix : DataLifeEngine(
+    "Wiflix",
+    "https://wiflix.voto",
+    "fr",
+) {
+
+    override val categories = arrayOf(
+        Pair("<Sélectionner>", ""),
+        Pair("Séries", "/serie-en-streaming/"),
+        Pair("Films", "/film-en-streaming/"),
+    )
+
+    override val genres = arrayOf(
+        Pair("<Sélectionner>", ""),
+        Pair("Action", "/film-en-streaming/action/"),
+        Pair("Animation", "/film-en-streaming/animation/"),
+        Pair("Arts Martiaux", "/film-en-streaming/arts-martiaux/"),
+        Pair("Aventure", "/film-en-streaming/aventure/"),
+        Pair("Biopic", "/film-en-streaming/biopic/"),
+        Pair("Comédie", "/film-en-streaming/comedie/"),
+        Pair("Comédie Dramatique", "/film-en-streaming/comedie-dramatique/"),
+        Pair("Épouvante Horreur", "/film-en-streaming/horreur/"),
+        Pair("Drame", "/film-en-streaming/drame/"),
+        Pair("Documentaire", "/film-en-streaming/documentaire/"),
+        Pair("Espionnage", "/film-en-streaming/espionnage/"),
+        Pair("Famille", "/film-en-streaming/famille/"),
+        Pair("Fantastique", "/film-en-streaming/fantastique/"),
+        Pair("Guerre", "/film-en-streaming/guerre/"),
+        Pair("Historique", "/film-en-streaming/historique/"),
+        Pair("Musical", "/film-en-streaming/musical/"),
+        Pair("Policier", "/film-en-streaming/policier/"),
+        Pair("Romance", "/film-en-streaming/romance/"),
+        Pair("Science-Fiction", "/film-en-streaming/science-fiction/"),
+        Pair("Spectacles", "/film-en-streaming/spectacles/"),
+        Pair("Thriller", "/film-en-streaming/thriller/"),
+        Pair("Western", "/film-en-streaming/western/"),
+    )
+
+    // ============================== Popular ===============================
+
+    override fun popularAnimeRequest(page: Int): Request = GET("$baseUrl/serie-en-streaming/page/$page/")
+
+    // ============================== Episodes ==============================
+
+    override fun episodeListSelector(): String = ".hostsblock div:has(a[href*=https])"
+
+    override fun episodeListParse(response: Response): List<SEpisode> = super.episodeListParse(response).sort()
+
+    override fun episodeFromElement(element: Element): SEpisode = SEpisode.create().apply {
+        episode_number = element.className().filter { it.isDigit() }.toFloat()
+        name = "Episode ${episode_number.toInt()}"
+        scanlator = if (element.className().contains("vf")) "VF" else "VOSTFR"
+        url = element.select("a").joinToString(",") { it.attr("href").removePrefix("/vd.php?u=") }
+    }
+
+    // ============================ Video Links =============================
+
+    override fun fetchVideoList(episode: SEpisode): Observable<List<Video>> {
+        val list = episode.url.split(",").filter { it.isNotBlank() }.parallelCatchingFlatMap {
+            with(it) {
+                when {
+                    contains("doods.pro") -> DoodExtractor(client).videosFromUrl(this)
+                    contains("vido.lol") -> VidoExtractor(client).videosFromUrl(this)
+                    contains("uqload.co") -> UqloadExtractor(client).videosFromUrl(this)
+                    contains("waaw1.tv") -> emptyList()
+                    contains("vudeo.co") -> VudeoExtractor(client).videosFromUrl(this)
+                    contains("streamvid.net") -> StreamHideVidExtractor(client).videosFromUrl(this)
+                    contains("upstream.to") -> UpstreamExtractor(client).videosFromUrl(this)
+                    contains("streamdav.com") -> StreamDavExtractor(client).videosFromUrl(this)
+                    contains("voe.sx") -> listOfNotNull(VoeExtractor(client).videoFromUrl(this))
+                    else -> emptyList()
+                }
+            }
+        }.sort()
+        if (list.isEmpty()) throw Exception("no player found")
+        return Observable.just(list)
+    }
+
+    override fun videoFromElement(element: Element): Video = throw Exception("Not Used")
+
+    override fun videoListSelector(): String = throw Exception("Not Used")
+
+    override fun videoUrlParse(document: Document): String = throw Exception("Not Used")
+}

multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/datalifeengine/DataLifeEngine.kt

@@ -1,13 +1,8 @@
-package eu.kanade.tachiyomi.animeextension.fr.frenchanime
+package eu.kanade.tachiyomi.multisrc.datalifeengine
 
 import android.app.Application
-import android.content.SharedPreferences
 import androidx.preference.ListPreference
 import androidx.preference.PreferenceScreen
-import eu.kanade.tachiyomi.animeextension.fr.frenchanime.extractors.StreamHideExtractor
-import eu.kanade.tachiyomi.animeextension.fr.frenchanime.extractors.StreamVidExtractor
-import eu.kanade.tachiyomi.animeextension.fr.frenchanime.extractors.UpstreamExtractor
-import eu.kanade.tachiyomi.animeextension.fr.frenchanime.extractors.VidoExtractor
 import eu.kanade.tachiyomi.animesource.ConfigurableAnimeSource
 import eu.kanade.tachiyomi.animesource.model.AnimeFilter
 import eu.kanade.tachiyomi.animesource.model.AnimeFilterList
@@ -15,11 +10,6 @@ import eu.kanade.tachiyomi.animesource.model.SAnime
 import eu.kanade.tachiyomi.animesource.model.SEpisode
 import eu.kanade.tachiyomi.animesource.model.Video
 import eu.kanade.tachiyomi.animesource.online.ParsedAnimeHttpSource
-import eu.kanade.tachiyomi.lib.doodextractor.DoodExtractor
-import eu.kanade.tachiyomi.lib.okruextractor.OkruExtractor
-import eu.kanade.tachiyomi.lib.sibnetextractor.SibnetExtractor
-import eu.kanade.tachiyomi.lib.uqloadextractor.UqloadExtractor
-import eu.kanade.tachiyomi.lib.vudeoextractor.VudeoExtractor
 import eu.kanade.tachiyomi.network.GET
 import eu.kanade.tachiyomi.network.POST
 import eu.kanade.tachiyomi.network.asObservableSuccess
@@ -28,7 +18,6 @@ import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.async
 import kotlinx.coroutines.awaitAll
 import kotlinx.coroutines.runBlocking
-import kotlinx.serialization.json.Json
 import okhttp3.HttpUrl.Companion.toHttpUrl
 import okhttp3.MediaType.Companion.toMediaType
 import okhttp3.OkHttpClient
@@ -40,30 +29,21 @@ import org.jsoup.nodes.Element
 import rx.Observable
 import uy.kohesive.injekt.Injekt
 import uy.kohesive.injekt.api.get
-import uy.kohesive.injekt.injectLazy
 
-class FrenchAnime : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
-
-    override val name = "French Anime"
-
-    override val baseUrl = "https://french-anime.com"
-
-    override val lang = "fr"
+abstract class DataLifeEngine(
+    override val name: String,
+    override val baseUrl: String,
+    override val lang: String,
+) : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
 
     override val supportsLatest = false
 
     override val client: OkHttpClient = network.cloudflareClient
 
-    private val json: Json by injectLazy()
-
-    private val preferences: SharedPreferences by lazy {
-        Injekt.get<Application>().getSharedPreferences("source_$id", 0x0000)
-    }
+    private val preferences by lazy { Injekt.get<Application>().getSharedPreferences("source_$id", 0x0000) }
 
     // ============================== Popular ===============================
 
-    override fun popularAnimeRequest(page: Int): Request = GET("$baseUrl/animes-vostfr/page/$page/")
-
     override fun popularAnimeSelector(): String = "div#dle-content > div.mov"
 
     override fun popularAnimeNextPageSelector(): String = "span.navigation > span:not(.nav_ext) + a"
@@ -88,6 +68,7 @@ class FrenchAnime : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
 
     // =============================== Search ===============================
 
+    // TODO: Implement the *actual* search filters from : https://${baseUrl}/index.php?do=search
     override fun searchAnimeRequest(page: Int, query: String, filters: AnimeFilterList): Request {
         val filterList = if (filters.isEmpty()) getFilterList() else filters
         val genreFilter = filterList.find { it is GenreFilter } as GenreFilter
@@ -132,54 +113,30 @@ class FrenchAnime : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
 
     // ============================== Filters ===============================
 
+    abstract val categories: Array<Pair<String, String>>
+
+    abstract val genres: Array<Pair<String, String>>
+
     override fun getFilterList(): AnimeFilterList = AnimeFilterList(
         AnimeFilter.Header("La recherche de texte ignore les filtres"),
-        SubPageFilter(),
-        GenreFilter(),
+        SubPageFilter(categories),
+        GenreFilter(genres),
     )
 
-    private class SubPageFilter : UriPartFilter(
+    private class SubPageFilter(categories: Array<Pair<String, String>>) : UriPartFilter(
         "Catégories",
-        arrayOf(
-            Pair("<Sélectionner>", ""),
-            Pair("Animes VF", "/animes-vf/"),
-            Pair("Animes VOSTFR", "/animes-vostfr/"),
-            Pair("Films VF et VOSTFR", "/films-vf-vostfr/"),
-        ),
+        categories,
     )
 
-    private class GenreFilter : UriPartFilter(
-        "Animes par genre",
-        arrayOf(
-            Pair("<Sélectionner>", ""),
-            Pair("Action", "/genre/action/"),
-            Pair("Aventure", "/genre/aventure/"),
-            Pair("Arts martiaux", "/genre/arts-martiaux/"),
-            Pair("Combat", "/genre/combat/"),
-            Pair("Comédie", "/genre/comedie/"),
-            Pair("Drame", "/genre/drame/"),
-            Pair("Epouvante", "/genre/epouvante/"),
-            Pair("Fantastique", "/genre/fantastique/"),
-            Pair("Fantasy", "/genre/fantasy/"),
-            Pair("Mystère", "/genre/mystere/"),
-            Pair("Romance", "/genre/romance/"),
-            Pair("Shonen", "/genre/shonen/"),
-            Pair("Surnaturel", "/genre/surnaturel/"),
-            Pair("Sci-Fi", "/genre/sci-fi/"),
-            Pair("School life", "/genre/school-life/"),
-            Pair("Ninja", "/genre/ninja/"),
-            Pair("Seinen", "/genre/seinen/"),
-            Pair("Horreur", "/genre/horreur/"),
-            Pair("Tranche de vie", "/genre/tranchedevie/"),
-            Pair("Psychologique", "/genre/psychologique/"),
-        ),
+    private class GenreFilter(genres: Array<Pair<String, String>>) : UriPartFilter(
+        "Genres",
+        genres,
     )
 
     private open class UriPartFilter(displayName: String, val vals: Array<Pair<String, String>>) :
         AnimeFilter.Select<String>(displayName, vals.map { it.first }.toTypedArray()) {
         fun toUriPart() = vals[state].second
     }
-
     // =========================== Anime Details ============================
 
     override fun fetchAnimeDetails(anime: SAnime): Observable<SAnime> {
@@ -194,108 +151,26 @@ class FrenchAnime : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
 
     private fun animeDetailsParse(response: Response, baseAnime: SAnime): SAnime {
         val document = response.asJsoup()
-        val anime = SAnime.create()
-
-        anime.title = baseAnime.title
-        anime.thumbnail_url = baseAnime.thumbnail_url
-        anime.description = document.selectFirst("div.mov-desc span[itemprop=description]")?.text() ?: ""
-        anime.genre = document.select("div.mov-desc span[itemprop=genre] a").joinToString(", ") { it.text() }
-        return anime
-    }
-
-    // ============================== Episodes ==============================
-
-    override fun episodeListParse(response: Response): List<SEpisode> {
-        val document = response.asJsoup()
-        val episodeList = mutableListOf<SEpisode>()
-
-        val epsData = document.selectFirst("div.eps")?.text() ?: return emptyList()
-        epsData.split(" ").filter { it.isNotBlank() }.forEach {
-            val data = it.split("!", limit = 2)
-            val episode = SEpisode.create()
-            episode.episode_number = data[0].toFloatOrNull() ?: 0F
-            episode.name = "Episode ${data[0]}"
-            episode.url = data[1]
-            episodeList.add(episode)
-        }
-
-        return episodeList.reversed()
-    }
-
-    override fun episodeListSelector(): String = throw Exception("Not used")
-
-    override fun episodeFromElement(element: Element): SEpisode = throw Exception("Not used")
-
-    // ============================ Video Links =============================
-
-    override fun fetchVideoList(episode: SEpisode): Observable<List<Video>> {
-        val videoList = mutableListOf<Video>()
-
-        episode.url.split(",").filter { it.isNotBlank() }.parallelMap { source ->
-            runCatching {
-                when {
-                    source.contains("https://dood") -> {
-                        videoList.addAll(
-                            DoodExtractor(client).videosFromUrl(source),
-                        )
-                    }
-                    source.contains("https://upstream") -> {
-                        videoList.addAll(
-                            UpstreamExtractor(client).videosFromUrl(source, headers),
-                        )
-                    }
-                    source.contains("https://vudeo") -> {
-                        videoList.addAll(
-                            VudeoExtractor(client).videosFromUrl(source),
-                        )
-                    }
-                    source.contains("https://uqload") -> {
-                        videoList.addAll(
-                            UqloadExtractor(client).videosFromUrl(source),
-                        )
-                    }
-
-                    source.contains("https://guccihide") || source.contains("https://streamhide") -> {
-                        videoList.addAll(
-                            StreamHideExtractor(client).videosFromUrl(source, headers),
-                        )
-                    }
-                    source.contains("https://streamvid") -> {
-                        videoList.addAll(
-                            StreamVidExtractor(client).videosFromUrl(source, headers),
-                        )
-                    }
-                    source.contains("https://vido") -> {
-                        videoList.addAll(
-                            VidoExtractor(client).videosFromUrl(source, headers),
-                        )
-                    }
-                    source.contains("sibnet") -> {
-                        videoList.addAll(
-                            SibnetExtractor(client).videosFromUrl(source),
-                        )
-                    }
-                    source.contains("ok.ru") -> {
-                        videoList.addAll(
-                            OkruExtractor(client).videosFromUrl(source),
-                        )
-                    }
-                    else -> {}
+        return SAnime.create().apply {
+            title = baseAnime.title
+            thumbnail_url = baseAnime.thumbnail_url
+            description = document.selectFirst("div.mov-desc span[itemprop=description]")?.text() ?: ""
+            genre = document.select("div.mov-desc span[itemprop=genre] a").joinToString(", ") {
+                it.text()
             }
         }
     }
 
-        return Observable.just(videoList.sort())
-    }
-
-    override fun videoFromElement(element: Element): Video = throw Exception("Not Used")
-
-    override fun videoListSelector(): String = throw Exception("Not Used")
-
-    override fun videoUrlParse(document: Document): String = throw Exception("Not Used")
-
     // ============================= Utilities ==============================
 
+    @JvmName("sortSEpisode")
+    fun List<SEpisode>.sort(): List<SEpisode> = this.sortedWith(
+        compareBy(
+            { it.scanlator },
+            { it.episode_number },
+        ),
+    ).reversed()
+
     override fun List<Video>.sort(): List<Video> {
         val quality = preferences.getString("preferred_quality", "720")!!
         val server = preferences.getString("preferred_server", "Upstream")!!
@@ -308,18 +183,9 @@ class FrenchAnime : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
         ).reversed()
     }
 
-    // From Dopebox
-    private fun <A, B> Iterable<A>.parallelMap(f: suspend (A) -> B): List<B> =
+    inline fun <A, B> Iterable<A>.parallelCatchingFlatMap(crossinline f: suspend (A) -> Iterable<B>): List<B> =
         runBlocking {
-            map { async(Dispatchers.Default) { f(it) } }.awaitAll()
-        }
-
-    private fun parseStatus(statusString: String): Int {
-        return when (statusString) {
-            "Currently Airing" -> SAnime.ONGOING
-            "Finished Airing" -> SAnime.COMPLETED
-            else -> SAnime.UNKNOWN
-        }
+            map { async(Dispatchers.Default) { runCatching { f(it) }.getOrElse { emptyList() } } }.awaitAll().flatten()
         }
 
     override fun setupPreferenceScreen(screen: PreferenceScreen) {

multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/datalifeengine/DataLifeEngineGenerator.kt

@@ -0,0 +1,22 @@
+package eu.kanade.tachiyomi.multisrc.datalifeengine
+
+import generator.ThemeSourceData.SingleLang
+import generator.ThemeSourceGenerator
+
+class DataLifeEngineGenerator : ThemeSourceGenerator {
+    override val themePkg = "datalifeengine"
+
+    override val themeClass = "DataLifeEngine"
+
+    override val baseVersionCode = 1
+
+    override val sources = listOf(
+        SingleLang("Wiflix", "https://wiflix.voto", "fr"),
+        SingleLang("French Anime", "https://french-anime.com", "fr", overrideVersionCode = 5),
+    )
+
+    companion object {
+        @JvmStatic
+        fun main(args: Array<String>) = DataLifeEngineGenerator().createAll()
+    }
+}

src/fr/frenchanime/AndroidManifest.xml

@@ -1,2 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<manifest />

src/fr/frenchanime/build.gradle

@@ -1,21 +0,0 @@
-apply plugin: 'com.android.application'
-apply plugin: 'kotlin-android'
-
-ext {
-    extName = 'French Anime'
-    pkgNameSuffix = 'fr.frenchanime'
-    extClass = '.FrenchAnime'
-    extVersionCode = 5
-    libVersion = '13'
-}
-
-dependencies {
-    implementation(project(':lib-vudeo-extractor'))
-    implementation(project(':lib-uqload-extractor'))
-    implementation "dev.datlag.jsunpacker:jsunpacker:1.0.1"
-    implementation(project(':lib-okru-extractor'))
-    implementation(project(':lib-sibnet-extractor'))
-    implementation(project(':lib-dood-extractor'))
-}
-
-apply from: "$rootDir/common.gradle"

src/fr/frenchanime/src/eu/kanade/tachiyomi/animeextension/fr/frenchanime/extractors/StreamHideExtractor.kt

@@ -1,49 +0,0 @@
-package eu.kanade.tachiyomi.animeextension.fr.frenchanime.extractors
-
-import dev.datlag.jsunpacker.JsUnpacker
-import eu.kanade.tachiyomi.animesource.model.Video
-import eu.kanade.tachiyomi.network.GET
-import eu.kanade.tachiyomi.util.asJsoup
-import okhttp3.Headers
-import okhttp3.HttpUrl.Companion.toHttpUrl
-import okhttp3.OkHttpClient
-
-class StreamHideExtractor(private val client: OkHttpClient) {
-    fun videosFromUrl(url: String, headers: Headers): List<Video> {
-        val videoList = mutableListOf<Video>()
-
-        val packed = client.newCall(GET(url)).execute()
-            .asJsoup().selectFirst("script:containsData(eval)")?.data() ?: return emptyList()
-        val unpacked = JsUnpacker.unpackAndCombine(packed) ?: return emptyList()
-        val masterUrl = Regex("""file: ?"(.*?)"""").find(unpacked)?.groupValues?.get(1) ?: return emptyList()
-
-        val masterHeaders = headers.newBuilder()
-            .add("Accept", "*/*")
-            .add("Host", masterUrl.toHttpUrl().host)
-            .add("Origin", "https://${url.toHttpUrl().host}")
-            .add("Referer", "https://${url.toHttpUrl().host}/")
-            .build()
-        val masterPlaylist = client.newCall(
-            GET(masterUrl, headers = masterHeaders),
-        ).execute().body.string()
-
-        val masterBase = "https://${masterUrl.toHttpUrl().host}${masterUrl.toHttpUrl().encodedPath}"
-            .substringBeforeLast("/") + "/"
-
-        masterPlaylist.substringAfter("#EXT-X-STREAM-INF:").split("#EXT-X-STREAM-INF:")
-            .forEach {
-                val quality = "StreamHide - " + it.substringAfter("RESOLUTION=").substringAfter("x").substringBefore(",") + "p "
-                val videoUrl = masterBase + it.substringAfter("\n").substringBefore("\n")
-
-                val videoHeaders = headers.newBuilder()
-                    .add("Accept", "*/*")
-                    .add("Host", videoUrl.toHttpUrl().host)
-                    .add("Origin", "https://${url.toHttpUrl().host}")
-                    .add("Referer", "https://${url.toHttpUrl().host}/")
-                    .build()
-
-                videoList.add(Video(videoUrl, quality, videoUrl, headers = videoHeaders))
-            }
-        return videoList
-    }
-}

src/fr/frenchanime/src/eu/kanade/tachiyomi/animeextension/fr/frenchanime/extractors/StreamVidExtractor.kt

@@ -1,46 +0,0 @@
-package eu.kanade.tachiyomi.animeextension.fr.frenchanime.extractors
-
-import dev.datlag.jsunpacker.JsUnpacker
-import eu.kanade.tachiyomi.animesource.model.Video
-import eu.kanade.tachiyomi.network.GET
-import eu.kanade.tachiyomi.util.asJsoup
-import okhttp3.Headers
-import okhttp3.HttpUrl.Companion.toHttpUrl
-import okhttp3.OkHttpClient
-
-class StreamVidExtractor(private val client: OkHttpClient) {
-    fun videosFromUrl(url: String, headers: Headers): List<Video> {
-        val videoList = mutableListOf<Video>()
-
-        val packed = client.newCall(GET(url)).execute()
-            .asJsoup().selectFirst("script:containsData(m3u8)")?.data() ?: return emptyList()
-        val unpacked = JsUnpacker.unpackAndCombine(packed) ?: return emptyList()
-        val masterUrl = Regex("""src: ?"(.*?)"""").find(unpacked)?.groupValues?.get(1) ?: return emptyList()
-
-        val masterHeaders = headers.newBuilder()
-            .add("Accept", "*/*")
-            .add("Host", masterUrl.toHttpUrl().host)
-            .add("Origin", "https://${url.toHttpUrl().host}")
-            .add("Referer", "https://${url.toHttpUrl().host}/")
-            .build()
-        val masterPlaylist = client.newCall(
-            GET(masterUrl, headers = masterHeaders),
-        ).execute().body.string()
-
-        masterPlaylist.substringAfter("#EXT-X-STREAM-INF:").split("#EXT-X-STREAM-INF:")
-            .forEach {
-                val quality = "StreamVid - " + it.substringAfter("RESOLUTION=").substringAfter("x").substringBefore(",") + "p "
-                val videoUrl = it.substringAfter("\n").substringBefore("\n")
-
-                val videoHeaders = headers.newBuilder()
-                    .add("Accept", "*/*")
-                    .add("Host", videoUrl.toHttpUrl().host)
-                    .add("Origin", "https://${url.toHttpUrl().host}")
-                    .add("Referer", "https://${url.toHttpUrl().host}/")
-                    .build()
-
-                videoList.add(Video(videoUrl, quality, videoUrl, headers = videoHeaders))
-            }
-        return videoList
-    }
-}

src/fr/frenchanime/src/eu/kanade/tachiyomi/animeextension/fr/frenchanime/extractors/UpstreamExtractor.kt

@@ -1,38 +0,0 @@
-package eu.kanade.tachiyomi.animeextension.fr.frenchanime.extractors
-
-import dev.datlag.jsunpacker.JsUnpacker
-import eu.kanade.tachiyomi.animesource.model.Video
-import eu.kanade.tachiyomi.network.GET
-import eu.kanade.tachiyomi.util.asJsoup
-import okhttp3.Headers
-import okhttp3.HttpUrl.Companion.toHttpUrl
-import okhttp3.OkHttpClient
-
-class UpstreamExtractor(private val client: OkHttpClient) {
-    fun videosFromUrl(url: String, headers: Headers): List<Video> {
-        try {
-            val jsE = client.newCall(GET(url)).execute().asJsoup().selectFirst("script:containsData(eval)")!!.data()
-            val masterUrl = JsUnpacker.unpackAndCombine(jsE)!!
-                .substringAfter("{file:\"").substringBefore("\"}")
-            val masterBase = masterUrl.substringBefore("master")
-            val masterPlaylist = client.newCall(GET(masterUrl)).execute().body.string()
-            val videoList = mutableListOf<Video>()
-            masterPlaylist.substringAfter("#EXT-X-STREAM-INF:").split("#EXT-X-STREAM-INF:")
-                .forEach {
-                    val quality = "Upstream - " + it.substringAfter("RESOLUTION=").substringAfter("x").substringBefore(",") + "p "
-                    val videoUrl = masterBase + it.substringAfter("\n").substringBefore("\n")
-                    val videoHeaders = headers.newBuilder()
-                        .add("Accept", "*/*")
-                        .add("Host", videoUrl.toHttpUrl().host)
-                        .add("Origin", "https://upstream.to")
-                        .add("Referer", "https://upstream.to/")
-                        .build()
-
-                    videoList.add(Video(videoUrl, quality, videoUrl, headers = videoHeaders))
-                }
-            return videoList
-        } catch (e: Exception) {
-            return emptyList()
-        }
-    }
-}

src/fr/frenchanime/src/eu/kanade/tachiyomi/animeextension/fr/frenchanime/extractors/VidoExtractor.kt

@@ -1,81 +0,0 @@
-package eu.kanade.tachiyomi.animeextension.fr.frenchanime.extractors
-
-import eu.kanade.tachiyomi.animesource.model.Video
-import eu.kanade.tachiyomi.network.GET
-import eu.kanade.tachiyomi.network.POST
-import eu.kanade.tachiyomi.util.asJsoup
-import kotlinx.serialization.decodeFromString
-import kotlinx.serialization.json.Json
-import okhttp3.Headers
-import okhttp3.HttpUrl.Companion.toHttpUrl
-import okhttp3.MediaType.Companion.toMediaType
-import okhttp3.OkHttpClient
-import okhttp3.RequestBody.Companion.toRequestBody
-
-class VidoExtractor(private val client: OkHttpClient) {
-    fun videosFromUrl(url: String, headers: Headers): List<Video> {
-        val videoList = mutableListOf<Video>()
-
-        val id = url.substringAfterLast("/").substringBefore(".html")
-        val document = client.newCall(
-            GET(url),
-        ).execute().asJsoup()
-        val postBodyValues = mutableListOf<String>()
-
-        document.select("form > input").forEach {
-            val name = it.attr("name")
-            val value = if (name == "file_code") {
-                id
-            } else {
-                it.attr("value")
-            }
-            postBodyValues.add("$name=$value")
-        }
-
-        val postBody = postBodyValues.joinToString("&").toRequestBody("application/x-www-form-urlencoded".toMediaType())
-        val postHeaders = headers.newBuilder()
-            .add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8")
-            .add("Content-Type", "application/x-www-form-urlencoded")
-            .add("Host", url.toHttpUrl().host)
-            .add("Origin", "https://${url.toHttpUrl().host}")
-            .add("Referer", url)
-            .build()
-        val postDocument = client.newCall(
-            POST("https://${url.toHttpUrl().host}/dl", body = postBody, headers = postHeaders),
-        ).execute().asJsoup()
-
-        val sourcesScript = postDocument.selectFirst("script:containsData(sources)")?.data() ?: return emptyList()
-        val sourcesString = Regex("""sources: ?(\[.*?\])""").find(sourcesScript)?.groupValues?.get(1) ?: return emptyList()
-
-        val sources = Json.decodeFromString<List<String>>(sourcesString)
-        sources.forEach { source ->
-            val masterHeaders = headers.newBuilder()
-                .add("Accept", "*/*")
-                .add("Host", source.toHttpUrl().host)
-                .add("Origin", "https://${url.toHttpUrl().host}")
-                .add("Referer", "https://${url.toHttpUrl().host}/")
-                .build()
-
-            val masterPlaylist = client.newCall(
-                GET(source, headers = masterHeaders),
-            ).execute().body.string()
-
-            masterPlaylist.substringAfter("#EXT-X-STREAM-INF:").split("#EXT-X-STREAM-INF:")
-                .forEach {
-                    val quality = "Vido - " + it.substringAfter("RESOLUTION=").substringAfter("x").substringBefore(",") + "p "
-                    val videoUrl = it.substringAfter("\n").substringBefore("\n")
-
-                    val videoHeaders = headers.newBuilder()
-                        .add("Accept", "*/*")
-                        .add("Host", videoUrl.toHttpUrl().host)
-                        .add("Origin", "https://${url.toHttpUrl().host}")
-                        .add("Referer", "https://${url.toHttpUrl().host}/")
-                        .build()
-
-                    videoList.add(Video(videoUrl, quality, videoUrl, headers = videoHeaders))
-                }
-        }
-
-        return videoList
-    }
-}