feat(ar/WitAnime): Add Okru extractor + general refactoration (#1687)

src/ar/witanime/build.gradle

@@ -1,11 +1,13 @@
-apply plugin: 'com.android.application'
-apply plugin: 'kotlin-android'
+plugins {
+    alias(libs.plugins.android.application)
+    alias(libs.plugins.kotlin.android)
+}
 
 ext {
     extName = 'WIT ANIME'
     pkgNameSuffix = 'ar.witanime'
     extClass = '.WitAnime'
-    extVersionCode = 37
+    extVersionCode = 38
     libVersion = '13'
 }
 

src/ar/witanime/src/eu/kanade/tachiyomi/animeextension/ar/witanime/WitAnime.kt

@@ -14,9 +14,14 @@ import eu.kanade.tachiyomi.animesource.model.SEpisode
 import eu.kanade.tachiyomi.animesource.model.Video
 import eu.kanade.tachiyomi.animesource.online.ParsedAnimeHttpSource
 import eu.kanade.tachiyomi.lib.doodextractor.DoodExtractor
+import eu.kanade.tachiyomi.lib.okruextractor.OkruExtractor
 import eu.kanade.tachiyomi.lib.streamsbextractor.StreamSBExtractor
 import eu.kanade.tachiyomi.network.GET
 import eu.kanade.tachiyomi.util.asJsoup
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.async
+import kotlinx.coroutines.awaitAll
+import kotlinx.coroutines.runBlocking
 import okhttp3.OkHttpClient
 import okhttp3.Request
 import okhttp3.Response
@@ -38,212 +43,152 @@ class WitAnime : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
 
     override val client: OkHttpClient = network.cloudflareClient
 
+    override fun headersBuilder() = super.headersBuilder().add("Referer", baseUrl)
+
     private val preferences: SharedPreferences by lazy {
         Injekt.get<Application>().getSharedPreferences("source_$id", 0x0000)
     }
 
-    // ================================== popular ==================================
+    // ============================== Popular ===============================
+    override fun popularAnimeSelector() = "div.anime-list-content div.row div.anime-card-poster div.ehover6"
 
-    override fun popularAnimeSelector(): String = "div.anime-list-content div:nth-child(1) div.col-lg-2 div.anime-card-container"
-
-    override fun popularAnimeNextPageSelector(): String = "ul.pagination a.next"
+    override fun popularAnimeNextPageSelector() = "ul.pagination a.next"
 
     override fun popularAnimeRequest(page: Int): Request = GET("$baseUrl/قائمة-الانمي/page/$page")
 
-    override fun popularAnimeFromElement(element: Element): SAnime {
-        val anime = SAnime.create()
-        anime.setUrlWithoutDomain(element.select("div.anime-card-poster a").attr("href"))
-        anime.title = element.select("div.anime-card-poster div.ehover6 img").attr("alt")
-        anime.thumbnail_url = element.selectFirst("div.anime-card-poster div.ehover6 img")!!.attr("abs:src")
-        return anime
-    }
-
-    // ================================== episodes ==================================
-
-    override fun episodeListParse(response: Response): List<SEpisode> {
-        val episodes = mutableListOf<SEpisode>()
-        fun episodeExtract(element: Element): SEpisode {
-            val episode = SEpisode.create()
-            episode.setUrlWithoutDomain(element.attr("href"))
-            episode.name = element.text()
-            return episode
+    override fun popularAnimeFromElement(element: Element) = SAnime.create().apply {
+        setUrlWithoutDomain(element.selectFirst("a")!!.attr("href"))
+        element.selectFirst("img")!!.let {
+            title = it.attr("alt")
+            thumbnail_url = it.attr("abs:src")
         }
-        fun addEpisodes(document: Document) {
-            /*if (document.select(episodeListSelector()).isNullOrEmpty())
-                document.select("div.all-episodes ul.all-episodes-list li a").forEach { episodes.add(episodeExtract(it)) }
-            else*/
-            document.select(episodeListSelector()).map { episodes.add(episodeFromElement(it)) }
-        }
-        addEpisodes(response.asJsoup())
-        return episodes.reversed()
     }
 
-    override fun episodeListSelector() = "div.all-episodes ul.all-episodes-list li, div.ehover6 > div.episodes-card-title > h3"
+    // ============================== Episodes ==============================
+    override fun episodeListParse(response: Response) = super.episodeListParse(response).reversed()
 
-    override fun episodeFromElement(element: Element): SEpisode {
-        val episode = SEpisode.create()
-        episode.setUrlWithoutDomain(element.select("a").attr("href"))
-        episode.name = element.select("a").text()
-        val episodeNumberString = element.select("a").text().removePrefix("الحلقة ").removePrefix("الخاصة ").removePrefix("الأونا ").removePrefix("الفلم ").removePrefix("الأوفا ")
-        episode.episode_number = episodeNumberString.toFloat()
-        return episode
+    override fun episodeListSelector() = "div.ehover6 > div.episodes-card-title > h3 a"
+
+    override fun episodeFromElement(element: Element) = SEpisode.create().apply {
+        setUrlWithoutDomain(element.attr("href"))
+        name = element.text()
+        episode_number = name.substringAfterLast(" ").toFloatOrNull() ?: 0F
     }
 
-    // ================================== video urls ==================================
-
+    // ============================ Video Links =============================
     override fun videoListParse(response: Response): List<Video> {
         val document = response.asJsoup()
-        val videoList = mutableListOf<Video>()
-        var dailyMotion = true
-        document.select("ul#episode-servers li").forEach { it ->
-            val server = it.select("a").text()
-            val url = it.select("a").attr("data-ep-url")
-            when {
-                url.contains("soraplay") -> {
-                    val witAnime = "https://witanime.com/"
-                    val newHeaders = headers.newBuilder()
-                        .set("referer", witAnime)
-                        .build()
-                    val videos = SoraPlayExtractor(client).videosFromUrl(url, newHeaders)
-                    videoList.addAll(videos)
-                }
-                url.contains("yonaplay") -> {
-                    val newHeaders = headers.newBuilder().set("referer", "https://witanime.com/").build()
-                    val videos = client.newCall(GET(url, newHeaders)).execute().asJsoup()
-                    videos.select("div.OD li").forEach {
-                        val videoUrl = it.attr("onclick").substringAfter("go_to_player('").substringBefore("')")
-                        when {
-                            videoUrl.contains("soraplay") -> {
-                                val video = SoraPlayExtractor(client).videosFromUrl(videoUrl, newHeaders)
-                                videoList.addAll(video)
-                            }
-                            videoUrl.contains("dropbox") -> {
-                                videoList.add(Video(videoUrl, "Dropbox mirror", videoUrl))
-                            }
-                            videoUrl.contains("4shared") -> {
-                                val video = SharedExtractor(client).videosFromUrl(videoUrl, it.select("p").text().take(3).trim())
-                                if (video != null) videoList.add(video)
-                            }
-                            /*videoUrl.contains("drive") -> {
-                                val video = GdriveExtractor(client).getVideoList(videoUrl)
-                                videoList.addAll(video)
-                            }*/
-                        }
-                    }
-                }
-                url.contains("dood") -> {
-                    val video = DoodExtractor(client).videoFromUrl(url, "Dood mirror")
-                    if (video != null) {
-                        videoList.add(video)
-                    }
-                }
-                url.contains("4shared") -> {
-                    val video = SharedExtractor(client).videosFromUrl(url)
-                    if (video != null) videoList.add(video)
-                }
-                url.contains("sbanh") -> {
-                    val videos = StreamSBExtractor(client).videosFromUrl(url, headers)
-                    videoList.addAll(videos)
-                }
-                url.contains("dailymotion") && dailyMotion -> {
-                    val videos = DailymotionExtractor(client).videosFromUrl(url, headers)
-                    videoList.addAll(videos)
-                    dailyMotion = false
-                }
+        return document.select("ul#episode-servers li a")
+            .distinctBy { it.text().substringBefore(" -") } // remove duplicates by server name
+            .parallelMap {
+                val url = it.attr("data-ep-url")
+                runCatching { extractVideos(url) }.getOrElse { emptyList() }
+            }.flatten()
+    }
+
+    private fun extractVideos(url: String): List<Video> {
+        return when {
+            url.contains("yonaplay") -> extractFromMulti(url)
+            url.contains("soraplay") -> {
+                SoraPlayExtractor(client).videosFromUrl(url, headers)
             }
+            url.contains("dood") -> {
+                DoodExtractor(client).videoFromUrl(url, "Dood mirror")
+                    ?.let(::listOf)
+            }
+            url.contains("4shared") -> {
+                SharedExtractor(client).videosFromUrl(url)
+                    ?.let(::listOf)
+            }
+            url.contains("dropbox") -> {
+                listOf(Video(url, "Dropbox mirror", url))
+            }
+            url.contains("sbanh") -> {
+                StreamSBExtractor(client).videosFromUrl(url, headers)
+            }
+            url.contains("dailymotion") -> {
+                DailymotionExtractor(client).videosFromUrl(url, headers)
+            }
+            url.contains("ok.ru") -> {
+                OkruExtractor(client).videosFromUrl(url)
+            }
+            else -> null
+        } ?: emptyList()
+    }
+
+    private fun extractFromMulti(url: String): List<Video> {
+        val doc = client.newCall(GET(url, headers)).execute().asJsoup()
+        return doc.select("div.OD li").flatMap {
+            val videoUrl = it.attr("onclick").substringAfter("go_to_player('").substringBefore("')")
+            runCatching { extractVideos(videoUrl) }.getOrElse { emptyList() }
         }
-        return videoList
     }
 
     override fun videoListSelector() = throw Exception("not used")
-
     override fun videoFromElement(element: Element) = throw Exception("not used")
-
-    override fun List<Video>.sort(): List<Video> {
-        val quality = preferences.getString("preferred_quality", "1080p")
-        if (quality != null) {
-            val newList = mutableListOf<Video>()
-            var preferred = 0
-            for (video in this) {
-                if (video.quality == quality) {
-                    newList.add(preferred, video)
-                    preferred++
-                } else {
-                    newList.add(video)
-                }
-            }
-            return newList
-        }
-        return this
-    }
-
     override fun videoUrlParse(document: Document) = throw Exception("not used")
 
-    // ================================== search ==================================
-
-    override fun searchAnimeFromElement(element: Element): SAnime {
-        val anime = SAnime.create()
-        anime.setUrlWithoutDomain(element.select("div.anime-card-poster a").attr("href"))
-        anime.title = element.select("div.anime-card-poster div.ehover6 img").attr("alt")
-        anime.thumbnail_url = element.selectFirst("div.anime-card-poster div.ehover6 img")!!.attr("abs:src")
-        return anime
+    override fun List<Video>.sort(): List<Video> {
+        val quality = preferences.getString(PREF_QUALITY_KEY, PREF_QUALITY_DEFAULT)!!
+        return sortedWith(
+            compareBy { it.quality.contains(quality) },
+        ).reversed()
     }
 
-    override fun searchAnimeNextPageSelector(): String = "ul.pagination a.next"
+    // =============================== Search ===============================
+    override fun searchAnimeRequest(page: Int, query: String, filters: AnimeFilterList) = GET("$baseUrl/?search_param=animes&s=$query")
 
-    override fun searchAnimeSelector(): String = "div.anime-list-content div:nth-child(1) div.col-lg-2 div.anime-card-container"
-
-    override fun searchAnimeRequest(page: Int, query: String, filters: AnimeFilterList): Request = GET("$baseUrl/?search_param=animes&s=$query")
+    override fun searchAnimeFromElement(element: Element) = popularAnimeFromElement(element)
+    override fun searchAnimeNextPageSelector() = popularAnimeNextPageSelector()
+    override fun searchAnimeSelector() = popularAnimeSelector()
 
     // ================================== details ==================================
 
-    override fun animeDetailsParse(document: Document): SAnime {
-        val anime = SAnime.create()
-        val doc = if (!document.select("div.anime-page-link").isNullOrEmpty()) {
-            client.newCall(GET(document.select("div.anime-page-link a").attr("href"), headers)).execute().asJsoup()
-        } else {
-            document
-        }
-        anime.thumbnail_url = doc.selectFirst("img.thumbnail")!!.attr("src")
-        anime.title = doc.select("h1.anime-details-title").text()
-        anime.genre = doc.select("ul.anime-genres > li > a, div.anime-info > a").joinToString(", ") { it.text() }
-        anime.description = doc.select("p.anime-story").text()
-        doc.select("div.anime-info a").text()?.also { statusText ->
-            when {
-                statusText.contains("يعرض الان", true) -> anime.status = SAnime.ONGOING
-                statusText.contains("مكتمل", true) -> anime.status = SAnime.COMPLETED
-                else -> anime.status = SAnime.UNKNOWN
+    override fun animeDetailsParse(document: Document) = SAnime.create().apply {
+        val doc = document.selectFirst("div.anime-page-link a")?.let {
+            client.newCall(GET(it.attr("href"), headers)).execute().asJsoup()
+        } ?: document
+
+        thumbnail_url = doc.selectFirst("img.thumbnail")!!.attr("src")
+        title = doc.selectFirst("h1.anime-details-title")!!.text()
+        // Genres + useful info
+        genre = doc.select("ul.anime-genres > li > a, div.anime-info > a").eachText().joinToString()
+
+        description = buildString {
+            // Additional info
+            doc.select("div.anime-info").eachText().forEach {
+                append("$it\n")
+            }
+            // Description
+            doc.selectFirst("p.anime-story")?.text()?.also {
+                append("\n$it")
             }
         }
 
-        return anime
+        doc.selectFirst("div.anime-info:contains(حالة الأنمي)")?.text()?.also {
+            status = when {
+                it.contains("يعرض الان", true) -> SAnime.ONGOING
+                it.contains("مكتمل", true) -> SAnime.COMPLETED
+                else -> SAnime.UNKNOWN
+            }
+        }
     }
 
-    // ================================== latest ==================================
+    // =============================== Latest ===============================
+    override fun latestUpdatesRequest(page: Int) = GET("$baseUrl/episode/page/$page/")
 
-    override fun latestUpdatesSelector(): String = "div.anime-list-content div:nth-child(1) div.col-lg-2 div.anime-card-container"
-
-    override fun latestUpdatesNextPageSelector(): String = "ul.pagination a.next"
-
-    override fun latestUpdatesRequest(page: Int): Request = GET("$baseUrl/episode/page/$page/")
-
-    override fun latestUpdatesFromElement(element: Element): SAnime {
-        val anime = SAnime.create()
-        anime.setUrlWithoutDomain(element.select("div.anime-card-poster a").attr("href"))
-        anime.title = element.select("div.anime-card-poster div.ehover6 img").attr("alt")
-        anime.thumbnail_url = element.selectFirst("div.anime-card-poster div.ehover6 img")!!.attr("abs:src")
-        return anime
-    }
-
-    // ================================== preferences ==================================
+    override fun latestUpdatesSelector() = popularAnimeSelector()
+    override fun latestUpdatesNextPageSelector() = popularAnimeNextPageSelector()
+    override fun latestUpdatesFromElement(element: Element) = popularAnimeFromElement(element)
 
+    // =============================== Preferences ===============================
     override fun setupPreferenceScreen(screen: PreferenceScreen) {
         val videoQualityPref = ListPreference(screen.context).apply {
-            key = "preferred_quality"
-            title = "Preferred quality"
-            entries = arrayOf("1080p", "720p", "480p", "380p", "360p", "240p")
-            entryValues = arrayOf("1080", "720", "480", "380", "360", "240")
-            setDefaultValue("1080")
+            key = PREF_QUALITY_KEY
+            title = PREF_QUALITY_TITLE
+            entries = PREF_QUALITY_ENTRIES
+            entryValues = PREF_QUALITY_VALUES
+            setDefaultValue(PREF_QUALITY_DEFAULT)
             summary = "%s"
 
             setOnPreferenceChangeListener { _, newValue ->
@@ -255,4 +200,20 @@ class WitAnime : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
         }
         screen.addPreference(videoQualityPref)
     }
+
+    // ============================= Utilities ==============================
+    private inline fun <A, B> Iterable<A>.parallelMap(crossinline f: suspend (A) -> B): List<B> =
+        runBlocking {
+            map { async(Dispatchers.Default) { f(it) } }.awaitAll()
+        }
+
+    companion object {
+        private const val PREF_QUALITY_KEY = "preferred_quality"
+        private const val PREF_QUALITY_TITLE = "Preferred quality"
+        private const val PREF_QUALITY_DEFAULT = "1080"
+        private val PREF_QUALITY_ENTRIES = arrayOf("1080p", "720p", "480p", "380p", "360p", "240p")
+        private val PREF_QUALITY_VALUES by lazy {
+            PREF_QUALITY_ENTRIES.map { it.substringBefore("p") }.toTypedArray()
+        }
+    }
 }

src/ar/witanime/src/eu/kanade/tachiyomi/animeextension/ar/witanime/extractors/DailymotionExtractor.kt

@@ -2,22 +2,21 @@ package eu.kanade.tachiyomi.animeextension.ar.witanime.extractors
 
 import eu.kanade.tachiyomi.animesource.model.Video
 import eu.kanade.tachiyomi.network.GET
-import eu.kanade.tachiyomi.util.asJsoup
 import okhttp3.Headers
 import okhttp3.OkHttpClient
 
 class DailymotionExtractor(private val client: OkHttpClient) {
-
-    fun videosFromUrl(url: String, newHeaders: Headers): List<Video> {
+    fun videosFromUrl(url: String, headers: Headers): List<Video> {
         val id = url.split("/").last()
-        val document = client.newCall(GET("https://www.dailymotion.com/player/metadata/video/$id", newHeaders)).execute().asJsoup()
-        val streamURL = document.body().toString().substringAfter("{\"type\":\"application\\/x-mpegURL\",\"url\":\"").substringBefore("\"").replace("\\", "")
-        val sources = client.newCall(GET(streamURL)).execute().asJsoup().body().toString()
-        val videoList = mutableListOf<Video>()
-        val videos = Regex("#EXT(?:.*?)NAME=\"(.*?)\",PROGRESSIVE-URI=\"(.*?)\"").findAll(sources).map {
-            Video(it.groupValues[2], "Dailymotion: ${it.groupValues[1]}", it.groupValues[2])
-        }
-        videoList.addAll(videos)
-        return videoList
+        val request = GET("https://www.dailymotion.com/player/metadata/video/$id", headers)
+        val body = client.newCall(request).execute().body.string()
+        val streamURL = body.substringAfter("{\"type\":\"application\\/x-mpegURL\",\"url\":\"")
+            .substringBefore("\"")
+            .replace("\\", "")
+
+        val sources = client.newCall(GET(streamURL)).execute().body.string()
+        return Regex("#EXT(?:.*?)NAME=\"(.*?)\",PROGRESSIVE-URI=\"(.*?)\"").findAll(sources).map {
+            Video(it.groupValues[2], "Dailymotion: ${it.groupValues[1]}p", it.groupValues[2])
+        }.toList()
     }
 }

src/ar/witanime/src/eu/kanade/tachiyomi/animeextension/ar/witanime/extractors/SoraPlayExtractor.kt

@@ -7,23 +7,15 @@ import okhttp3.Headers
 import okhttp3.OkHttpClient
 
 class SoraPlayExtractor(private val client: OkHttpClient) {
+    fun videosFromUrl(url: String, headers: Headers): List<Video> {
+        val doc = client.newCall(GET(url)).execute().asJsoup()
+        val script = doc.selectFirst("script:containsData(sources)")!!
+        val data = script.data().substringAfter("sources: [").substringBefore("],")
 
-    fun videosFromUrl(url: String, newHeaders: Headers): List<Video> {
-        val document = client.newCall(GET(url, newHeaders)).execute().asJsoup()
-
-        // val videoList = mutableListOf<Video>()
-        /*val script = element.select("script")
-            .firstOrNull { it.data().contains("sources:") }*/
-
-        val data = document.data().substringAfter("sources: [").substringBefore("],")
-        val sources = data.split("\"file\":\"").drop(1)
-        val videoList = mutableListOf<Video>()
-        for (source in sources) {
+        return data.split("\"file\":\"").drop(1).map { source ->
             val src = source.substringBefore("\"")
-            val quality = "Soraplay: " + source.substringAfter("\"label\":\"").substringBefore("\"") // .substringAfter("format: '")
-            val video = Video(src, quality, src, headers = newHeaders)
-            videoList.add(video)
+            val quality = "Soraplay: " + source.substringAfter("\"label\":\"").substringBefore("\"")
+            Video(src, quality, src, headers = headers)
         }
-        return videoList
     }
 }