New extension: Myanime (#1501)

src/en/myanime/AndroidManifest.xml

@@ -0,0 +1,2 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest package="eu.kanade.tachiyomi.animeextension" />

src/en/myanime/build.gradle

@@ -0,0 +1,18 @@
+apply plugin: 'com.android.application'
+apply plugin: 'kotlin-android'
+apply plugin: 'kotlinx-serialization'
+
+ext {
+    extName = 'Myanime'
+    pkgNameSuffix = 'en.myanime'
+    extClass = '.Myanime'
+    extVersionCode = 1
+    libVersion = '13'
+}
+
+dependencies {
+    implementation(project(':lib-okru-extractor'))
+}
+
+
+apply from: "$rootDir/common.gradle"

src/en/myanime/src/eu/kanade/tachiyomi/animeextension/en/myanime/Myanime.kt

@@ -0,0 +1,304 @@
+package eu.kanade.tachiyomi.animeextension.en.myanime
+
+import android.app.Application
+import android.content.SharedPreferences
+import androidx.preference.ListPreference
+import androidx.preference.PreferenceScreen
+import eu.kanade.tachiyomi.animeextension.en.myanime.extractors.DailymotionExtractor
+import eu.kanade.tachiyomi.animeextension.en.myanime.extractors.YouTubeExtractor
+import eu.kanade.tachiyomi.animesource.ConfigurableAnimeSource
+import eu.kanade.tachiyomi.animesource.model.AnimeFilter
+import eu.kanade.tachiyomi.animesource.model.AnimeFilterList
+import eu.kanade.tachiyomi.animesource.model.SAnime
+import eu.kanade.tachiyomi.animesource.model.SEpisode
+import eu.kanade.tachiyomi.animesource.model.Video
+import eu.kanade.tachiyomi.animesource.online.ParsedAnimeHttpSource
+import eu.kanade.tachiyomi.lib.okruextractor.OkruExtractor
+import eu.kanade.tachiyomi.network.GET
+import eu.kanade.tachiyomi.util.asJsoup
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.async
+import kotlinx.coroutines.awaitAll
+import kotlinx.coroutines.runBlocking
+import kotlinx.serialization.json.Json
+import okhttp3.HttpUrl.Companion.toHttpUrl
+import okhttp3.OkHttpClient
+import okhttp3.Request
+import okhttp3.Response
+import org.jsoup.nodes.Document
+import org.jsoup.nodes.Element
+import rx.Observable
+import uy.kohesive.injekt.Injekt
+import uy.kohesive.injekt.api.get
+import uy.kohesive.injekt.injectLazy
+import java.text.SimpleDateFormat
+import java.util.Locale
+
+class Myanime : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
+
+    override val name = "Myanime"
+
+    override val baseUrl = "https://myanime.live"
+
+    override val lang = "en"
+
+    override val supportsLatest = true
+
+    override val client: OkHttpClient = network.cloudflareClient
+
+    private var postBody = ""
+
+    private var postHeaders = headers.newBuilder()
+
+    private val json: Json by injectLazy()
+
+    private val preferences: SharedPreferences by lazy {
+        Injekt.get<Application>().getSharedPreferences("source_$id", 0x0000)
+    }
+
+    companion object {
+        private val DateFormatter by lazy {
+            SimpleDateFormat("d MMMM yyyy", Locale.ENGLISH)
+        }
+    }
+
+    // ============================== Popular ===============================
+
+    override fun popularAnimeRequest(page: Int): Request {
+        return GET("$baseUrl/category/donghua-list/page/$page/")
+    }
+
+    override fun popularAnimeSelector(): String = "main#main > article.post"
+
+    override fun popularAnimeNextPageSelector(): String = "script:containsData(infiniteScroll)"
+
+    override fun popularAnimeFromElement(element: Element): SAnime {
+        return SAnime.create().apply {
+            setUrlWithoutDomain(element.selectFirst("h2.entry-header-title > a")!!.attr("href").toHttpUrl().encodedPath)
+            thumbnail_url = element.selectFirst("img[src]")?.attr("src") ?: ""
+            title = element.selectFirst("h2.entry-header-title > a")!!.text().removePrefix("Playlist ")
+        }
+    }
+
+    // =============================== Latest ===============================
+
+    override fun latestUpdatesRequest(page: Int): Request = GET("$baseUrl/page/$page/")
+
+    override fun latestUpdatesSelector(): String = popularAnimeSelector()
+
+    override fun latestUpdatesNextPageSelector(): String = popularAnimeNextPageSelector()
+
+    override fun latestUpdatesFromElement(element: Element): SAnime {
+        return SAnime.create().apply {
+            setUrlWithoutDomain(element.selectFirst("h2.entry-header-title > a")!!.attr("href").toHttpUrl().encodedPath)
+            thumbnail_url = element.selectFirst("img[src]")?.attr("src") ?: ""
+            title = element.selectFirst("h2.entry-header-title > a")!!.text()
+                .substringBefore(" Episode")
+                .substringBefore(" episode")
+        }
+    }
+
+    // =============================== Search ===============================
+
+    override fun searchAnimeRequest(page: Int, query: String, filters: AnimeFilterList): Request {
+        val filterList = if (filters.isEmpty()) getFilterList() else filters
+        val subPageFilter = filterList.find { it is SubPageFilter } as SubPageFilter
+        val cleanQuery = query.replace(" ", "+")
+
+        return when {
+            query.isNotBlank() -> GET("$baseUrl/page/$page/?s=$cleanQuery", headers)
+            subPageFilter.state != 0 -> GET("$baseUrl${subPageFilter.toUriPart()}page/$page/")
+            else -> popularAnimeRequest(page)
+        }
+    }
+
+    override fun searchAnimeSelector(): String = popularAnimeSelector()
+
+    override fun searchAnimeNextPageSelector(): String = popularAnimeNextPageSelector()
+
+    override fun searchAnimeFromElement(element: Element): SAnime = latestUpdatesFromElement(element)
+
+    // ============================== Filters ===============================
+
+    override fun getFilterList(): AnimeFilterList = AnimeFilterList(
+        AnimeFilter.Header("Text search ignores filters"),
+        SubPageFilter(),
+    )
+
+    private class SubPageFilter : UriPartFilter(
+        "Sup-page",
+        arrayOf(
+            Pair("<select>", ""),
+            Pair("izfanmade", "/category/anime/"),
+        ),
+    )
+
+    private open class UriPartFilter(displayName: String, val vals: Array<Pair<String, String>>) :
+        AnimeFilter.Select<String>(displayName, vals.map { it.first }.toTypedArray()) {
+        fun toUriPart() = vals[state].second
+    }
+
+    // =========================== Anime Details ============================
+
+    override fun fetchAnimeDetails(anime: SAnime): Observable<SAnime> {
+        return Observable.just(anime)
+    }
+
+    override fun animeDetailsParse(document: Document): SAnime = throw Exception("Not used")
+
+    // ============================== Episodes ==============================
+
+    override fun episodeListParse(response: Response): List<SEpisode> {
+        val document = response.asJsoup()
+        val paths = response.request.url.encodedPathSegments
+        val itemName = paths[paths.size - 2]
+        val episodeList = mutableListOf<SEpisode>()
+
+        if (itemName.startsWith("playlist-")) {
+            episodeList.addAll(
+                document.select(
+                    "div.dpt-wrapper > div.dpt-entry",
+                ).map {
+                    val a = it.selectFirst("a.dpt-permalink")!!
+                    SEpisode.create().apply {
+                        name = a.text()
+                        episode_number = a.text().substringAfter("pisode ").substringBefore(" ").toFloatOrNull() ?: 0F
+                        setUrlWithoutDomain(a.attr("href").toHttpUrl().encodedPath)
+                    }
+                },
+            )
+        } else if (document.selectFirst("a:contains(All Episodes)[href]") != null) {
+            val url = document.selectFirst("a:contains(All Episodes)[href]")!!.attr("href")
+            episodeList.addAll(
+                episodeListParse(client.newCall(GET(url)).execute()),
+            )
+        } else if (paths.first() == "tag") {
+            var page = 1
+            var infiniteScroll = true
+
+            while (infiniteScroll) {
+                val epDocument = client.newCall(
+                    GET("${response.request.url}page/$page/"),
+                ).execute().asJsoup()
+
+                epDocument.select("main#main > article.post").forEach {
+                    val a = it.selectFirst("h2.entry-header-title > a")!!
+                    val episode = SEpisode.create()
+
+                    episode.name = a.text()
+                    episode.episode_number = a.text().substringAfter("pisode ").substringBefore(" ").toFloatOrNull() ?: 0F
+                    episode.setUrlWithoutDomain(a.attr("href").toHttpUrl().encodedPath)
+                    episodeList.add(episode)
+                }
+
+                infiniteScroll = epDocument.selectFirst("script:containsData(infiniteScroll)") != null
+                page++
+            }
+        } else if (document.selectFirst("iframe.youtube-player[src]") != null) {
+            val episode = SEpisode.create()
+
+            episode.name = document.selectFirst("title")!!.text()
+            episode.episode_number = 0F
+            episode.setUrlWithoutDomain(response.request.url.encodedPath)
+            episodeList.add(episode)
+        }
+
+        return episodeList
+    }
+
+    override fun episodeListSelector(): String = "div#episodes-tab-pane > div.row > div > div.card"
+
+    override fun episodeFromElement(element: Element): SEpisode = throw Exception("Not used")
+
+    // ============================ Video Links =============================
+
+    override fun videoListParse(response: Response): List<Video> {
+        val document = response.asJsoup()
+        val videoList = mutableListOf<Video>()
+
+        videoList.addAll(
+            document.select(videoListSelector()).parallelMap { element ->
+                runCatching {
+                    val url = element.attr("src")
+                        .replace("""^\/\/""".toRegex(), "https://")
+
+                    when {
+                        url.contains("dailymotion") -> {
+                            DailymotionExtractor(client).videosFromUrl(url)
+                        }
+                        url.contains("ok.ru") -> {
+                            OkruExtractor(client).videosFromUrl(url)
+                        }
+                        url.contains("youtube.com") -> {
+                            YouTubeExtractor(client).videosFromUrl(url, "YouTube - ")
+                        }
+                        else -> null
+                    }
+                }.getOrNull()
+            }.filterNotNull().flatten(),
+        )
+
+        return videoList
+    }
+
+    override fun videoFromElement(element: Element): Video = throw Exception("Not Used")
+
+    override fun videoListSelector(): String = "div.entry-content iframe[src]"
+
+    override fun videoUrlParse(document: Document): String = throw Exception("Not Used")
+
+    // ============================= Utilities ==============================
+
+    override fun List<Video>.sort(): List<Video> {
+        val quality = preferences.getString("preferred_quality", "1080")!!
+        val server = preferences.getString("preferred_server", "dailymotion")!!
+
+        return this.sortedWith(
+            compareBy(
+                { it.quality.contains(quality, true) },
+                { it.quality.contains(server, true) },
+            ),
+        ).reversed()
+    }
+
+    // From Dopebox
+    private fun <A, B> Iterable<A>.parallelMap(f: suspend (A) -> B): List<B> =
+        runBlocking {
+            map { async(Dispatchers.Default) { f(it) } }.awaitAll()
+        }
+
+    override fun setupPreferenceScreen(screen: PreferenceScreen) {
+        val videoQualityPref = ListPreference(screen.context).apply {
+            key = "preferred_quality"
+            title = "Preferred quality"
+            entries = arrayOf("1080p", "720p", "480p", "360p")
+            entryValues = arrayOf("1080", "720", "480", "360")
+            setDefaultValue("1080")
+            summary = "%s"
+
+            setOnPreferenceChangeListener { _, newValue ->
+                val selected = newValue as String
+                val index = findIndexOfValue(selected)
+                val entry = entryValues[index] as String
+                preferences.edit().putString(key, entry).commit()
+            }
+        }
+        val videoServerPref = ListPreference(screen.context).apply {
+            key = "preferred_server"
+            title = "Preferred server"
+            entries = arrayOf("YouTube", "Dailymotion", "ok.ru")
+            entryValues = arrayOf("youtube", "dailymotion", "okru")
+            setDefaultValue("dailymotion")
+            summary = "%s"
+
+            setOnPreferenceChangeListener { _, newValue ->
+                val selected = newValue as String
+                val index = findIndexOfValue(selected)
+                val entry = entryValues[index] as String
+                preferences.edit().putString(key, entry).commit()
+            }
+        }
+        screen.addPreference(videoQualityPref)
+        screen.addPreference(videoServerPref)
+    }
+}

src/en/myanime/src/eu/kanade/tachiyomi/animeextension/en/myanime/extractors/DailymotionExtractor.kt

@@ -0,0 +1,85 @@
+package eu.kanade.tachiyomi.animeextension.en.myanime.extractors
+
+import eu.kanade.tachiyomi.animesource.model.Track
+import eu.kanade.tachiyomi.animesource.model.Video
+import eu.kanade.tachiyomi.network.GET
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.decodeFromString
+import kotlinx.serialization.json.Json
+import kotlinx.serialization.json.JsonElement
+import okhttp3.HttpUrl.Companion.toHttpUrl
+import okhttp3.OkHttpClient
+import uy.kohesive.injekt.injectLazy
+
+@Serializable
+data class DailyQuality(
+    val qualities: Auto,
+    val subtitles: Subtitle? = null,
+) {
+    @Serializable
+    data class Auto(
+        val auto: List<Item>,
+    ) {
+        @Serializable
+        data class Item(
+            val type: String,
+            val url: String,
+        )
+    }
+
+//    @Serializable
+//    data class SubtitleObject(
+//        val label: String,
+//        val urls: List<String>,
+//    )
+
+    @Serializable
+    data class Subtitle(
+        // data can be either an empty list, or `Map<String, SubtitleObject>`
+        val data: JsonElement? = null,
+    )
+}
+
+class DailymotionExtractor(private val client: OkHttpClient) {
+
+    private val json: Json by injectLazy()
+
+    fun videosFromUrl(url: String, prefix: String = "Dailymotion - "): List<Video> {
+        val videoList = mutableListOf<Video>()
+        val htmlString = client.newCall(GET(url)).execute().body.string()
+
+        val internalData = htmlString.substringAfter("\"dmInternalData\":").substringBefore("</script>")
+        val ts = internalData.substringAfter("\"ts\":").substringBefore(",")
+        val v1st = internalData.substringAfter("\"v1st\":\"").substringBefore("\",")
+
+        val jsonUrl = "https://www.dailymotion.com/player/metadata/video${url.toHttpUrl().encodedPath}?locale=en-US&dmV1st=$v1st&dmTs=$ts&is_native_app=0"
+        val parsed = json.decodeFromString<DailyQuality>(
+            client.newCall(GET(jsonUrl))
+                .execute().body.string(),
+        )
+        val subtitleList = mutableListOf<Track>()
+//        if (parsed.subtitles != null) {
+//            if (parsed.subtitles.data.toString() != "[]") {
+//
+//            }
+//        }
+
+        val masterUrl = parsed.qualities.auto.first().url
+
+        val masterPlaylist = client.newCall(GET(masterUrl)).execute().body.string()
+
+        val separator = "#EXT-X-STREAM-INF"
+        masterPlaylist.substringAfter(separator).split(separator).map {
+            val quality = it.substringAfter("RESOLUTION=").substringAfter("x").substringBefore(",NAME") + "p"
+            var videoUrl = it.substringAfter("\n").substringBefore("\n")
+
+            try {
+                videoList.add(Video(videoUrl, prefix + quality, videoUrl, subtitleTracks = subtitleList))
+            } catch (a: Exception) {
+                videoList.add(Video(videoUrl, prefix + quality, videoUrl))
+            }
+        }
+
+        return videoList
+    }
+}

src/en/myanime/src/eu/kanade/tachiyomi/animeextension/en/myanime/extractors/YouTubeExtractor.kt

@@ -0,0 +1,181 @@
+package eu.kanade.tachiyomi.animeextension.en.myanime.extractors
+
+import android.annotation.SuppressLint
+import eu.kanade.tachiyomi.animesource.model.Track
+import eu.kanade.tachiyomi.animesource.model.Video
+import eu.kanade.tachiyomi.network.GET
+import eu.kanade.tachiyomi.network.POST
+import eu.kanade.tachiyomi.util.asJsoup
+import kotlinx.serialization.decodeFromString
+import kotlinx.serialization.json.Json
+import kotlinx.serialization.json.JsonObject
+import kotlinx.serialization.json.jsonArray
+import kotlinx.serialization.json.jsonObject
+import kotlinx.serialization.json.jsonPrimitive
+import kotlinx.serialization.json.long
+import okhttp3.Headers
+import okhttp3.MediaType.Companion.toMediaType
+import okhttp3.OkHttpClient
+import okhttp3.RequestBody.Companion.toRequestBody
+import uy.kohesive.injekt.injectLazy
+import java.text.CharacterIterator
+import java.text.StringCharacterIterator
+
+class YouTubeExtractor(private val client: OkHttpClient) {
+
+    private val json: Json by injectLazy()
+
+    fun videosFromUrl(url: String, prefix: String): List<Video> {
+        // Ported from https://github.com/dermasmid/scrapetube/blob/master/scrapetube/scrapetube.py
+        // TODO: Make code prettier
+        // GET KEY
+        var ytcfgString = ""
+        val videoId = url.substringAfter("/embed/").substringBefore("?")
+
+        val document = client.newCall(
+            GET(url.replace("/embed/", "/watch?v=")),
+        ).execute().asJsoup()
+
+        for (element in document.select("script")) {
+            val scriptData = element.data()
+            if (scriptData.startsWith("(function() {window.ytplayer={};")) {
+                ytcfgString = scriptData
+            }
+        }
+
+        val apiKey = getKey(ytcfgString, "innertubeApiKey")
+
+        val playerUrl = "https://www.youtube.com/youtubei/v1/player?key=$apiKey&prettyPrint=false"
+
+        val body = """
+            {
+               "context":{
+                  "client":{
+                     "clientName":"ANDROID",
+                     "clientVersion":"17.31.35",
+                     "androidSdkVersion":30,
+                     "userAgent":"com.google.android.youtube/17.31.35 (Linux; U; Android 11) gzip",
+                     "hl":"en",
+                     "timeZone":"UTC",
+                     "utcOffsetMinutes":0
+                  }
+               },
+               "videoId":"$videoId",
+               "params":"8AEB",
+               "playbackContext":{
+                  "contentPlaybackContext":{
+                     "html5Preference":"HTML5_PREF_WANTS"
+                  }
+               },
+               "contentCheckOk":true,
+               "racyCheckOk":true
+            }
+        """.trimIndent().toRequestBody("application/json".toMediaType())
+
+        val headers = Headers.headersOf(
+            "X-YouTube-Client-Name", "3",
+            "X-YouTube-Client-Version", "17.31.35",
+            "Origin", "https://www.youtube.com",
+            "User-Agent", "com.google.android.youtube/17.31.35 (Linux; U; Android 11) gzip",
+            "content-type", "application/json",
+        )
+
+        val postResponse = client.newCall(
+            POST(playerUrl, headers = headers, body = body),
+        ).execute()
+
+        val responseObject = json.decodeFromString<JsonObject>(postResponse.body.string())
+        val videoList = mutableListOf<Video>()
+
+        val formats = responseObject["streamingData"]!!
+            .jsonObject["adaptiveFormats"]!!
+            .jsonArray
+
+        val audioTracks = mutableListOf<Track>()
+        val subtitleTracks = mutableListOf<Track>()
+
+        // Get Audio
+        for (format in formats) {
+            if (format.jsonObject["mimeType"]!!.jsonPrimitive.content.startsWith("audio/webm")) {
+                try {
+                    audioTracks.add(
+                        Track(
+                            format.jsonObject["url"]!!.jsonPrimitive.content,
+                            format.jsonObject["audioQuality"]!!.jsonPrimitive.content +
+                                " (${formatBits(format.jsonObject["averageBitrate"]!!.jsonPrimitive.long)}ps)",
+                        ),
+                    )
+                } catch (a: Exception) { }
+            }
+        }
+
+        // Get Subtitles
+        if (responseObject.containsKey("captions")) {
+            val captionTracks = responseObject["captions"]!!
+                .jsonObject["playerCaptionsTracklistRenderer"]!!
+                .jsonObject["captionTracks"]!!
+                .jsonArray
+
+            for (caption in captionTracks) {
+                val captionJson = caption.jsonObject
+                try {
+                    subtitleTracks.add(
+                        Track(
+                            // TODO: Would replacing srv3 with vtt work for every video?
+                            captionJson["baseUrl"]!!.jsonPrimitive.content.replace("srv3", "vtt"),
+                            captionJson["name"]!!.jsonObject["runs"]!!.jsonArray[0].jsonObject["text"]!!.jsonPrimitive.content,
+                        ),
+                    )
+                } catch (a: Exception) { }
+            }
+        }
+
+        // List formats
+        for (format in formats) {
+            val mimeType = format.jsonObject["mimeType"]!!.jsonPrimitive.content
+            if (mimeType.startsWith("video/mp4")) {
+                videoList.add(
+                    try {
+                        Video(
+                            format.jsonObject["url"]!!.jsonPrimitive.content,
+                            prefix + format.jsonObject["qualityLabel"]!!.jsonPrimitive.content +
+                                " (${mimeType.substringAfter("codecs=\"").substringBefore("\"")})",
+                            format.jsonObject["url"]!!.jsonPrimitive.content,
+                            audioTracks = audioTracks,
+                            subtitleTracks = subtitleTracks,
+                        )
+                    } catch (a: Exception) {
+                        Video(
+                            format.jsonObject["url"]!!.jsonPrimitive.content,
+                            prefix + format.jsonObject["qualityLabel"]!!.jsonPrimitive.content +
+                                " (${mimeType.substringAfter("codecs=\"").substringBefore("\"")})",
+                            format.jsonObject["url"]!!.jsonPrimitive.content,
+                        )
+                    },
+
+                )
+            }
+        }
+
+        return videoList
+    }
+
+    fun getKey(string: String, key: String): String {
+        var pattern = Regex("\"$key\":\"(.*?)\"")
+        return pattern.find(string)?.groupValues?.get(1) ?: ""
+    }
+
+    @SuppressLint("DefaultLocale")
+    fun formatBits(bits: Long): String? {
+        var bits = bits
+        if (-1000 < bits && bits < 1000) {
+            return "${bits}b"
+        }
+        val ci: CharacterIterator = StringCharacterIterator("kMGTPE")
+        while (bits <= -999950 || bits >= 999950) {
+            bits /= 1000
+            ci.next()
+        }
+        return java.lang.String.format("%.0f%cb", bits / 1000.0, ci.current())
+    }
+}