feat(lib): Add new lib for playlists and implement it for KAA (#1918)

lib/playlist-utils/build.gradle.kts

@@ -0,0 +1,18 @@
+
+plugins {
+    id("com.android.library")
+    kotlin("android")
+}
+
+android {
+    compileSdk = AndroidConfig.compileSdk
+    namespace = "eu.kanade.tachiyomi.lib.playlistutils"
+
+    defaultConfig {
+        minSdk = AndroidConfig.minSdk
+    }
+}
+
+dependencies {
+    compileOnly(libs.bundles.common)
+}

lib/playlist-utils/src/main/java/eu/kanade/tachiyomi/lib/playlistutils/PlaylistUtils.kt

@@ -0,0 +1,323 @@
+package eu.kanade.tachiyomi.lib.playlistutils
+
+import eu.kanade.tachiyomi.animesource.model.Track
+import eu.kanade.tachiyomi.animesource.model.Video
+import eu.kanade.tachiyomi.network.GET
+import eu.kanade.tachiyomi.util.asJsoup
+import okhttp3.Headers
+import okhttp3.HttpUrl.Companion.toHttpUrl
+import okhttp3.OkHttpClient
+import okhttp3.internal.commonEmptyHeaders
+
+class PlaylistUtils(private val client: OkHttpClient, private val headers: Headers = commonEmptyHeaders) {
+
+    // ================================ M3U8 ================================
+
+    /**
+     * Extracts videos from a .m3u8 file.
+     *
+     * @param playlistUrl the URL of the HLS playlist
+     * @param referer the referer header value to be sent in the HTTP request (default: "")
+     * @param masterHeaders header for the master playlist
+     * @param videoHeaders headers for each video
+     * @param videoNameGen a function that generates a custom name for each video based on its quality
+     *     - The parameter `quality` represents the quality of the video
+     *     - Returns the custom name for the video (default: identity function)
+     * @param subtitleList a list of subtitle tracks associated with the HLS playlist, non-empty values will override subtitles present in the m3u8 playlist (default: empty list)
+     * @param audioList a list of audio tracks associated with the HLS playlist, non-empty values will override audio tracks present in the m3u8 playlist (default: empty list)
+     * @return a list of Video objects
+     */
+    fun extractFromHls(
+        playlistUrl: String,
+        referer: String = "",
+        masterHeaders: Headers,
+        videoHeaders: Headers,
+        videoNameGen: (String) -> String = { quality -> quality },
+        subtitleList: List<Track> = emptyList(),
+        audioList: List<Track> = emptyList(),
+    ): List<Video> {
+        return extractFromHls(
+            playlistUrl,
+            referer,
+            { _, _ -> masterHeaders },
+            { _, _, _ -> videoHeaders },
+            videoNameGen,
+            subtitleList,
+            audioList
+        )
+    }
+
+    /**
+     * Extracts videos from a .m3u8 file.
+     *
+     * @param playlistUrl the URL of the HLS playlist
+     * @param referer the referer header value to be sent in the HTTP request (default: "")
+     * @param masterHeadersGen a function that generates headers for the master playlist request
+     *     - The first parameter `baseHeaders` represents the class constructor `headers`
+     *     - The second parameter `referer` represents the referer header value
+     *     - Returns the updated headers for the master playlist request (default: generateMasterHeaders(baseHeaders, referer))
+     * @param videoHeadersGen a function that generates headers for each video request
+     *     - The first parameter `baseHeaders` represents the class constructor `headers`
+     *     - The second parameter `referer` represents the referer header value
+     *     - The third parameter `videoUrl` represents the URL of the video
+     *     - Returns the updated headers for the video request (default: generateMasterHeaders(baseHeaders, referer))
+     * @param videoNameGen a function that generates a custom name for each video based on its quality
+     *     - The parameter `quality` represents the quality of the video
+     *     - Returns the custom name for the video (default: identity function)
+     * @param subtitleList a list of subtitle tracks associated with the HLS playlist, non-empty values will override subtitles present in the m3u8 playlist (default: empty list)
+     * @param audioList a list of audio tracks associated with the HLS playlist, non-empty values will override audio tracks present in the m3u8 playlist (default: empty list)
+     * @return a list of Video objects
+     */
+    fun extractFromHls(
+        playlistUrl: String,
+        referer: String = "",
+        masterHeadersGen: (Headers, String) -> Headers = { baseHeaders, referer ->
+            generateMasterHeaders(baseHeaders, referer)
+        },
+        videoHeadersGen: (Headers, String, String) -> Headers = { baseHeaders, referer, videoUrl ->
+            generateMasterHeaders(baseHeaders, referer)
+        },
+        videoNameGen: (String) -> String = { quality -> quality },
+        subtitleList: List<Track> = emptyList(),
+        audioList: List<Track> = emptyList(),
+    ): List<Video> {
+        val masterHeaders = masterHeadersGen(headers, referer)
+
+        val masterPlaylist = client.newCall(
+            GET(playlistUrl, headers = masterHeaders)
+        ).execute().body.string()
+
+        // Check if there isn't multiple streams available
+        if (PLAYLIST_SEPARATOR !in masterPlaylist) {
+            return listOf(
+                Video(
+                    playlistUrl, videoNameGen("Video"), playlistUrl, headers = masterHeaders, subtitleTracks = subtitleList, audioTracks = audioList
+                )
+            )
+        }
+
+        val masterBase = "https://${playlistUrl.toHttpUrl().host}${playlistUrl.toHttpUrl().encodedPath}"
+            .substringBeforeLast("/") + "/"
+
+        // Get subtitles
+        val subtitleTracks = subtitleList + SUBTITLE_REGEX.findAll(masterPlaylist).mapNotNull {
+            Track(
+                getAbsoluteUrl(it.groupValues[2], playlistUrl, masterBase) ?: return@mapNotNull null,
+                it.groupValues[1]
+            )
+        }.toList()
+
+        // Get audio tracks
+        val audioTracks = audioList + AUDIO_REGEX.findAll(masterPlaylist).mapNotNull {
+            Track(
+                getAbsoluteUrl(it.groupValues[2], playlistUrl, masterBase) ?: return@mapNotNull null,
+                it.groupValues[1]
+            )
+        }.toList()
+
+        return masterPlaylist.substringAfter(PLAYLIST_SEPARATOR).split(PLAYLIST_SEPARATOR).mapNotNull {
+            val resolution = it.substringAfter("RESOLUTION=")
+                .substringBefore("\n")
+                .substringAfter("x")
+                .substringBefore(",") + "p"
+
+            val videoUrl = it.substringAfter("\n").substringBefore("\n").let { url ->
+                getAbsoluteUrl(url, playlistUrl, masterBase)
+            } ?: return@mapNotNull null
+
+            Video(
+                videoUrl, videoNameGen(resolution), videoUrl,
+                headers = videoHeadersGen(headers, referer, videoUrl),
+                subtitleTracks = subtitleTracks, audioTracks = audioTracks
+            )
+        }
+    }
+
+    private fun getAbsoluteUrl(url: String, playlistUrl: String, masterBase: String): String? {
+        return when {
+            url.isEmpty() -> null
+            url.startsWith("http") -> url
+            url.startsWith("//") -> "https:$url"
+            url.startsWith("/") -> "https://" + playlistUrl.toHttpUrl().host + url
+            else -> masterBase + url
+        }
+    }
+
+    fun generateMasterHeaders(baseHeaders: Headers, referer: String): Headers {
+        return baseHeaders.newBuilder().apply {
+            add("Accept", "*/*")
+            if (referer.isNotEmpty()) {
+                add("Origin", "https://${referer.toHttpUrl().host}")
+                add("Referer", referer)
+            }
+        }.build()
+    }
+
+    // ================================ DASH ================================
+
+    /**
+     * Extracts video information from a DASH .mpd file.
+     *
+     * @param mpdUrl the URL of the .mpd file
+     * @param videoNameGen a function that generates a custom name for each video based on its quality
+     *     - The parameter `quality` represents the quality of the video
+     *     - Returns the custom name for the video
+     * @param mpdHeaders the headers to be sent in the HTTP request for the MPD file
+     * @param videoHeaders the headers to be sent in the HTTP requests for video segments
+     * @param referer the referer header value to be sent in the HTTP requests (default: "")
+     * @param subtitleList a list of subtitle tracks associated with the DASH file, non-empty values will override subtitles present in the m3u8 playlist (default: empty list)
+     * @param audioList a list of audio tracks associated with the DASH file, non-empty values will override audio tracks present in the m3u8 playlist (default: empty list)
+     * @return a list of Video objects
+     */
+    fun extractFromDash(
+        mpdUrl: String,
+        videoNameGen: (String) -> String,
+        mpdHeaders: Headers,
+        videoHeaders: Headers,
+        referer: String = "",
+        subtitleList: List<Track> = emptyList(),
+        audioList: List<Track> = emptyList(),
+    ): List<Video> {
+        return extractFromDash(
+            mpdUrl,
+            { videoRes, bandwidth ->
+                videoNameGen(videoRes) + " - ${formatBytes(bandwidth.toLongOrNull())}"
+            },
+            referer,
+            { _, _ -> mpdHeaders},
+            { _, _, _ -> videoHeaders},
+            subtitleList,
+            audioList
+        )
+    }
+
+    /**
+     * Extracts video information from a DASH .mpd file.
+     *
+     * @param mpdUrl the URL of the .mpd file
+     * @param videoNameGen a function that generates a custom name for each video based on its quality
+     *     - The parameter `quality` represents the quality of the video
+     *     - Returns the custom name for the video, with ` - <BANDWIDTH>` added to the end
+     * @param referer the referer header value to be sent in the HTTP requests (default: "")
+     * @param mpdHeadersGen a function that generates headers for the .mpd request
+     *     - The first parameter `baseHeaders` represents the class constructor `headers`
+     *     - The second parameter `referer` represents the referer header value
+     *     - Returns the updated headers for the .mpd request (default: generateMasterHeaders(baseHeaders, referer))
+     * @param videoHeadersGen a function that generates headers for each video request
+     *     - The first parameter `baseHeaders` represents the class constructor `headers`
+     *     - The second parameter `referer` represents the referer header value
+     *     - The third parameter `videoUrl` represents the URL of the video
+     *     - Returns the updated headers for the video segment request (default: generateMasterHeaders(baseHeaders, referer))
+     * @param subtitleList a list of subtitle tracks associated with the DASH file, non-empty values will override subtitles present in the dash file (default: empty list)
+     * @param audioList a list of audio tracks associated with the DASH file, non-empty values will override audio tracks present in the dash file (default: empty list)
+     * @return a list of Video objects
+     */
+    fun extractFromDash(
+        mpdUrl: String,
+        videoNameGen: (String) -> String,
+        referer: String = "",
+        mpdHeadersGen: (Headers, String) -> Headers = { baseHeaders, referer ->
+            generateMasterHeaders(baseHeaders, referer)
+        },
+        videoHeadersGen: (Headers, String, String) -> Headers = { baseHeaders, referer, videoUrl ->
+            generateMasterHeaders(baseHeaders, referer)
+        },
+        subtitleList: List<Track> = emptyList(),
+        audioList: List<Track> = emptyList(),
+    ): List<Video> {
+        return extractFromDash(
+            mpdUrl,
+            { videoRes, bandwidth ->
+                videoNameGen(videoRes) + " - ${formatBytes(bandwidth.toLongOrNull())}"
+            },
+            referer,
+            mpdHeadersGen,
+            videoHeadersGen,
+            subtitleList,
+            audioList
+        )
+    }
+
+    /**
+     * Extracts video information from a DASH .mpd file.
+     *
+     * @param mpdUrl the URL of the .mpd file
+     * @param videoNameGen a function that generates a custom name for each video based on its quality and bandwidth
+     *     - The parameter `quality` represents the quality of the video segment
+     *     - The parameter `bandwidth` represents the bandwidth of the video segment, in bytes
+     *     - Returns the custom name for the video
+     * @param referer the referer header value to be sent in the HTTP requests (default: "")
+     * @param mpdHeadersGen a function that generates headers for the .mpd request
+     *     - The first parameter `baseHeaders` represents the class constructor `headers`
+     *     - The second parameter `referer` represents the referer header value
+     *     - Returns the updated headers for the .mpd request (default: generateMasterHeaders(baseHeaders, referer))
+     * @param videoHeadersGen a function that generates headers for each video request
+     *     - The first parameter `baseHeaders` represents the class constructor `headers`
+     *     - The second parameter `referer` represents the referer header value
+     *     - The third parameter `videoUrl` represents the URL of the video
+     *     - Returns the updated headers for the video segment request (default: generateMasterHeaders(baseHeaders, referer))
+     * @param subtitleList a list of subtitle tracks associated with the DASH file, non-empty values will override subtitles present in the dash file (default: empty list)
+     * @param audioList a list of audio tracks associated with the DASH file, non-empty values will override audio tracks present in the dash file (default: empty list)
+     * @return a list of Video objects
+     */
+    fun extractFromDash(
+        mpdUrl: String,
+        videoNameGen: (String, String) -> String,
+        referer: String = "",
+        mpdHeadersGen: (Headers, String) -> Headers = { baseHeaders, referer ->
+            generateMasterHeaders(baseHeaders, referer)
+        },
+        videoHeadersGen: (Headers, String, String) -> Headers = { baseHeaders, referer, videoUrl ->
+            generateMasterHeaders(baseHeaders, referer)
+        },
+        subtitleList: List<Track> = emptyList(),
+        audioList: List<Track> = emptyList(),
+    ): List<Video> {
+        val mpdHeaders = mpdHeadersGen(headers, referer)
+
+        val doc = client.newCall(
+            GET(mpdUrl, headers = mpdHeaders)
+        ).execute().asJsoup()
+
+        // Get audio tracks
+        val audioTracks = audioList + doc.select("Representation[mimetype~=audio]").map { audioSrc ->
+            val bandwidth = audioSrc.attr("bandwidth").toLongOrNull()
+            Track(audioSrc.text(), formatBytes(bandwidth))
+        }
+
+        return doc.select("Representation[mimetype~=video]").map { videoSrc ->
+            val bandwidth = videoSrc.attr("bandwidth")
+            val res = videoSrc.attr("height") + "p"
+
+            Video(
+                videoSrc.text(),
+                videoNameGen(res, bandwidth),
+                videoSrc.text(),
+                audioTracks = audioTracks,
+                subtitleTracks = subtitleList,
+                headers = videoHeadersGen(headers, referer, videoSrc.text())
+            )
+        }
+    }
+
+    private fun formatBytes(bytes: Long?): String {
+        return when {
+            bytes == null -> ""
+            bytes >= 1_000_000_000 -> "%.2f GB/s".format(bytes / 1_000_000_000.0)
+            bytes >= 1_000_000 -> "%.2f MB/s".format(bytes / 1_000_000.0)
+            bytes >= 1_000 -> "%.2f KB/s".format(bytes / 1_000.0)
+            bytes > 1 -> "$bytes bytes/s"
+            bytes == 1L -> "$bytes byte/s"
+            else -> ""
+        }
+    }
+
+    // ============================= Utilities ==============================
+
+    companion object {
+        private const val PLAYLIST_SEPARATOR = "#EXT-X-STREAM-INF:"
+
+        private val SUBTITLE_REGEX by lazy { Regex("""#EXT-X-MEDIA:TYPE=SUBTITLES.*?NAME="(.*?)".*?URI="(.*?)"""") }
+        private val AUDIO_REGEX by lazy { Regex("""#EXT-X-MEDIA:TYPE=AUDIO.*?NAME="(.*?)".*?URI="(.*?)"""") }
+    }
+}

src/en/kickassanime/build.gradle

@@ -14,6 +14,7 @@ ext {
 
 dependencies {
     implementation(project(":lib-cryptoaes"))
+    implementation(project(":lib-playlist-utils"))
 }
 
 apply from: "$rootDir/common.gradle"

src/en/kickassanime/src/eu/kanade/tachiyomi/animeextension/en/kickassanime/KickAssAnime.kt

@@ -289,8 +289,8 @@ class KickAssAnime : ConfigurableAnimeSource, AnimeHttpSource() {
             compareBy(
                 { it.quality.contains(quality) },
                 { Regex("""(\d+)p""").find(it.quality)?.groupValues?.get(1)?.toIntOrNull() ?: 0 },
-                { Regex("""([\d,]+)[kMGTPE]bs""").find(it.quality)?.groupValues?.get(1)?.replace(",", ".")?.toFloatOrNull() ?: 0F },
                 { it.quality.contains(server, true) },
+                { Regex("""([\d,]+) [KMGTPE]B/s""").find(it.quality)?.groupValues?.get(1)?.replace(",", ".")?.toFloatOrNull() ?: 0F },
             ),
         ).reversed()
     }

src/en/kickassanime/src/eu/kanade/tachiyomi/animeextension/en/kickassanime/extractors/KickAssAnimeExtractor.kt

@@ -1,11 +1,11 @@
 package eu.kanade.tachiyomi.animeextension.en.kickassanime.extractors
 
-import android.annotation.SuppressLint
 import eu.kanade.tachiyomi.animeextension.en.kickassanime.dto.VideoDto
 import eu.kanade.tachiyomi.animesource.model.Track
 import eu.kanade.tachiyomi.animesource.model.Video
 import eu.kanade.tachiyomi.lib.cryptoaes.CryptoAES
 import eu.kanade.tachiyomi.lib.cryptoaes.CryptoAES.decodeHex
+import eu.kanade.tachiyomi.lib.playlistutils.PlaylistUtils
 import eu.kanade.tachiyomi.network.GET
 import kotlinx.serialization.json.Json
 import okhttp3.CacheControl
@@ -13,11 +13,7 @@ import okhttp3.Headers
 import okhttp3.HttpUrl.Companion.toHttpUrl
 import okhttp3.OkHttpClient
 import okhttp3.Response
-import org.jsoup.Jsoup
-import org.jsoup.nodes.Element
 import java.security.MessageDigest
-import java.text.CharacterIterator
-import java.text.StringCharacterIterator
 
 class KickAssAnimeExtractor(
     private val client: OkHttpClient,
@@ -106,13 +102,10 @@ class KickAssAnimeExtractor(
             Track(subUrl, language)
         }
 
-        val masterPlaylist = client.newCall(GET(videoObject.playlistUrl)).execute()
-            .body.string()
-
         return when {
             videoObject.hls.isBlank() ->
-                extractVideosFromDash(masterPlaylist, name, subtitles)
+                PlaylistUtils(client, headers).extractFromDash(videoObject.playlistUrl, videoNameGen = { res -> "$name - $res" }, subtitleList = subtitles)
-            else -> extractVideosFromHLS(masterPlaylist, name, subtitles, videoObject.playlistUrl)
+            else -> PlaylistUtils(client, headers).extractFromHls(videoObject.playlistUrl, videoNameGen = { "$name - $it" }, subtitleList = subtitles)
         }
     }
 
@@ -156,82 +149,8 @@ class KickAssAnimeExtractor(
         }
     }
 
-    private fun extractVideosFromHLS(playlist: String, prefix: String, subs: List<Track>, playlistUrl: String): List<Video> {
-        val separator = "#EXT-X-STREAM-INF"
-
-        val masterBase = "https://${playlistUrl.toHttpUrl().host}${playlistUrl.toHttpUrl().encodedPath}"
-            .substringBeforeLast("/") + "/"
-
-        // Get audio tracks
-        val audioTracks = AUDIO_REGEX.findAll(playlist).mapNotNull {
-            Track(
-                getAbsoluteUrl(it.groupValues[2], playlistUrl, masterBase) ?: return@mapNotNull null,
-                it.groupValues[1],
-            )
-        }.toList()
-
-        return playlist.substringAfter(separator).split(separator).mapNotNull {
-            val resolution = it.substringAfter("RESOLUTION=")
-                .substringBefore("\n")
-                .substringAfter("x")
-                .substringBefore(",") + "p"
-
-            val videoUrl = it.substringAfter("\n").substringBefore("\n").let { url ->
-                getAbsoluteUrl(url, playlistUrl, masterBase)
-            } ?: return@mapNotNull null
-
-            Video(videoUrl, "$prefix - $resolution", videoUrl, audioTracks = audioTracks, subtitleTracks = subs)
-        }
-    }
-
-    private fun extractVideosFromDash(playlist: String, prefix: String, subs: List<Track>): List<Video> {
-        // Parsing dash with Jsoup :YEP:
-        val document = Jsoup.parse(playlist)
-        val audioList = document.select("Representation[mimetype~=audio]").map { audioSrc ->
-            Track(audioSrc.text(), audioSrc.formatBits() ?: "audio")
-        }
-        return document.select("Representation[mimetype~=video]").map { videoSrc ->
-            Video(
-                videoSrc.text(),
-                "$prefix - ${videoSrc.attr("height")}p - ${videoSrc.formatBits()}",
-                videoSrc.text(),
-                audioTracks = audioList,
-                subtitleTracks = subs,
-            )
-        }
-    }
-
     // ============================= Utilities ==============================
-
-    companion object {
-        private val AUDIO_REGEX by lazy { Regex("""#EXT-X-MEDIA:TYPE=AUDIO.*?NAME="(.*?)".*?URI="(.*?)"""") }
-    }
-
-    private fun getAbsoluteUrl(url: String, playlistUrl: String, masterBase: String): String? {
-        return when {
-            url.isEmpty() -> null
-            url.startsWith("http") -> url
-            url.startsWith("//") -> "https:$url"
-            url.startsWith("/") -> "https://" + playlistUrl.toHttpUrl().host + url
-            else -> masterBase + url
-        }
-    }
-
     private inline fun <reified T> Response.parseAs(): T {
         return body.string().let(json::decodeFromString)
     }
-
-    @SuppressLint("DefaultLocale")
-    private fun Element.formatBits(attribute: String = "bandwidth"): String? {
-        var bits = attr(attribute).toLongOrNull() ?: 0L
-        if (-1000 < bits && bits < 1000) {
-            return "${bits}b"
-        }
-        val ci: CharacterIterator = StringCharacterIterator("kMGTPE")
-        while (bits <= -999950 || bits >= 999950) {
-            bits /= 1000
-            ci.next()
-        }
-        return java.lang.String.format("%.2f%cbs", bits / 1000.0, ci.current())
-    }
 }