INSANEMODE/aniyomi-extensions-backup
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(all/javguru): parallelMap and new extractor (#1953)

Browse Source
This commit is contained in:
AwkwardPeak7
2023-07-24 19:19:14 +05:00
committed by GitHub
parent 3a58f5b620
commit 9c9445d8b5
4 changed files with 146 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/all/javguru/build.gradle
View File

@ -7,7 +7,7 @@ ext {
extName = 'Jav Guru'
pkgNameSuffix = 'all.javguru'
extClass = '.JavGuru'
extVersionCode = 2
extVersionCode = 3
containsNsfw = true
}

182
src/all/javguru/src/eu/kanade/tachiyomi/animeextension/all/javguru/JavGuru.kt
View File

@ -1,6 +1,7 @@
package eu.kanade.tachiyomi.animeextension.all.javguru
import android.util.Base64
import eu.kanade.tachiyomi.animeextension.all.javguru.extractors.EmTurboExtractor
import eu.kanade.tachiyomi.animeextension.all.javguru.extractors.MaxStreamExtractor
import eu.kanade.tachiyomi.animesource.model.AnimeFilterList
import eu.kanade.tachiyomi.animesource.model.AnimesPage
@ -17,6 +18,10 @@ import eu.kanade.tachiyomi.network.asObservable
import eu.kanade.tachiyomi.network.asObservableSuccess
import eu.kanade.tachiyomi.network.interceptor.rateLimit
import eu.kanade.tachiyomi.util.asJsoup
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.async
import kotlinx.coroutines.awaitAll
import kotlinx.coroutines.runBlocking
import okhttp3.Call
import okhttp3.HttpUrl.Companion.toHttpUrl
import okhttp3.HttpUrl.Companion.toHttpUrlOrNull
@ -40,29 +45,13 @@ class JavGuru : AnimeHttpSource() {
.rateLimit(2)
.build()
private val noRedirectClient = client.newBuilder()
.followRedirects(false)
.build()
override fun headersBuilder() = super.headersBuilder()
.add("Referer", "$baseUrl/")
private val streamSbExtractor: StreamSBExtractor by lazy {
StreamSBExtractor(client)
}
private val streamTapeExtractor: StreamTapeExtractor by lazy {
StreamTapeExtractor(client)
}
private val doodExtractor: DoodExtractor by lazy {
DoodExtractor(client)
}
private val mixDropExtractor: MixDropExtractor by lazy {
MixDropExtractor(client)
}
private val maxStreamExtractor: MaxStreamExtractor by lazy {
MaxStreamExtractor(client)
}
private lateinit var popularElements: Elements
override fun fetchPopularAnime(page: Int): Observable<AnimesPage> {
@ -121,12 +110,12 @@ class JavGuru : AnimeHttpSource() {
}
val page = document.location()
.substringBeforeLast("/").toHttpUrlOrNull()
?.pathSegments?.last()?.toIntOrNull() ?: 1
.pageNumberFromUrlOrNull() ?: 1
val lastPage = document.select("div.wp-pagenavi a")
.last()?.attr("href")?.substringBeforeLast("/")
?.toHttpUrlOrNull()?.pathSegments?.last()?.toIntOrNull() ?: 1
.last()
?.attr("href")
.pageNumberFromUrlOrNull() ?: 1
return AnimesPage(entries, page < lastPage)
}
@ -157,7 +146,7 @@ class JavGuru : AnimeHttpSource() {
val url = "$baseUrl${filter.toUrlPart()}" + if (page > 1) "page/$page/" else ""
val request = GET(url, headers)
return client.newCall(request)
.asObservableIgnoreCode(404)
.asObservableSuccess()
.map(::searchAnimeParse)
}
}
@ -238,55 +227,94 @@ class JavGuru : AnimeHttpSource() {
.map { Base64.decode(it, Base64.DEFAULT).let(::String) }
.toList()
return iframeUrls.mapNotNull { url ->
runCatching {
val iframeResponse = client.newCall(GET(url, headers)).execute()
if (iframeResponse.isSuccessful.not()) {
iframeResponse.close()
return@mapNotNull null
}
val iframeDocument = iframeResponse.asJsoup()
val script = iframeDocument.selectFirst("script:containsData(start_player)")
?.html() ?: return@mapNotNull null
val olid = IFRAME_OLID_REGEX.find(script)?.groupValues?.get(1)?.reversed()
?: return@mapNotNull null
val olidUrl = IFRAME_OLID_URL.find(script)?.groupValues?.get(1)?.substringBeforeLast("=")?.let { "$it=$olid" }
?: return@mapNotNull null
val newHeaders = headersBuilder()
.set("Referer", url)
.build()
val redirectUrl = client.newCall(GET(olidUrl, newHeaders))
.execute().request.url.toString()
when {
STREAM_SB_DOMAINS.any { it in redirectUrl } -> {
streamSbExtractor.videosFromUrl(redirectUrl, headers)
}
redirectUrl.contains("streamtape") -> {
streamTapeExtractor.videoFromUrl(redirectUrl)?.let { listOf(it) }
}
redirectUrl.contains("dood") -> {
doodExtractor.videosFromUrl(redirectUrl)
}
MIXDROP_DOMAINS.any { it in redirectUrl } -> {
mixDropExtractor.videoFromUrl(redirectUrl)
}
redirectUrl.contains("maxstream") -> {
maxStreamExtractor.videoFromUrl(redirectUrl)
}
else -> { null }
}
}.getOrNull()
}.flatten()
return iframeUrls
.mapNotNull(::resolveHosterUrl)
.parallelMap(::getVideos)
.flatten()
}
private fun resolveHosterUrl(iframeUrl: String): String? {
val iframeResponse = client.newCall(GET(iframeUrl, headers)).execute()
if (iframeResponse.isSuccessful.not()) {
iframeResponse.close()
return null
}
val iframeDocument = iframeResponse.asJsoup()
val script = iframeDocument.selectFirst("script:containsData(start_player)")
?.html() ?: return null
val olid = IFRAME_OLID_REGEX.find(script)?.groupValues?.get(1)?.reversed()
?: return null
val olidUrl = IFRAME_OLID_URL.find(script)?.groupValues?.get(1)
?.substringBeforeLast("=")?.let { "$it=$olid" }
?: return null
val newHeaders = headersBuilder()
.set("Referer", iframeUrl)
.build()
val redirectUrl = noRedirectClient.newCall(GET(olidUrl, newHeaders))
.execute().header("location")
?: return null
if (redirectUrl.toHttpUrlOrNull() == null) {
return null
}
return redirectUrl
}
private val streamSbExtractor by lazy { StreamSBExtractor(client) }
private val streamTapeExtractor by lazy { StreamTapeExtractor(client) }
private val doodExtractor by lazy { DoodExtractor(client) }
private val mixDropExtractor by lazy { MixDropExtractor(client) }
private val maxStreamExtractor by lazy { MaxStreamExtractor(client) }
private val emTurboExtractor by lazy { EmTurboExtractor(client) }
private fun getVideos(hosterUrl: String): List<Video> {
return runCatching {
when {
hosterUrl.contains("streamtape") -> {
streamTapeExtractor.videoFromUrl(hosterUrl)
?.let(::listOf) ?: emptyList()
}
hosterUrl.contains("dood") -> {
doodExtractor.videosFromUrl(hosterUrl)
}
MIXDROP_DOMAINS.any { it in hosterUrl } -> {
mixDropExtractor.videoFromUrl(hosterUrl)
}
hosterUrl.contains("maxstream") -> {
maxStreamExtractor.videoFromUrl(hosterUrl)
}
hosterUrl.contains("emturbovid") -> {
emTurboExtractor.getVideos(hosterUrl)
}
STREAM_SB_DOMAINS.any { it in hosterUrl } -> {
streamSbExtractor.videosFromUrl(hosterUrl, headers)
}
else -> {
emptyList()
}
}
}.getOrDefault(emptyList())
}
private fun <A, B> Iterable<A>.parallelMap(f: suspend (A) -> B): List<B> =
runBlocking {
map { async(Dispatchers.Default) { f(it) } }.awaitAll()
}
private fun getIDFromUrl(element: Elements): String? {
return element.attr("abs:href")
.toHttpUrlOrNull()
@ -297,6 +325,14 @@ class JavGuru : AnimeHttpSource() {
?.let { "/$it/" }
}
private fun String?.pageNumberFromUrlOrNull() =
this
?.substringBeforeLast("/")
?.toHttpUrlOrNull()
?.pathSegments
?.last()
?.toIntOrNull()
private fun Call.asObservableIgnoreCode(code: Int): Observable<Response> {
return asObservable().doOnNext { response ->
if (!response.isSuccessful && response.code != code) {
@ -321,7 +357,7 @@ class JavGuru : AnimeHttpSource() {
"sbfast", "sbfull.com", "javplaya.com", "ssbstream.net",
"p1ayerjavseen.com", "sbthe.com", "vidmovie.xyz", "sbspeed.com",
"streamsss.net", "sblanh.com", "tvmshow.com", "sbanh.com",
"streamovies.xyz", "sblona.com",
"streamovies.xyz", "sblona.com", "likessb.com",
)
private val MIXDROP_DOMAINS = listOf(
"mixdrop",

35
src/all/javguru/src/eu/kanade/tachiyomi/animeextension/all/javguru/extractors/EmTurboExtractor.kt Normal file
View File

@ -0,0 +1,35 @@
package eu.kanade.tachiyomi.animeextension.all.javguru.extractors
import eu.kanade.tachiyomi.animesource.model.Video
import eu.kanade.tachiyomi.lib.playlistutils.PlaylistUtils
import eu.kanade.tachiyomi.network.GET
import eu.kanade.tachiyomi.util.asJsoup
import okhttp3.HttpUrl.Companion.toHttpUrlOrNull
import okhttp3.OkHttpClient
class EmTurboExtractor(private val client: OkHttpClient) {
private val playlistExtractor by lazy { PlaylistUtils(client) }
fun getVideos(url: String): List<Video> {
val document = client.newCall(GET(url)).execute().asJsoup()
val script = document.selectFirst("script:containsData(urlplay)")
?.data()
?: return emptyList()
val urlPlay = URLPLAY.find(script)?.groupValues?.get(1)
?: return emptyList()
if (urlPlay.toHttpUrlOrNull() == null) {
return emptyList()
}
return playlistExtractor.extractFromHls(urlPlay, url, videoNameGen = { quality -> "EmTurboVid: $quality" })
.distinctBy { it.url } // they have the same stream repeated twice in the playlist file
}
companion object {
private val URLPLAY = Regex("""urlPlay\s*=\s*\'([^\']+)""")
}
}

4
src/all/javguru/src/eu/kanade/tachiyomi/animeextension/all/javguru/extractors/MaxStreamExtractor.kt
View File

@ -10,9 +10,7 @@ import okhttp3.OkHttpClient
class MaxStreamExtractor(private val client: OkHttpClient) {
private val playListUtils: PlaylistUtils by lazy {
PlaylistUtils(client)
}
private val playListUtils by lazy { PlaylistUtils(client) }
fun videoFromUrl(url: String): List<Video> {
val document = client.newCall(GET(url)).execute().asJsoup()