feat(src/all): MissAV & JavGuru, Get hd covers (#2305)

lib/javcoverfetcher/build.gradle.kts

@@ -0,0 +1,17 @@
+plugins {
+    id("com.android.library")
+    kotlin("android")
+}
+
+android {
+    compileSdk = AndroidConfig.compileSdk
+    namespace = "eu.kanade.tachiyomi.lib.javcoverfetcher"
+
+    defaultConfig {
+        minSdk = AndroidConfig.minSdk
+    }
+}
+
+dependencies {
+    compileOnly(libs.bundles.common)
+}

lib/javcoverfetcher/src/main/java/eu/kanade/tachiyomi/lib/javcoverfetcher/JavCoverFetcher.kt

@@ -0,0 +1,153 @@
+package eu.kanade.tachiyomi.lib.javcoverfetcher
+
+import android.content.SharedPreferences
+import android.util.Log
+import eu.kanade.tachiyomi.network.GET
+import eu.kanade.tachiyomi.network.NetworkHelper
+import eu.kanade.tachiyomi.network.POST
+import eu.kanade.tachiyomi.util.asJsoup
+import okhttp3.FormBody
+import okhttp3.Interceptor
+import okhttp3.Response
+import okhttp3.internal.commonEmptyHeaders
+import uy.kohesive.injekt.Injekt
+import uy.kohesive.injekt.api.get
+import java.io.IOException
+import androidx.preference.PreferenceScreen
+import androidx.preference.SwitchPreferenceCompat
+
+object JavCoverFetcher {
+
+    private val CLIENT by lazy {
+        Injekt.get<NetworkHelper>().cloudflareClient.newBuilder()
+            .addInterceptor(::amazonAgeVerifyIntercept)
+            .build()
+    }
+
+    private val HEADERS by lazy {
+        commonEmptyHeaders.newBuilder()
+            .set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36")
+            .build()
+    }
+
+    private fun amazonAgeVerifyIntercept(chain: Interceptor.Chain): Response {
+        val request = chain.request()
+        val response = chain.proceed(request)
+
+        if (!request.url.host.contains("amazon.co.jp") || !response.request.url.pathSegments.contains("black-curtain")) {
+            return response
+        }
+
+        val document = response.use { it.asJsoup() }
+        val targetUrl = document.selectFirst("#black-curtain-yes-button a")?.attr("abs:href")
+            ?: throw IOException("Failed to bypass Amazon Age Gate")
+
+        val newRequest = request.newBuilder().apply {
+            url(targetUrl)
+        }.build()
+
+        return chain.proceed(newRequest)
+    }
+
+    /**
+     *  Get HD Jav Cover from Amazon
+     *
+     * @param jpTitle title of jav in japanese
+     */
+    fun getCoverByTitle(jpTitle: String): String? {
+        return runCatching {
+            val amazonUrl = getDDGSearchResult(jpTitle)
+                ?: return@runCatching null
+
+            getHDCoverFromAmazonUrl(amazonUrl)
+        }.getOrElse {
+            Log.e("JavCoverFetcher", it.stackTraceToString())
+            null
+        }
+    }
+
+    /**
+     *  Get HD Jav Cover from Amazon
+     *
+     * @param javId standard JAV code e.g PRIN-006
+     */
+    fun getCoverById(javId: String): String? {
+        return runCatching {
+            val jpTitle = getJPTitleFromID(javId)
+                ?: return@runCatching null
+
+            val amazonUrl = getDDGSearchResult(jpTitle)
+                ?: return@runCatching null
+
+            getHDCoverFromAmazonUrl(amazonUrl)
+        }.getOrElse {
+            Log.e("JavCoverFetcher", it.stackTraceToString())
+            null
+        }
+    }
+
+    private fun getJPTitleFromID(javId: String): String? {
+        val url = "https://www.javlibrary.com/ja/vl_searchbyid.php?keyword=$javId"
+
+        val request = GET(url, HEADERS)
+
+        val response = CLIENT.newCall(request).execute()
+
+        var document = response.use { it.asJsoup() }
+
+        // possibly multiple results or none
+        if (response.request.url.pathSegments.contains("vl_searchbyid.php")) {
+            val targetUrl = document.selectFirst(".videos a[href*=\"?v=\"]")?.attr("abs:href")
+                ?: return null
+
+            document = CLIENT.newCall(GET(targetUrl, HEADERS)).execute().use { it.asJsoup() }
+        }
+
+        val dirtyTitle = document.selectFirst(".post-title")?.text()
+
+        val id = document.select("#video_info tr > td:contains(品番) + td").text()
+
+        return dirtyTitle?.substringAfter(id)?.trim()
+    }
+
+    private fun getDDGSearchResult(jpTitle: String): String? {
+        val url = "https://lite.duckduckgo.com/lite"
+
+        val form = FormBody.Builder()
+            .add("q", "site:amazon.co.jp inurl:/dp/$jpTitle")
+            .build()
+
+        val request = POST(url, HEADERS, form)
+
+        val response = CLIENT.newCall(request).execute()
+
+        val document = response.use { it.asJsoup() }
+
+        return document.selectFirst("a.result-link")?.attr("href")
+    }
+
+    private fun getHDCoverFromAmazonUrl(amazonUrl: String): String? {
+        val request = GET(amazonUrl, HEADERS)
+
+        val response = CLIENT.newCall(request).execute()
+
+        val document = response.use { it.asJsoup() }
+
+        val smallImage = document.selectFirst("#landingImage")?.attr("src")
+
+        return smallImage?.replace(Regex("""(\._\w+_\.jpg)"""), ".jpg")
+    }
+
+    fun addPreferenceToScreen(screen: PreferenceScreen) {
+        SwitchPreferenceCompat(screen.context).apply {
+            key = "JavCoverFetcherPref"
+            title = "Fetch HD covers from Amazon"
+            summary = "Attempts to fetch HD covers from Amazon.\nMay result in incorrect cover."
+            setDefaultValue(false)
+        }.also(screen::addPreference)
+    }
+
+    val SharedPreferences.fetchHDCovers
+        get() = getBoolean("JavCoverFetcherPref", false)
+
+}

src/all/javguru/build.gradle

@@ -13,11 +13,13 @@ ext {
 }
 
 dependencies {
+    implementation(project(':lib-streamwish-extractor'))
     implementation(project(':lib-streamtape-extractor'))
     implementation(project(':lib-dood-extractor'))
     implementation(project(':lib-mixdrop-extractor'))
     implementation "dev.datlag.jsunpacker:jsunpacker:1.0.1"
     implementation(project(':lib-playlist-utils'))
+    implementation(project(':lib-javcoverfetcher'))
 }
 
 apply from: "$rootDir/common.gradle"

src/all/javguru/src/eu/kanade/tachiyomi/animeextension/all/javguru/JavGuru.kt

@@ -1,8 +1,12 @@
 package eu.kanade.tachiyomi.animeextension.all.javguru
 
+import android.app.Application
 import android.util.Base64
+import androidx.preference.ListPreference
+import androidx.preference.PreferenceScreen
 import eu.kanade.tachiyomi.animeextension.all.javguru.extractors.EmTurboExtractor
 import eu.kanade.tachiyomi.animeextension.all.javguru.extractors.MaxStreamExtractor
+import eu.kanade.tachiyomi.animesource.ConfigurableAnimeSource
 import eu.kanade.tachiyomi.animesource.model.AnimeFilterList
 import eu.kanade.tachiyomi.animesource.model.AnimesPage
 import eu.kanade.tachiyomi.animesource.model.SAnime
@@ -10,12 +14,14 @@ import eu.kanade.tachiyomi.animesource.model.SEpisode
 import eu.kanade.tachiyomi.animesource.model.Video
 import eu.kanade.tachiyomi.animesource.online.AnimeHttpSource
 import eu.kanade.tachiyomi.lib.doodextractor.DoodExtractor
+import eu.kanade.tachiyomi.lib.javcoverfetcher.JavCoverFetcher
+import eu.kanade.tachiyomi.lib.javcoverfetcher.JavCoverFetcher.fetchHDCovers
 import eu.kanade.tachiyomi.lib.mixdropextractor.MixDropExtractor
 import eu.kanade.tachiyomi.lib.streamtapeextractor.StreamTapeExtractor
+import eu.kanade.tachiyomi.lib.streamwishextractor.StreamWishExtractor
 import eu.kanade.tachiyomi.network.GET
 import eu.kanade.tachiyomi.network.asObservable
 import eu.kanade.tachiyomi.network.asObservableSuccess
-import eu.kanade.tachiyomi.network.interceptor.rateLimit
 import eu.kanade.tachiyomi.util.asJsoup
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.async
@@ -28,9 +34,11 @@ import okhttp3.Request
 import okhttp3.Response
 import org.jsoup.select.Elements
 import rx.Observable
+import uy.kohesive.injekt.Injekt
+import uy.kohesive.injekt.api.get
 import kotlin.math.min
 
-class JavGuru : AnimeHttpSource() {
+class JavGuru : AnimeHttpSource(), ConfigurableAnimeSource {
 
     override val name = "Jav Guru"
 
@@ -40,16 +48,15 @@ class JavGuru : AnimeHttpSource() {
 
     override val supportsLatest = true
 
-    override val client = network.cloudflareClient.newBuilder()
+    override val client = network.cloudflareClient
-        .rateLimit(2)
-        .build()
 
     private val noRedirectClient = client.newBuilder()
         .followRedirects(false)
         .build()
 
-    override fun headersBuilder() = super.headersBuilder()
+    private val preference by lazy {
-        .add("Referer", "$baseUrl/")
+        Injekt.get<Application>().getSharedPreferences("source_$id", 0x0000)
+    }
 
     private lateinit var popularElements: Elements
 
@@ -186,9 +193,11 @@ class JavGuru : AnimeHttpSource() {
     override fun animeDetailsParse(response: Response): SAnime {
         val document = response.asJsoup()
 
+        val javId = document.selectFirst(".infoleft li:contains(code)")?.ownText()
+        val siteCover = document.select(".large-screenshot img").attr("abs:src")
+
         return SAnime.create().apply {
             title = document.select(".titl").text()
-            thumbnail_url = document.select(".large-screenshot img").attr("abs:src")
             genre = document.select(".infoleft a[rel*=tag]").joinToString { it.text() }
             author = document.selectFirst(".infoleft li:contains(studio) a")?.text()
             artist = document.selectFirst(".infoleft li:contains(label) a")?.text()
@@ -201,6 +210,11 @@ class JavGuru : AnimeHttpSource() {
                 document.selectFirst(".infoleft li:contains(actor)")?.text()?.let { append("$it\n") }
                 document.selectFirst(".infoleft li:contains(actress)")?.text()?.let { append("$it\n") }
             }
+            thumbnail_url = if (preference.fetchHDCovers) {
+                javId?.let { JavCoverFetcher.getCoverById(it) } ?: siteCover
+            } else {
+                siteCover
+            }
         }
     }
 
@@ -257,7 +271,7 @@ class JavGuru : AnimeHttpSource() {
             .build()
 
         val redirectUrl = noRedirectClient.newCall(GET(olidUrl, newHeaders))
-            .execute().header("location")
+            .execute().use { it.header("location") }
             ?: return null
 
         if (redirectUrl.toHttpUrlOrNull() == null) {
@@ -267,18 +281,22 @@ class JavGuru : AnimeHttpSource() {
         return redirectUrl
     }
 
+    private val streamWishExtractor by lazy { StreamWishExtractor(client, headers) }
     private val streamTapeExtractor by lazy { StreamTapeExtractor(client) }
     private val doodExtractor by lazy { DoodExtractor(client) }
     private val mixDropExtractor by lazy { MixDropExtractor(client) }
-    private val maxStreamExtractor by lazy { MaxStreamExtractor(client) }
+    private val maxStreamExtractor by lazy { MaxStreamExtractor(client, headers) }
-    private val emTurboExtractor by lazy { EmTurboExtractor(client) }
+    private val emTurboExtractor by lazy { EmTurboExtractor(client, headers) }
 
     private fun getVideos(hosterUrl: String): List<Video> {
         return runCatching {
             when {
+                hosterUrl.contains("javplaya") -> {
+                    streamWishExtractor.videosFromUrl(hosterUrl)
+                }
+
                 hosterUrl.contains("streamtape") -> {
-                    streamTapeExtractor.videoFromUrl(hosterUrl)
+                    streamTapeExtractor.videoFromUrl(hosterUrl).let(::listOfNotNull)
-                        ?.let(::listOf) ?: emptyList()
                 }
 
                 hosterUrl.contains("dood") -> {
@@ -304,6 +322,14 @@ class JavGuru : AnimeHttpSource() {
         }.getOrDefault(emptyList())
     }
 
+    override fun List<Video>.sort(): List<Video> {
+        val quality = preference.getString(PREF_QUALITY, PREF_QUALITY_DEFAULT)!!
+
+        return sortedWith(
+            compareBy { it.quality.contains(quality) },
+        ).reversed()
+    }
+
     private fun <A, B> Iterable<A>.parallelMap(f: suspend (A) -> B): List<B> =
         runBlocking {
             map { async(Dispatchers.Default) { f(it) } }.awaitAll()
@@ -336,6 +362,19 @@ class JavGuru : AnimeHttpSource() {
         }
     }
 
+    override fun setupPreferenceScreen(screen: PreferenceScreen) {
+        ListPreference(screen.context).apply {
+            key = PREF_QUALITY
+            title = PREF_QUALITY_TITLE
+            entries = arrayOf("1080p", "720p", "480p", "360p")
+            entryValues = arrayOf("1080", "720", "480", "360")
+            setDefaultValue(PREF_QUALITY_DEFAULT)
+            summary = "%s"
+        }.also(screen::addPreference)
+
+        JavCoverFetcher.addPreferenceToScreen(screen)
+    }
+
     companion object {
         const val PREFIX_ID = "id:"
 
@@ -347,6 +386,10 @@ class JavGuru : AnimeHttpSource() {
             "mixdrop",
             "mixdroop",
         )
+
+        private const val PREF_QUALITY = "preferred_quality"
+        private const val PREF_QUALITY_TITLE = "Preferred quality"
+        private const val PREF_QUALITY_DEFAULT = "720"
     }
 
     override fun episodeListParse(response: Response): List<SEpisode> {

src/all/javguru/src/eu/kanade/tachiyomi/animeextension/all/javguru/extractors/EmTurboExtractor.kt

@@ -4,15 +4,16 @@ import eu.kanade.tachiyomi.animesource.model.Video
 import eu.kanade.tachiyomi.lib.playlistutils.PlaylistUtils
 import eu.kanade.tachiyomi.network.GET
 import eu.kanade.tachiyomi.util.asJsoup
+import okhttp3.Headers
 import okhttp3.HttpUrl.Companion.toHttpUrlOrNull
 import okhttp3.OkHttpClient
 
-class EmTurboExtractor(private val client: OkHttpClient) {
+class EmTurboExtractor(private val client: OkHttpClient, private val headers: Headers) {
 
-    private val playlistExtractor by lazy { PlaylistUtils(client) }
+    private val playlistExtractor by lazy { PlaylistUtils(client, headers) }
 
     fun getVideos(url: String): List<Video> {
-        val document = client.newCall(GET(url)).execute().asJsoup()
+        val document = client.newCall(GET(url, headers)).execute().asJsoup()
 
         val script = document.selectFirst("script:containsData(urlplay)")
             ?.data()

src/all/javguru/src/eu/kanade/tachiyomi/animeextension/all/javguru/extractors/MaxStreamExtractor.kt

@@ -5,15 +5,16 @@ import eu.kanade.tachiyomi.animesource.model.Video
 import eu.kanade.tachiyomi.lib.playlistutils.PlaylistUtils
 import eu.kanade.tachiyomi.network.GET
 import eu.kanade.tachiyomi.util.asJsoup
+import okhttp3.Headers
 import okhttp3.HttpUrl.Companion.toHttpUrlOrNull
 import okhttp3.OkHttpClient
 
-class MaxStreamExtractor(private val client: OkHttpClient) {
+class MaxStreamExtractor(private val client: OkHttpClient, private val headers: Headers) {
 
-    private val playListUtils by lazy { PlaylistUtils(client) }
+    private val playListUtils by lazy { PlaylistUtils(client, headers) }
 
     fun videoFromUrl(url: String): List<Video> {
-        val document = client.newCall(GET(url)).execute().asJsoup()
+        val document = client.newCall(GET(url, headers)).execute().asJsoup()
 
         val script = document.selectFirst("script:containsData(function(p,a,c,k,e,d))")
             ?.data()

src/all/missav/build.gradle

@@ -14,6 +14,7 @@ ext {
 dependencies {
     implementation(project(':lib-unpacker'))
     implementation(project(':lib-playlist-utils'))
+    implementation(project(':lib-javcoverfetcher'))
 }
 
 apply from: "$rootDir/common.gradle"

src/all/missav/src/eu/kanade/tachiyomi/animeextension/all/missav/MissAV.kt

@@ -10,6 +10,8 @@ import eu.kanade.tachiyomi.animesource.model.SAnime
 import eu.kanade.tachiyomi.animesource.model.SEpisode
 import eu.kanade.tachiyomi.animesource.model.Video
 import eu.kanade.tachiyomi.animesource.online.AnimeHttpSource
+import eu.kanade.tachiyomi.lib.javcoverfetcher.JavCoverFetcher
+import eu.kanade.tachiyomi.lib.javcoverfetcher.JavCoverFetcher.fetchHDCovers
 import eu.kanade.tachiyomi.lib.playlistutils.PlaylistUtils
 import eu.kanade.tachiyomi.lib.unpacker.Unpacker
 import eu.kanade.tachiyomi.network.GET
@@ -98,6 +100,9 @@ class MissAV : AnimeHttpSource(), ConfigurableAnimeSource {
     override fun animeDetailsParse(response: Response): SAnime {
         val document = response.use { it.asJsoup() }
 
+        val jpTitle = document.select("div.text-secondary span:contains(title) + span").text()
+        val siteCover = document.selectFirst("video.player")?.attr("abs:data-poster")
+
         return SAnime.create().apply {
             title = document.selectFirst("h1.text-base")!!.text()
             genre = document.getInfo("/genres/")
@@ -107,8 +112,6 @@ class MissAV : AnimeHttpSource(), ConfigurableAnimeSource {
             ).joinToString()
             artist = document.getInfo("/actresses/")
             status = SAnime.COMPLETED
-            thumbnail_url = document.selectFirst("video.player")?.attr("abs:data-poster")
-
             description = buildString {
                 document.selectFirst("div.mb-1")?.text()?.also { append("$it\n") }
 
@@ -119,6 +122,11 @@ class MissAV : AnimeHttpSource(), ConfigurableAnimeSource {
                     .eachText()
                     .forEach { append("\n$it") }
             }
+            thumbnail_url = if (preferences.fetchHDCovers) {
+                JavCoverFetcher.getCoverByTitle(jpTitle) ?: siteCover
+            } else {
+                siteCover
+            }
         }
     }
 
@@ -169,6 +177,8 @@ class MissAV : AnimeHttpSource(), ConfigurableAnimeSource {
             setDefaultValue(PREF_QUALITY_DEFAULT)
             summary = "%s"
         }.also(screen::addPreference)
+
+        JavCoverFetcher.addPreferenceToScreen(screen)
     }
 
     override fun episodeListParse(response: Response): List<SEpisode> {