MMRCMS update (#1789)

MMRCMS update

src/all/mmrcms/build.gradle

@@ -5,7 +5,7 @@ ext {
     appName = 'Tachiyomi: My Manga Reader CMS (Many sources)'
     pkgNameSuffix = 'all.mmrcms'
     extClass = '.MyMangaReaderCMSSources'
-    extVersionCode = 22
+    extVersionCode = 23
     libVersion = '1.2'
 }
 

src/all/mmrcms/src/eu/kanade/tachiyomi/extension/all/mmrcms/Generator.kt

@@ -1,5 +1,6 @@
 package eu.kanade.tachiyomi.extension.all.mmrcms
 
+import android.annotation.SuppressLint
 import android.annotation.TargetApi
 import android.os.Build
 import com.google.gson.Gson
@@ -34,7 +35,7 @@ class Generator {
         var number = 1
         sources.forEach {
             try {
-                var map = mutableMapOf<String, Any>()
+                val map = mutableMapOf<String, Any>()
                 map["language"] = it.first
                 map["name"] = it.second
                 map["base_url"] = it.third
@@ -47,7 +48,7 @@ class Generator {
                     parseCategories = parseCategories(advancedSearchDocument)
                 }
 
-                val homePageDocument = getDocument("${it.third}")!!
+                val homePageDocument = getDocument(it.third)!!
 
                 val itemUrl = getItemUrl(homePageDocument)
 
@@ -109,11 +110,28 @@ class Generator {
     }
 
     private fun getDocument(url: String, printStackTrace: Boolean = true): Document? {
-        try {
+        val serverCheck = arrayOf("cloudflare-nginx", "cloudflare")
 
-            val response = getOkHttpClient().newCall(Request.Builder().url(url).build()).execute()
+        try {
-            if (response.code() == 200) {
+            val request = Request.Builder().url(url)
-                return Jsoup.parse(response.body()?.string())
+            getOkHttpClient().newCall(request.build()).execute().let { response ->
+                // Bypass Cloudflare ("Please wait 5 seconds" page)
+                if (response.code() == 503 && response.header("Server") in serverCheck) {
+                    var cookie = "${response.header("Set-Cookie")!!.substringBefore(";")}; "
+                    Jsoup.parse(response.body()!!.string()).let { document ->
+                        val path = document.select("[id=\"challenge-form\"]").attr("action")
+                        val chk = document.select("[name=\"s\"]").attr("value")
+                        getOkHttpClient().newCall(Request.Builder().url("$url/$path?s=$chk").build()).execute().let { solved ->
+                            cookie += solved.header("Set-Cookie")!!.substringBefore(";")
+                            request.addHeader("Cookie", cookie).build().let {
+                                return Jsoup.parse(getOkHttpClient().newCall(it).execute().body()?.string())
+                            }
+                        }
+                    }
+                }
+                if (response.code() == 200) {
+                    return Jsoup.parse(response.body()?.string())
+                }
             }
         } catch (e: Exception) {
             if (printStackTrace) {
@@ -129,9 +147,9 @@ class Generator {
         if (elements.isEmpty()) {
             return mutableListOf()
         }
-        var array = mutableListOf<Map<String, String>>()
+        val array = mutableListOf<Map<String, String>>()
         elements.forEach {
-            var map = mutableMapOf<String, String>()
+            val map = mutableMapOf<String, String>()
             map["id"] = it.attr("href").substringAfterLast("/")
             map["name"] = it.text()
             array.add(map)
@@ -148,19 +166,19 @@ class Generator {
     }
 
     private fun supportsLatest(third: String): Boolean {
-        getDocument("$third/filterList?page=1&sortBy=last_release&asc=false", false) ?: return false
+        val document = getDocument("$third/filterList?page=1&sortBy=last_release&asc=false", false) ?: return false
-        return true
+        return document.select("div[class^=col-sm], div.col-xs-6").isNotEmpty()
     }
 
     private fun parseCategories(document: Document): MutableList<Map<String, String>> {
-        var array = mutableListOf<Map<String, String>>()
+        val array = mutableListOf<Map<String, String>>()
-        var elements = document.select("select[name^=categories] option")
+        val elements = document.select("select[name^=categories] option")
         if (elements.size == 0) {
             return mutableListOf()
         }
         var id = 1
         elements.forEach {
-            var map = mutableMapOf<String, String>()
+            val map = mutableMapOf<String, String>()
             map["id"] = id.toString()
             map["name"] = it.text()
             array.add(map)
@@ -173,10 +191,12 @@ class Generator {
     @Throws(Exception::class)
     private fun getOkHttpClient(): OkHttpClient {
         val trustAllCerts = arrayOf<TrustManager>(object : X509TrustManager {
+            @SuppressLint("TrustAllX509TrustManager")
             @Throws(CertificateException::class)
             override fun checkClientTrusted(chain: Array<java.security.cert.X509Certificate>, authType: String) {
             }
 
+            @SuppressLint("TrustAllX509TrustManager")
             @Throws(CertificateException::class)
             override fun checkServerTrusted(chain: Array<java.security.cert.X509Certificate>, authType: String) {
             }
@@ -211,10 +231,7 @@ class Generator {
             Triple("en", "Fallen Angels", "http://manga.fascans.com"),
             Triple("en", "Hatigarm Scans", "https://hatigarmscans.net"),
             Triple("en", "Mangawww Reader", "http://mangawww.club"),
-            Triple("en", "ZXComic", "http://zxcomic.com"),
             Triple("en", "White Cloud Pavilion", "http://www.whitecloudpavilion.com/manga/free"),
-            Triple("en", "MangaTreat Scans", "http://www.mangatreat.com"),
-            Triple("es", "SOS Scanlation", "https://sosscanlation.com"),
             Triple("fr", "Scan FR", "http://www.scan-fr.io"),
             Triple("fr", "Scan VF", "https://www.scan-vf.co"),
             Triple("id", "Komikid", "http://www.komikid.com"),
@@ -227,16 +244,20 @@ class Generator {
             Triple("tr", "MangaHanta", "http://mangahanta.com"),
             Triple("vi", "Fallen Angels Scans", "http://truyen.fascans.com"),
             Triple("es", "LeoManga", "https://leomanga.me"),
-            Triple("es", "submanga", "https://submanga.online"),
+            Triple("es", "submanga", "https://submanga.li"),
             Triple("es", "Mangadoor", "https://mangadoor.com"),
             Triple("es", "Mangas.pw", "https://mangas.pw"),
+            Triple("es", "Tumangaonline.co", "http://tumangaonline.co"),
+            Triple("bg", "Utsukushii", "https://manga.utsukushii-bg.com"),
             //NOTE: THIS SOURCE CONTAINS A CUSTOM LANGUAGE SYSTEM (which will be ignored)!
             Triple("other", "HentaiShark", "https://www.hentaishark.com"))
-            //Now uses wpmanga
+            //Changed CMS
+            //Triple("en", "MangaTreat Scans", "http://www.mangatreat.com"),
             //Triple("en", "Chibi Manga Reader", "http://www.cmreader.info"),
-            //Blocks bots (like this one)
             //Triple("tr", "Epikmanga", "http://www.epikmanga.com"),
             //Went offline
+            //Triple("en", "ZXComic", "http://zxcomic.com"),
+            //Triple("es", "SOS Scanlation", "https://sosscanlation.com"),
             //Triple("es", "MangaCasa", "https://mangacasa.com"))
             //Triple("ja", "RAW MANGA READER", "https://rawmanga.site"),
             //Triple("ar", "Manga FYI", "http://mangafyi.com/manga/arabic"),

src/all/mmrcms/src/eu/kanade/tachiyomi/extension/all/mmrcms/MyMangaReaderCMSSource.kt

@@ -1,5 +1,6 @@
 package eu.kanade.tachiyomi.extension.all.mmrcms
 
+import android.annotation.SuppressLint
 import android.net.Uri
 import com.github.salomonbrys.kotson.array
 import com.github.salomonbrys.kotson.get
@@ -31,7 +32,12 @@ class MyMangaReaderCMSSource(override val lang: String,
 
     override val client: OkHttpClient = network.cloudflareClient
 
-    override fun popularMangaRequest(page: Int) = GET("$baseUrl/filterList?page=$page&sortBy=views&asc=false", headers)
+    override fun popularMangaRequest(page: Int): Request {
+        return when (name) {
+            "Utsukushii" -> GET("$baseUrl/manga-list", headers)
+            else -> GET("$baseUrl/filterList?page=$page&sortBy=views&asc=false", headers)
+        }
+    }
     override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
         //Query overrides everything
         val url: Uri.Builder
@@ -74,7 +80,11 @@ class MyMangaReaderCMSSource(override val lang: String,
     private fun internalMangaParse(response: Response): MangasPage {
         val document = response.asJsoup()
 
-        return MangasPage(document.select("div[class^=col-sm], div.col-xs-6").map {
+        val internalMangaSelector = when (name) {
+            "Utsukushii" -> "div.content div.col-sm-6"
+            else -> "div[class^=col-sm], div.col-xs-6"
+        }
+        return MangasPage(document.select(internalMangaSelector).map {
             SManga.create().apply {
                 val urlElement = it.getElementsByClass("chart-title")
                 if (urlElement.size == 0) {
@@ -86,34 +96,24 @@ class MyMangaReaderCMSSource(override val lang: String,
                     title = urlElement.text().trim()
                 }
 
-                val cover = it.select(".media-left img").attr("src")
+                it.select("img").let { img ->
-                thumbnail_url =
+                    thumbnail_url = when {
-                        if (cover.isEmpty()) {
+                        it.hasAttr("data-background-image") -> it.attr("data-background-image") // Utsukushii
-                            coverGuess(it.select("img").attr("src"), url)
+                        img.hasAttr("data-src") -> coverGuess(img.attr("abs:data-src"), url)
-                        } else {
+                        else -> coverGuess(img.attr("abs:src"), url)
-                            coverGuess(cover, url)
+                    }
-                        }
+                }
             }
         }, document.select(".pagination a[rel=next]").isNotEmpty())
     }
 
     // Guess thumbnails on broken websites
-
+    private fun coverGuess(url: String, mangaUrl: String): String {
-    private fun coverGuess(url: String?, mangaUrl: String): String {
+        return if (url.endsWith("no-image.png")) {
-        // Guess thumbnails on broken websites
+            "$baseUrl/uploads/manga/${mangaUrl.substringAfterLast('/')}/cover/cover_250x350.jpg"
-        if (url != null && url.isNotBlank()) {
+        } else {
-            if (url.startsWith("//")) {
+            url
-                return "$baseUrl/uploads/manga/${url.substringBeforeLast("/cover/").substringAfter("/manga/")}/cover/cover_250x350.jpg"
-            }
-            if (url.endsWith("no-image.png")) {
-                return "$baseUrl/uploads/manga/${mangaUrl?.substringAfterLast('/')}/cover/cover_250x350.jpg"
-            }
-            if (url.startsWith("/uploads/")) {
-                return "$baseUrl$url"
-            }
-            return url
         }
-        return ""
     }
 
     private fun getUrlWithoutBaseUrl(newUrl: String): String {
@@ -129,7 +129,7 @@ class MyMangaReaderCMSSource(override val lang: String,
         val builtUrl = parsedNewUrl.buildUpon().path("/")
         newPathSegments.forEach { builtUrl.appendPath(it) }
 
-        var out = builtUrl.build().encodedPath
+        var out = builtUrl.build().encodedPath!!
         if (parsedNewUrl.encodedQuery != null)
             out += "?" + parsedNewUrl.encodedQuery
         if (parsedNewUrl.encodedFragment != null)
@@ -138,19 +138,20 @@ class MyMangaReaderCMSSource(override val lang: String,
         return out
     }
 
+    @SuppressLint("DefaultLocale")
     override fun mangaDetailsParse(response: Response) = SManga.create().apply {
         val document = response.asJsoup()
         title = document.getElementsByClass("widget-title").text().trim()
-        thumbnail_url = coverGuess(document.select(".row .img-responsive").attr("src"), document.location())
+        thumbnail_url = coverGuess(document.select(".row .img-responsive").attr("abs:src"), document.location())
         description = document.select(".row .well p").text().trim()
 
-        val detailAuthor = setOf<String>("author(s)","autor(es)","auteur(s)","著作","yazar(lar)","mangaka(lar)","pengarang/penulis","pengarang","penulis","autor","المؤلف","перевод")
+        val detailAuthor = setOf("author(s)","autor(es)","auteur(s)","著作","yazar(lar)","mangaka(lar)","pengarang/penulis","pengarang","penulis","autor","المؤلف","перевод")
-        val detailArtist = setOf<String>("artist(s)","artiste(s)","sanatçi(lar)","artista(s)","artist(s)/ilustrator","الرسام","seniman")
+        val detailArtist = setOf("artist(s)","artiste(s)","sanatçi(lar)","artista(s)","artist(s)/ilustrator","الرسام","seniman")
-        val detailGenre = setOf<String>("categories","categorías","catégories","ジャンル","kategoriler","categorias","kategorie","التصنيفات","жанр","kategori" )
+        val detailGenre = setOf("categories","categorías","catégories","ジャンル","kategoriler","categorias","kategorie","التصنيفات","жанр","kategori" )
-        val detailStatus = setOf<String>("status","statut","estado","状態","durum","الحالة","статус")
+        val detailStatus = setOf("status","statut","estado","状態","durum","الحالة","статус")
-        val detailStatusComplete = setOf<String>("complete","مكتملة","complet","completo")
+        val detailStatusComplete = setOf("complete","مكتملة","complet","completo")
-        val detailStatusOngoing = setOf<String>("ongoing","مستمرة","en cours","em lançamento")
+        val detailStatusOngoing = setOf("ongoing","مستمرة","en cours","em lançamento")
-        val detailDescription = setOf<String>("description","resumen")
+        val detailDescription = setOf("description","resumen")
 
         var cur: String? = null
         for (element in document.select(".row .dl-horizontal").select("dt,dd")) {
@@ -203,7 +204,7 @@ class MyMangaReaderCMSSource(override val lang: String,
     /**
      * Returns the Jsoup selector that returns a list of [Element] corresponding to each chapter.
      */
-    fun chapterListSelector() = "ul[class^=chapters] > li:not(.btn), table.table tr"
+    private fun chapterListSelector() = "ul[class^=chapters] > li:not(.btn), table.table tr"
     //Some websites add characters after "chapters" thus the need of checking classes that starts with "chapters"
 
     /**
@@ -322,8 +323,8 @@ class MyMangaReaderCMSSource(override val lang: String,
      * If `firstIsUnspecified` is set to true, if the first entry is selected, nothing will be appended on the the URI.
      */
     //vals: <name, display>
-    open class UriSelectFilter(displayName: String, val uriParam: String, val vals: Array<Pair<String, String>>,
+    open class UriSelectFilter(displayName: String, private val uriParam: String, private val vals: Array<Pair<String, String>>,
-                               val firstIsUnspecified: Boolean = true,
+                               private val firstIsUnspecified: Boolean = true,
                                defaultValue: Int = 0) :
             Filter.Select<String>(displayName, vals.map { it.second }.toTypedArray(), defaultValue), UriFilter {
         override fun addToUri(uri: Uri.Builder) {
@@ -340,7 +341,7 @@ class MyMangaReaderCMSSource(override val lang: String,
 
     class SortFilter : Filter.Sort("Sort",
             sortables.map { it.second }.toTypedArray(),
-            Filter.Sort.Selection(0, true)), UriFilter {
+            Selection(0, true)), UriFilter {
         override fun addToUri(uri: Uri.Builder) {
             uri.appendQueryParameter("sortBy", sortables[state!!.index].first)
             uri.appendQueryParameter("asc", state!!.ascending.toString())