MAL API Workaround (#1647)

* Mal API workaround

* remove unused import

* Reuse existing token preference

* Minor code format

app/src/main/java/eu/kanade/tachiyomi/data/track/TrackService.kt

@@ -60,12 +60,11 @@ abstract class TrackService(val id: Int) {
         get() = !getUsername().isEmpty() &&
                 !getPassword().isEmpty()
 
-    fun getUsername() = preferences.trackUsername(this)
+    fun getUsername() = preferences.trackUsername(this)!!
 
-    fun getPassword() = preferences.trackPassword(this)
+    fun getPassword() = preferences.trackPassword(this)!!
 
     fun saveCredentials(username: String, password: String) {
         preferences.setTrackCredentials(this, username, password)
     }
-
 }

app/src/main/java/eu/kanade/tachiyomi/data/track/myanimelist/MyAnimeList.kt

@@ -4,10 +4,12 @@ import android.content.Context
 import android.graphics.Color
 import eu.kanade.tachiyomi.R
 import eu.kanade.tachiyomi.data.database.models.Track
-import eu.kanade.tachiyomi.data.track.model.TrackSearch
+import eu.kanade.tachiyomi.data.preference.getOrDefault
 import eu.kanade.tachiyomi.data.track.TrackService
+import eu.kanade.tachiyomi.data.track.model.TrackSearch
 import rx.Completable
 import rx.Observable
+import java.net.URI
 
 class Myanimelist(private val context: Context, id: Int) : TrackService(id) {
 
@@ -21,9 +23,13 @@ class Myanimelist(private val context: Context, id: Int) : TrackService(id) {
 
         const val DEFAULT_STATUS = READING
         const val DEFAULT_SCORE = 0
+
+        const val BASE_URL = "https://myanimelist.net"
+        const val USER_SESSION_COOKIE = "MALSESSIONID"
+        const val LOGGED_IN_COOKIE = "is_logged_in"
     }
 
-    private val api by lazy { MyanimelistApi(client, getUsername(), getPassword()) }
+    private val api by lazy { MyanimelistApi(client) }
 
     override val name: String
         get() = "MyAnimeList"
@@ -56,7 +62,7 @@ class Myanimelist(private val context: Context, id: Int) : TrackService(id) {
     }
 
     override fun add(track: Track): Observable<Track> {
-        return api.addLibManga(track)
+        return api.addLibManga(track, getCSRF())
     }
 
     override fun update(track: Track): Observable<Track> {
@@ -64,11 +70,11 @@ class Myanimelist(private val context: Context, id: Int) : TrackService(id) {
             track.status = COMPLETED
         }
 
-        return api.updateLibManga(track)
+        return api.updateLibManga(track, getCSRF())
     }
 
     override fun bind(track: Track): Observable<Track> {
-        return api.findLibManga(track, getUsername())
+        return api.findLibManga(track, getCSRF())
                 .flatMap { remoteTrack ->
                     if (remoteTrack != null) {
                         track.copyPersonalFrom(remoteTrack)
@@ -83,11 +89,11 @@ class Myanimelist(private val context: Context, id: Int) : TrackService(id) {
     }
 
     override fun search(query: String): Observable<List<TrackSearch>> {
-        return api.search(query, getUsername())
+        return api.search(query)
     }
 
     override fun refresh(track: Track): Observable<Track> {
-        return api.getLibManga(track, getUsername())
+        return api.getLibManga(track, getCSRF())
                 .map { remoteTrack ->
                     track.copyPersonalFrom(remoteTrack)
                     track.total_chapters = remoteTrack.total_chapters
@@ -96,10 +102,40 @@ class Myanimelist(private val context: Context, id: Int) : TrackService(id) {
     }
 
     override fun login(username: String, password: String): Completable {
+        logout()
+
         return api.login(username, password)
+                .doOnNext { csrf -> saveCSRF(csrf) }
                 .doOnNext { saveCredentials(username, password) }
                 .doOnError { logout() }
                 .toCompletable()
     }
 
+    override fun logout() {
+        super.logout()
+        preferences.trackToken(this).delete()
+        networkService.cookies.remove(URI(BASE_URL))
+    }
+
+    override val isLogged: Boolean
+        get() = !getUsername().isEmpty() &&
+                !getPassword().isEmpty() &&
+                checkCookies(URI(BASE_URL)) &&
+                !getCSRF().isEmpty()
+
+    private fun getCSRF(): String = preferences.trackToken(this).getOrDefault()
+
+    private fun saveCSRF(csrf: String) = preferences.trackToken(this).set(csrf)
+
+    private fun checkCookies(uri: URI): Boolean {
+        var ckCount = 0
+
+        for (ck in networkService.cookies.get(uri)) {
+            if (ck.name() == USER_SESSION_COOKIE || ck.name() == LOGGED_IN_COOKIE)
+                ckCount++
+        }
+
+        return ckCount == 2
+    }
+
 }

app/src/main/java/eu/kanade/tachiyomi/data/track/myanimelist/MyanimelistApi.kt

@@ -1,7 +1,6 @@
 package eu.kanade.tachiyomi.data.track.myanimelist
 
 import android.net.Uri
-import android.util.Xml
 import eu.kanade.tachiyomi.data.database.models.Track
 import eu.kanade.tachiyomi.data.track.TrackManager
 import eu.kanade.tachiyomi.data.track.model.TrackSearch
@@ -12,191 +11,266 @@ import eu.kanade.tachiyomi.network.asObservableSuccess
 import eu.kanade.tachiyomi.util.selectInt
 import eu.kanade.tachiyomi.util.selectText
 import okhttp3.*
+import org.json.JSONObject
 import org.jsoup.Jsoup
+import org.jsoup.nodes.Document
+import org.jsoup.nodes.Element
 import org.jsoup.parser.Parser
-import org.xmlpull.v1.XmlSerializer
 import rx.Observable
-import java.io.StringWriter
+import java.io.BufferedReader
+import java.io.InputStreamReader
+import java.util.zip.GZIPInputStream
 
-class MyanimelistApi(private val client: OkHttpClient, username: String, password: String) {
 
-    private var headers = createHeaders(username, password)
+class MyanimelistApi(private val client: OkHttpClient) {
 
-    fun addLibManga(track: Track): Observable<Track> {
+    fun addLibManga(track: Track, csrf: String): Observable<Track> {
         return Observable.defer {
-            client.newCall(POST(getAddUrl(track), headers, getMangaPostPayload(track)))
+            client.newCall(POST(url = getAddUrl(), body = getMangaPostPayload(track, csrf)))
                     .asObservableSuccess()
                     .map { track }
         }
     }
 
-    fun updateLibManga(track: Track): Observable<Track> {
+    fun updateLibManga(track: Track, csrf: String): Observable<Track> {
         return Observable.defer {
-            client.newCall(POST(getUpdateUrl(track), headers, getMangaPostPayload(track)))
+            client.newCall(POST(url = getUpdateUrl(), body = getMangaPostPayload(track, csrf)))
                     .asObservableSuccess()
                     .map { track }
         }
     }
 
-    fun search(query: String, username: String): Observable<List<TrackSearch>> {
+    fun search(query: String): Observable<List<TrackSearch>> {
-        return if (query.startsWith(PREFIX_MY)) {
+        return client.newCall(GET(getSearchUrl(query)))
-            val realQuery = query.substring(PREFIX_MY.length).toLowerCase().trim()
-            getList(username)
-                    .flatMap { Observable.from(it) }
-                    .filter { realQuery in it.title.toLowerCase() }
-                    .toList()
-        } else {
-            client.newCall(GET(getSearchUrl(query), headers))
                 .asObservable()
-                    .map { Jsoup.parse(Parser.unescapeEntities(it.body()!!.string(), false), "", Parser.xmlParser()) }
+                .flatMap { response ->
-                    .flatMap { Observable.from(it.select("entry")) }
+                    Observable.from(Jsoup.parse(response.consumeBody())
-                    .filter { it.select("type").text() != "Novel" }
+                            .select("div.js-categories-seasonal.js-block-list.list")
-                    .map {
+                            .select("table").select("tbody")
+                            .select("tr").drop(1))
+                }
+                .filter { row ->
+                    row.select(TD)[2].text() != "Novel"
+                }
+                .map { row ->
                     TrackSearch.create(TrackManager.MYANIMELIST).apply {
-                            title = it.selectText("title")!!
+                        title = row.searchTitle()
-                            media_id = it.selectInt("id")
+                        media_id = row.searchMediaId()
-                            total_chapters = it.selectInt("chapters")
+                        total_chapters = row.searchTotalChapters()
-                            summary = it.selectText("synopsis")!!
+                        summary = row.searchSummary()
-                            cover_url = it.selectText("image")!!
+                        cover_url = row.searchCoverUrl()
-                            tracking_url = MyanimelistApi.mangaUrl(media_id)
+                        tracking_url = mangaUrl(media_id)
-                            publishing_status = it.selectText("status")!!
+                        publishing_status = row.searchPublishingStatus()
-                            publishing_type = it.selectText("type")!!
+                        publishing_type = row.searchPublishingType()
-                            start_date = it.selectText("start_date")!!
+                        start_date = row.searchStartDate()
                     }
                 }
                 .toList()
     }
-    }
 
-    fun getList(username: String): Observable<List<TrackSearch>> {
+    private fun getList(csrf: String): Observable<List<TrackSearch>> {
-        return client
+        return getListUrl(csrf)
-                .newCall(GET(getListUrl(username), headers))
+                .flatMap { url ->
-                .asObservable()
+                    getListXml(url)
-                .map { Jsoup.parse(Parser.unescapeEntities(it.body()!!.string(), false), "", Parser.xmlParser()) }
+                }
-                .flatMap { Observable.from(it.select("manga")) }
+                .flatMap { doc ->
-                .map {
+                    Observable.from(doc.select("manga"))
+                }
+                .map { it ->
                     TrackSearch.create(TrackManager.MYANIMELIST).apply {
-                        title = it.selectText("series_title")!!
+                        title = it.selectText("manga_title")!!
-                        media_id = it.selectInt("series_mangadb_id")
+                        media_id = it.selectInt("manga_mangadb_id")
                         last_chapter_read = it.selectInt("my_read_chapters")
-                        status = it.selectInt("my_status")
+                        status = getStatus(it.selectText("my_status")!!)
                         score = it.selectInt("my_score").toFloat()
-                        total_chapters = it.selectInt("series_chapters")
+                        total_chapters = it.selectInt("manga_chapters")
-                        cover_url = it.selectText("series_image")!!
+                        tracking_url = mangaUrl(media_id)
-                        tracking_url = MyanimelistApi.mangaUrl(media_id)
                     }
                 }
                 .toList()
     }
 
-    fun findLibManga(track: Track, username: String): Observable<Track?> {
+    private fun getListXml(url: String): Observable<Document> {
-        return getList(username)
+        return client.newCall(GET(url))
+                .asObservable()
+                .map { response ->
+                    Jsoup.parse(response.consumeXmlBody(), "", Parser.xmlParser())
+                }
+    }
+
+    fun findLibManga(track: Track, csrf: String): Observable<Track?> {
+        return getList(csrf)
                 .map { list -> list.find { it.media_id == track.media_id } }
     }
 
-    fun getLibManga(track: Track, username: String): Observable<Track> {
+    fun getLibManga(track: Track, csrf: String): Observable<Track> {
-        return findLibManga(track, username)
+        return findLibManga(track, csrf)
                 .map { it ?: throw Exception("Could not find manga") }
     }
 
-    fun login(username: String, password: String): Observable<Response> {
+    fun login(username: String, password: String): Observable<String> {
-        headers = createHeaders(username, password)
+        return getSessionInfo()
-        return client.newCall(GET(getLoginUrl(), headers))
+                .flatMap { csrf ->
+                    login(username, password, csrf)
+                }
+    }
+
+    private fun getSessionInfo(): Observable<String> {
+        return client.newCall(GET(getLoginUrl()))
                 .asObservable()
-                .doOnNext { response ->
+                .map { response ->
-                    response.close()
+                    Jsoup.parse(response.consumeBody())
-                    if (response.code() != 200) throw Exception("Login error")
+                            .select("meta[name=csrf_token]")
+                            .attr("content")
                 }
     }
 
-    private fun getMangaPostPayload(track: Track): RequestBody {
+    private fun login(username: String, password: String, csrf: String): Observable<String> {
-        val data = xml {
+        return client.newCall(POST(url = getLoginUrl(), body = getLoginPostBody(username, password, csrf)))
-            element(ENTRY_TAG) {
+                .asObservable()
-                if (track.last_chapter_read != 0) {
+                .map { response ->
-                    text(CHAPTER_TAG, track.last_chapter_read.toString())
+                    response.use {
+                        if (response.priorResponse()?.code() != 302) throw Exception("Authentication error")
                     }
-                text(STATUS_TAG, track.status.toString())
+                    csrf
-                text(SCORE_TAG, track.score.toString())
                 }
     }
 
+    private fun getLoginPostBody(username: String, password: String, csrf: String): RequestBody {
         return FormBody.Builder()
-                .add("data", data)
+                .add("user_name", username)
+                .add("password", password)
+                .add("cookie", "1")
+                .add("sublogin", "Login")
+                .add("submit", "1")
+                .add(CSRF, csrf)
                 .build()
     }
 
-    private inline fun xml(block: XmlSerializer.() -> Unit): String {
+    private fun getExportPostBody(csrf: String): RequestBody {
-        val x = Xml.newSerializer()
+        return FormBody.Builder()
-        val writer = StringWriter()
+                .add("type", "2")
-
+                .add("subexport", "Export My List")
-        with(x) {
+                .add(CSRF, csrf)
-            setOutput(writer)
+                .build()
-            startDocument("UTF-8", false)
-            block()
-            endDocument()
     }
 
-        return writer.toString()
+    private fun getMangaPostPayload(track: Track, csrf: String): RequestBody {
+        val body = JSONObject()
+                .put("manga_id", track.media_id)
+                .put("status", track.status)
+                .put("score", track.score)
+                .put("num_read_chapters", track.last_chapter_read)
+                .put(CSRF, csrf)
+
+        return RequestBody.create(MediaType.parse("application/json; charset=utf-8"), body.toString())
     }
 
-    private inline fun XmlSerializer.element(tag: String, block: XmlSerializer.() -> Unit) {
+    private fun getLoginUrl() = Uri.parse(baseUrl).buildUpon()
-        startTag("", tag)
+            .appendPath("login.php")
-        block()
-        endTag("", tag)
-    }
-
-    private fun XmlSerializer.text(tag: String, body: String) {
-        startTag("", tag)
-        text(body)
-        endTag("", tag)
-    }
-
-    fun getLoginUrl() = Uri.parse(baseUrl).buildUpon()
-            .appendEncodedPath("api/account/verify_credentials.xml")
             .toString()
 
-    fun getSearchUrl(query: String) = Uri.parse(baseUrl).buildUpon()
+    private fun getSearchUrl(query: String): String {
-            .appendEncodedPath("api/manga/search.xml")
+        val col = "c[]"
+        return Uri.parse(baseUrl).buildUpon()
+                .appendPath("manga.php")
                 .appendQueryParameter("q", query)
+                .appendQueryParameter(col, "a")
+                .appendQueryParameter(col, "b")
+                .appendQueryParameter(col, "c")
+                .appendQueryParameter(col, "d")
+                .appendQueryParameter(col, "e")
+                .appendQueryParameter(col, "g")
+                .toString()
+    }
+
+    private fun getExportListUrl() = Uri.parse(baseUrl).buildUpon()
+            .appendPath("panel.php")
+            .appendQueryParameter("go", "export")
             .toString()
 
-    fun getListUrl(username: String) = Uri.parse(baseUrl).buildUpon()
+    private fun getListUrl(csrf: String): Observable<String> {
-            .appendPath("malappinfo.php")
+        return client.newCall(POST(url = getExportListUrl(), body = getExportPostBody(csrf)))
-            .appendQueryParameter("u", username)
+                .asObservable()
-            .appendQueryParameter("status", "all")
+                .map {response ->
-            .appendQueryParameter("type", "manga")
+                    baseUrl + Jsoup.parse(response.consumeBody())
+                            .select("div.goodresult")
+                            .select("a")
+                            .attr("href")
+                }
+    }
+
+    private fun getUpdateUrl() = Uri.parse(baseModifyListUrl).buildUpon()
+            .appendPath("edit.json")
             .toString()
 
-    fun getUpdateUrl(track: Track) = Uri.parse(baseUrl).buildUpon()
+    private fun getAddUrl() = Uri.parse(baseModifyListUrl).buildUpon()
-            .appendEncodedPath("api/mangalist/update")
+            .appendPath( "add.json")
-            .appendPath("${track.media_id}.xml")
             .toString()
     
-    fun getAddUrl(track: Track) = Uri.parse(baseUrl).buildUpon()
+    private fun Response.consumeBody(): String? {
-            .appendEncodedPath("api/mangalist/add")
+        use {
-            .appendPath("${track.media_id}.xml")
+            if (it.code() != 200) throw Exception("Login error")
-            .toString()
+            return it.body()?.string()
+        }
+    }
 
-    fun createHeaders(username: String, password: String): Headers {
+    private fun Response.consumeXmlBody(): String? {
-        return Headers.Builder()
+        use { res ->
-                .add("Authorization", Credentials.basic(username, password))
+            if (res.code() != 200) throw Exception("Export list error")
-                .add("User-Agent", "api-indiv-9F93C52A963974CF674325391990191C")
+            BufferedReader(InputStreamReader(GZIPInputStream(res.body()?.source()?.inputStream()))).use { reader ->
-                .build()
+                val sb = StringBuilder()
+                reader.forEachLine { line ->
+                    sb.append(line)
+                }
+                return sb.toString()
+            }
+        }
     }
 
     companion object {
         const val baseUrl = "https://myanimelist.net"
-        const val baseMangaUrl = baseUrl + "/manga/"
+        private const val baseMangaUrl = "$baseUrl/manga/"
+        private const val baseModifyListUrl = "$baseUrl/ownlist/manga"
 
-        fun mangaUrl(remoteId: Int): String {
+        fun mangaUrl(remoteId: Int) = baseMangaUrl + remoteId
-            return baseMangaUrl + remoteId
+
+        fun Element.searchTitle() = select("strong").text()!!
+
+        fun Element.searchTotalChapters() = if (select(TD)[4].text() == "-") 0 else select(TD)[4].text().toInt()
+
+        fun Element.searchCoverUrl() = select("img")
+                .attr("data-src")
+                .split("\\?")[0]
+                .replace("/r/50x70/", "/")
+
+        fun Element.searchMediaId() = select("div.picSurround")
+                .select("a").attr("id")
+                .replace("sarea", "")
+                .toInt()
+
+        fun Element.searchSummary() = select("div.pt4")
+                .first()
+                .ownText()!!
+
+        fun Element.searchPublishingStatus() = if (select(TD).last().text() == "-") PUBLISHING else FINISHED
+
+        fun Element.searchPublishingType() = select(TD)[2].text()!!
+
+        fun Element.searchStartDate() = select(TD)[6].text()!!
+
+        fun getStatus(status: String) = when (status) {
+            "Reading" -> 1
+            "Completed" -> 2
+            "On-Hold" -> 3
+            "Dropped" -> 4
+            "Plan to Read" -> 6
+            else -> 1
             }
 
-        private val ENTRY_TAG = "entry"
+        const val CSRF = "csrf_token"
-        private val CHAPTER_TAG = "chapter"
+        const val TD = "td"
-        private val SCORE_TAG = "score"
+        private const val FINISHED = "Finished"
-        private val STATUS_TAG = "status"
+        private const val PUBLISHING = "Publishing"
-
-        const val PREFIX_MY = "my:"
     }
 }

app/src/main/java/eu/kanade/tachiyomi/network/PersistentCookieStore.kt

@@ -60,6 +60,11 @@ class PersistentCookieStore(context: Context) {
         cookieMap.clear()
     }
 
+    fun remove(uri: URI) {
+        prefs.edit().remove(uri.host).apply()
+        cookieMap.remove(uri.host)
+    }
+
     fun get(url: HttpUrl) = get(url.uri().host)
 
     fun get(uri: URI) = get(uri.host)

app/src/main/java/eu/kanade/tachiyomi/widget/preference/TrackLoginDialog.kt

@@ -46,7 +46,6 @@ class TrackLoginDialog(bundle: Bundle? = null) : LoginDialogPreference(bundle) {
                         login.setText(R.string.unknown_error)
                         error.message?.let { context.toast(it) }
                     })
-
         }
     }