github/tachiyomi
Archived
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Improve sources' paging

Browse source
This commit is contained in:
inorichi 2015-11-15 17:51:14 +01:00
parent f63b7c8141
commit a055cc07d8
9 changed files with 186 additions and 82 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
app/src/main/java/eu/kanade/mangafeed/data/source/SourceManager.java
View file

@ -6,10 +6,10 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import eu.kanade.mangafeed.data.source.online.english.Batoto;
import eu.kanade.mangafeed.data.source.online.english.Mangahere;
import eu.kanade.mangafeed.data.source.online.english.Mangafox;
import eu.kanade.mangafeed.data.source.base.Source;
import eu.kanade.mangafeed.data.source.online.english.Batoto;
import eu.kanade.mangafeed.data.source.online.english.Mangafox;
import eu.kanade.mangafeed.data.source.online.english.Mangahere;
public class SourceManager {

25
app/src/main/java/eu/kanade/mangafeed/data/source/base/BaseSource.java
View file

@ -3,10 +3,13 @@ package eu.kanade.mangafeed.data.source.base;
import com.squareup.okhttp.Headers;
import com.squareup.okhttp.Response;
import org.jsoup.nodes.Document;
import java.util.List;
import eu.kanade.mangafeed.data.database.models.Chapter;
import eu.kanade.mangafeed.data.database.models.Manga;
import eu.kanade.mangafeed.data.source.model.MangasPage;
import rx.Observable;
public abstract class BaseSource {
@ -20,17 +23,23 @@ public abstract class BaseSource {
// True if the source requires a login
public abstract boolean isLoginRequired();
// Given a page number, it should return the URL of the page where the manga list is found
protected abstract String getUrlFromPageNumber(int page);
// Return the initial popular mangas URL
protected abstract String getInitialPopularMangasUrl();
// From the URL obtained before, this method must return a list of mangas
protected abstract List<Manga> parsePopularMangasFromHtml(String unparsedHtml);
// Return the initial search url given a query
protected abstract String getInitialSearchUrl(String query);
// Given a query and a page number, return the URL of the results
protected abstract String getSearchUrl(String query, int page);
// Get the popular list of mangas from the source's parsed document
protected abstract List<Manga> parsePopularMangasFromHtml(Document parsedHtml);
// From the URL obtained before, this method must return a list of mangas
protected abstract List<Manga> parseSearchFromHtml(String unparsedHtml);
// Get the next popular page URL or null if it's the last
protected abstract String parseNextPopularMangasUrl(Document parsedHtml, MangasPage page);
// Get the searched list of mangas from the source's parsed document
protected abstract List<Manga> parseSearchFromHtml(Document parsedHtml);
// Get the next search page URL or null if it's the last
protected abstract String parseNextSearchUrl(Document parsedHtml, MangasPage page, String query);
// Given the URL of a manga and the result of the request, return the details of the manga
protected abstract Manga parseHtmlToManga(String mangaUrl, String unparsedHtml);

33
app/src/main/java/eu/kanade/mangafeed/data/source/base/Source.java
View file

@ -1,11 +1,12 @@
package eu.kanade.mangafeed.data.source.base;
import android.content.Context;
import com.squareup.okhttp.Headers;
import com.squareup.okhttp.Response;
import org.jsoup.Jsoup;
import java.util.ArrayList;
import java.util.List;
@ -13,10 +14,11 @@ import javax.inject.Inject;
import eu.kanade.mangafeed.App;
import eu.kanade.mangafeed.data.cache.CacheManager;
import eu.kanade.mangafeed.data.network.NetworkHelper;
import eu.kanade.mangafeed.data.preference.PreferencesHelper;
import eu.kanade.mangafeed.data.database.models.Chapter;
import eu.kanade.mangafeed.data.database.models.Manga;
import eu.kanade.mangafeed.data.network.NetworkHelper;
import eu.kanade.mangafeed.data.preference.PreferencesHelper;
import eu.kanade.mangafeed.data.source.model.MangasPage;
import eu.kanade.mangafeed.data.source.model.Page;
import rx.Observable;
import rx.schedulers.Schedulers;
@ -34,18 +36,29 @@ public abstract class Source extends BaseSource {
}
// Get the most popular mangas from the source
public Observable<List<Manga>> pullPopularMangasFromNetwork(int page) {
String url = getUrlFromPageNumber(page);
public Observable<MangasPage> pullPopularMangasFromNetwork(MangasPage page) {
if (page.page == 1)
page.url = getInitialPopularMangasUrl();
return mNetworkService
.getStringResponse(url, mRequestHeaders, null)
.flatMap(response -> Observable.just(parsePopularMangasFromHtml(response)));
.getStringResponse(page.url, mRequestHeaders, null)
.map(Jsoup::parse)
.doOnNext(doc -> page.mangas = parsePopularMangasFromHtml(doc))
.doOnNext(doc -> page.nextPageUrl = parseNextPopularMangasUrl(doc, page))
.map(response -> page);
}
// Get mangas from the source with a query
public Observable<List<Manga>> searchMangasFromNetwork(String query, int page) {
public Observable<MangasPage> searchMangasFromNetwork(MangasPage page, String query) {
if (page.page == 1)
page.url = getInitialSearchUrl(query);
return mNetworkService
.getStringResponse(getSearchUrl(query, page), mRequestHeaders, null)
.flatMap(response -> Observable.just(parseSearchFromHtml(response)));
.getStringResponse(page.url, mRequestHeaders, null)
.map(Jsoup::parse)
.doOnNext(doc -> page.mangas = parseSearchFromHtml(doc))
.doOnNext(doc -> page.nextPageUrl = parseNextSearchUrl(doc, page, query))
.map(response -> page);
}
// Get manga details from the source

18
app/src/main/java/eu/kanade/mangafeed/data/source/model/MangasPage.java Normal file
View file

@ -0,0 +1,18 @@
package eu.kanade.mangafeed.data.source.model;
import java.util.List;
import eu.kanade.mangafeed.data.database.models.Manga;
public class MangasPage {
public List<Manga> mangas;
public int page;
public String url;
public String nextPageUrl;
public MangasPage(int page) {
this.page = page;
}
}

60
app/src/main/java/eu/kanade/mangafeed/data/source/online/english/Batoto.java
View file

@ -25,13 +25,14 @@ import eu.kanade.mangafeed.data.source.SourceManager;
import eu.kanade.mangafeed.data.database.models.Chapter;
import eu.kanade.mangafeed.data.database.models.Manga;
import eu.kanade.mangafeed.data.source.base.Source;
import eu.kanade.mangafeed.data.source.model.MangasPage;
import rx.Observable;
public class Batoto extends Source {
public static final String NAME = "Batoto (EN)";
public static final String BASE_URL = "http://bato.to";
public static final String INITIAL_UPDATE_URL =
public static final String INITIAL_POPULAR_MANGAS_URL =
"http://bato.to/search_ajax?order_cond=views&order=desc&p=";
public static final String INITIAL_SEARCH_URL = "http://bato.to/search_ajax?";
public static final String INITIAL_PAGE_URL = "http://bato.to/areader?";
@ -112,13 +113,13 @@ public class Batoto extends Source {
}
@Override
protected String getUrlFromPageNumber(int page) {
return INITIAL_UPDATE_URL + page;
public String getInitialPopularMangasUrl() {
return INITIAL_POPULAR_MANGAS_URL + "1";
}
@Override
protected String getSearchUrl(String query, int page) {
return INITIAL_SEARCH_URL + "name=" + query + "&p=" + page;
public String getInitialSearchUrl(String query) {
return INITIAL_SEARCH_URL + "name=" + query + "&p=1";
}
@Override
@ -141,33 +142,49 @@ public class Batoto extends Source {
return INITIAL_PAGE_URL + "id=" + id + "&p=" + defaultPageUrl.substring(end+1);
}
private List<Manga> parseMangasFromHtml(String unparsedHtml) {
if (unparsedHtml.contains("No (more) comics found!")) {
@Override
protected List<Manga> parsePopularMangasFromHtml(Document parsedHtml) {
if (parsedHtml.text().contains("No (more) comics found!")) {
return new ArrayList<>();
}
Document parsedDocument = Jsoup.parse(unparsedHtml);
List<Manga> mangaList = new ArrayList<>();
List<Manga> updatedMangaList = new ArrayList<>();
Elements updatedHtmlBlocks = parsedDocument.select("tr:not([id]):not([class])");
Elements updatedHtmlBlocks = parsedHtml.select("tr:not([id]):not([class])");
for (Element currentHtmlBlock : updatedHtmlBlocks) {
Manga currentlyUpdatedManga = constructMangaFromHtmlBlock(currentHtmlBlock);
updatedMangaList.add(currentlyUpdatedManga);
mangaList.add(currentlyUpdatedManga);
}
return updatedMangaList;
return mangaList;
}
@Override
public List<Manga> parsePopularMangasFromHtml(String unparsedHtml) {
return parseMangasFromHtml(unparsedHtml);
protected String parseNextPopularMangasUrl(Document parsedHtml, MangasPage page) {
Element next = parsedHtml.select("#show_more_row").first();
if (next == null)
return null;
return INITIAL_POPULAR_MANGAS_URL + (page.page + 1);
}
@Override
protected List<Manga> parseSearchFromHtml(String unparsedHtml) {
return parseMangasFromHtml(unparsedHtml);
protected List<Manga> parseSearchFromHtml(Document parsedHtml) {
if (parsedHtml.text().contains("No (more) comics found!")) {
return new ArrayList<>();
}
List<Manga> mangaList = new ArrayList<>();
Elements updatedHtmlBlocks = parsedHtml.select("tr:not([id]):not([class])");
for (Element currentHtmlBlock : updatedHtmlBlocks) {
Manga currentlyUpdatedManga = constructMangaFromHtmlBlock(currentHtmlBlock);
mangaList.add(currentlyUpdatedManga);
}
return mangaList;
}
private Manga constructMangaFromHtmlBlock(Element htmlBlock) {
@ -195,6 +212,15 @@ public class Batoto extends Source {
return mangaFromHtmlBlock;
}
@Override
protected String parseNextSearchUrl(Document parsedHtml, MangasPage page, String query) {
Element next = parsedHtml.select("#show_more_row").first();
if (next == null)
return null;
return INITIAL_SEARCH_URL + "name=" + query + "&p=" + (page.page + 1);
}
private long parseUpdateFromElement(Element updateElement) {
String updatedDateAsString = updateElement.text();

43
app/src/main/java/eu/kanade/mangafeed/data/source/online/english/Mangafox.java
View file

@ -14,13 +14,14 @@ import eu.kanade.mangafeed.data.source.SourceManager;
import eu.kanade.mangafeed.data.database.models.Chapter;
import eu.kanade.mangafeed.data.database.models.Manga;
import eu.kanade.mangafeed.data.source.base.Source;
import eu.kanade.mangafeed.data.source.model.MangasPage;
public class Mangafox extends Source {
public static final String NAME = "Mangafox (EN)";
private static final String INITIAL_UPDATE_URL = "http://mangafox.me/directory/";
private static final String INITIAL_SEARCH_URL =
public static final String BASE_URL = "http://mangafox.me";
public static final String INITIAL_POPULAR_MANGAS_URL = "http://mangafox.me/directory/";
public static final String INITIAL_SEARCH_URL =
"http://mangafox.me/search.php?name_method=cw&advopts=1&order=az&sort=name";
public Mangafox(Context context) {
@ -43,17 +44,20 @@ public class Mangafox extends Source {
}
@Override
protected String getUrlFromPageNumber(int page) {
return INITIAL_UPDATE_URL + page + ".htm";
protected String getInitialPopularMangasUrl() {
return INITIAL_POPULAR_MANGAS_URL;
}
@Override
public List<Manga> parsePopularMangasFromHtml(String unparsedHtml) {
Document parsedDocument = Jsoup.parse(unparsedHtml);
protected String getInitialSearchUrl(String query) {
return INITIAL_SEARCH_URL + "&name=" + query + "&page=1";
}
@Override
protected List<Manga> parsePopularMangasFromHtml(Document parsedHtml) {
List<Manga> mangaList = new ArrayList<>();
Elements mangaHtmlBlocks = parsedDocument.select("div#mangalist > ul.list > li");
Elements mangaHtmlBlocks = parsedHtml.select("div#mangalist > ul.list > li");
for (Element currentHtmlBlock : mangaHtmlBlocks) {
Manga currentManga = constructPopularMangaFromHtmlBlock(currentHtmlBlock);
mangaList.add(currentManga);
@ -77,17 +81,19 @@ public class Mangafox extends Source {
}
@Override
protected String getSearchUrl(String query, int page) {
return INITIAL_SEARCH_URL + "&name=" + query + "&page=" + page;
protected String parseNextPopularMangasUrl(Document parsedHtml, MangasPage page) {
Element next = parsedHtml.select("a:has(span.next)").first();
if (next == null)
return null;
return INITIAL_POPULAR_MANGAS_URL + next.attr("href");
}
@Override
protected List<Manga> parseSearchFromHtml(String unparsedHtml) {
Document parsedDocument = Jsoup.parse(unparsedHtml);
protected List<Manga> parseSearchFromHtml(Document parsedHtml) {
List<Manga> mangaList = new ArrayList<>();
Elements mangaHtmlBlocks = parsedDocument.select("table#listing > tbody > tr:gt(0)");
Elements mangaHtmlBlocks = parsedHtml.select("table#listing > tbody > tr:gt(0)");
for (Element currentHtmlBlock : mangaHtmlBlocks) {
Manga currentManga = constructSearchMangaFromHtmlBlock(currentHtmlBlock);
mangaList.add(currentManga);
@ -110,6 +116,15 @@ public class Mangafox extends Source {
return mangaFromHtmlBlock;
}
@Override
protected String parseNextSearchUrl(Document parsedHtml, MangasPage page, String query) {
Element next = parsedHtml.select("a:has(span.next)").first();
if (next == null)
return null;
return BASE_URL + next.attr("href");
}
@Override
protected Manga parseHtmlToManga(String mangaUrl, String unparsedHtml) {
Document parsedDocument = Jsoup.parse(unparsedHtml);

56
app/src/main/java/eu/kanade/mangafeed/data/source/online/english/Mangahere.java
View file

@ -15,18 +15,19 @@ import java.util.Date;
import java.util.List;
import java.util.Locale;
import eu.kanade.mangafeed.data.source.SourceManager;
import eu.kanade.mangafeed.data.database.models.Chapter;
import eu.kanade.mangafeed.data.database.models.Manga;
import eu.kanade.mangafeed.data.source.SourceManager;
import eu.kanade.mangafeed.data.source.base.Source;
import eu.kanade.mangafeed.data.source.model.MangasPage;
import rx.Observable;
public class Mangahere extends Source {
public static final String NAME = "Mangahere (EN)";
public static final String BASE_URL = "www.mangahere.co";
public static final String BASE_URL = "http://www.mangahere.co";
private static final String INITIAL_UPDATE_URL = "http://www.mangahere.co/directory/";
private static final String INITIAL_POPULAR_MANGAS_URL = "http://www.mangahere.co/directory/";
private static final String INITIAL_SEARCH_URL = "http://www.mangahere.co/search.php?";
public Mangahere(Context context) {
@ -43,22 +44,21 @@ public class Mangahere extends Source {
return SourceManager.MANGAHERE;
}
@Override
protected String getUrlFromPageNumber(int page) {
return INITIAL_UPDATE_URL + page + ".htm?views.za";
}
@Override
protected String getSearchUrl(String query, int page) {
return INITIAL_SEARCH_URL + "name=" + query + "&page=" + page;
}
@Override
public boolean isLoginRequired() {
return false;
}
@Override
protected String getInitialPopularMangasUrl() {
return INITIAL_POPULAR_MANGAS_URL;
}
@Override
protected String getInitialSearchUrl(String query) {
return INITIAL_SEARCH_URL + "name=" + query + "&page=1";
}
public Observable<List<String>> getGenres() {
List<String> genres = new ArrayList<>(30);
@ -98,12 +98,10 @@ public class Mangahere extends Source {
}
@Override
public List<Manga> parsePopularMangasFromHtml(String unparsedHtml) {
Document parsedDocument = Jsoup.parse(unparsedHtml);
public List<Manga> parsePopularMangasFromHtml(Document parsedHtml) {
List<Manga> mangaList = new ArrayList<>();
Elements mangaHtmlBlocks = parsedDocument.select("div.directory_list > ul > li");
Elements mangaHtmlBlocks = parsedHtml.select("div.directory_list > ul > li");
for (Element currentHtmlBlock : mangaHtmlBlocks) {
Manga currentManga = constructPopularMangaFromHtmlBlock(currentHtmlBlock);
mangaList.add(currentManga);
@ -127,12 +125,19 @@ public class Mangahere extends Source {
}
@Override
protected List<Manga> parseSearchFromHtml(String unparsedHtml) {
Document parsedDocument = Jsoup.parse(unparsedHtml);
protected String parseNextPopularMangasUrl(Document parsedHtml, MangasPage page) {
Element next = parsedHtml.select("div.next-page > a.next").first();
if (next == null)
return null;
return INITIAL_POPULAR_MANGAS_URL + next.attr("href");
}
@Override
protected List<Manga> parseSearchFromHtml(Document parsedHtml) {
List<Manga> mangaList = new ArrayList<>();
Elements mangaHtmlBlocks = parsedDocument.select("div.result_search > dl");
Elements mangaHtmlBlocks = parsedHtml.select("div.result_search > dl");
for (Element currentHtmlBlock : mangaHtmlBlocks) {
Manga currentManga = constructSearchMangaFromHtmlBlock(currentHtmlBlock);
mangaList.add(currentManga);
@ -155,6 +160,15 @@ public class Mangahere extends Source {
return mangaFromHtmlBlock;
}
@Override
protected String parseNextSearchUrl(Document parsedHtml, MangasPage page, String query) {
Element next = parsedHtml.select("div.next-page > a.next").first();
if (next == null)
return null;
return BASE_URL + next.attr("href");
}
private long parseUpdateFromElement(Element updateElement) {
String updatedDateAsString = updateElement.text();

2
app/src/main/java/eu/kanade/mangafeed/ui/catalogue/CatalogueFragment.java
View file

@ -22,8 +22,8 @@ import butterknife.ButterKnife;
import butterknife.OnItemClick;
import eu.kanade.mangafeed.R;
import eu.kanade.mangafeed.data.database.models.Manga;
import eu.kanade.mangafeed.ui.manga.MangaActivity;
import eu.kanade.mangafeed.ui.base.fragment.BaseRxFragment;
import eu.kanade.mangafeed.ui.manga.MangaActivity;
import eu.kanade.mangafeed.util.PageBundle;
import eu.kanade.mangafeed.widget.EndlessScrollListener;
import nucleus.factory.RequiresPresenter;

25
app/src/main/java/eu/kanade/mangafeed/ui/catalogue/CataloguePresenter.java
View file

@ -10,9 +10,10 @@ import java.util.concurrent.TimeUnit;
import javax.inject.Inject;
import eu.kanade.mangafeed.data.database.DatabaseHelper;
import eu.kanade.mangafeed.data.source.SourceManager;
import eu.kanade.mangafeed.data.database.models.Manga;
import eu.kanade.mangafeed.data.source.SourceManager;
import eu.kanade.mangafeed.data.source.base.Source;
import eu.kanade.mangafeed.data.source.model.MangasPage;
import eu.kanade.mangafeed.ui.base.presenter.BasePresenter;
import eu.kanade.mangafeed.util.PageBundle;
import eu.kanade.mangafeed.util.RxPager;
@ -36,6 +37,7 @@ public class CataloguePresenter extends BasePresenter<CatalogueFragment> {
private int mCurrentPage;
private RxPager pager;
private MangasPage lastMangasPage;
private Subscription mQueryDebouncerSubscription;
private Subscription mMangaDetailFetchSubscription;
@ -91,21 +93,28 @@ public class CataloguePresenter extends BasePresenter<CatalogueFragment> {
}
public void requestNext() {
if (getView() != null)
getView().showGridProgressBar();
pager.requestNext(++mCurrentPage);
}
private Observable<List<Manga>> getMangaObs(int page) {
Observable<List<Manga>> obs;
MangasPage nextMangasPage = new MangasPage(page);
if (page != 1) {
if (lastMangasPage.nextPageUrl == null)
return Observable.empty();
nextMangasPage.url = lastMangasPage.nextPageUrl;
}
if (getView() != null)
getView().showGridProgressBar();
Observable<MangasPage> obs;
if (mSearchMode)
obs = selectedSource.searchMangasFromNetwork(mSearchName, page);
obs = selectedSource.searchMangasFromNetwork(nextMangasPage, mSearchName);
else
obs = selectedSource.pullPopularMangasFromNetwork(page);
obs = selectedSource.pullPopularMangasFromNetwork(nextMangasPage);
return obs.subscribeOn(Schedulers.io())
.flatMap(Observable::from)
.doOnNext(mangasPage -> lastMangasPage = mangasPage)
.flatMap(mangasPage -> Observable.from(mangasPage.mangas))
.map(this::networkToLocalManga)
.toList();
}