Don't store sources domain in database

This commit is contained in:
inorichi 2015-11-30 13:07:57 +01:00
parent a05cc934d0
commit 8bda39ee26
9 changed files with 140 additions and 112 deletions

View file

@ -4,6 +4,7 @@ import com.pushtorefresh.storio.sqlite.annotations.StorIOSQLiteColumn;
import com.pushtorefresh.storio.sqlite.annotations.StorIOSQLiteType;
import eu.kanade.mangafeed.data.database.tables.ChapterTable;
import eu.kanade.mangafeed.util.UrlUtil;
@StorIOSQLiteType(table = ChapterTable.TABLE)
public class Chapter {
@ -44,6 +45,10 @@ public class Chapter {
public Chapter() {}
public void setUrl(String url) {
this.url = UrlUtil.getPath(url);
}
@Override
public boolean equals(Object o) {
if (this == o) return true;

View file

@ -4,6 +4,7 @@ import com.pushtorefresh.storio.sqlite.annotations.StorIOSQLiteColumn;
import com.pushtorefresh.storio.sqlite.annotations.StorIOSQLiteType;
import eu.kanade.mangafeed.data.database.tables.MangaTable;
import eu.kanade.mangafeed.util.UrlUtil;
@StorIOSQLiteType(table = MangaTable.TABLE)
public class Manga {
@ -57,6 +58,10 @@ public class Manga {
public Manga() {}
public void setUrl(String url) {
this.url = UrlUtil.getPath(url);
}
public static void copyFromNetwork(Manga local, Manga network) {
if (network.title != null)
local.title = network.title;

View file

@ -20,6 +20,9 @@ public abstract class BaseSource {
// Id of the source (must be declared and obtained from SourceManager to avoid conflicts)
public abstract int getSourceId();
// Base url of the source, like: http://example.com
public abstract String getBaseUrl();
// True if the source requires a login
public abstract boolean isLoginRequired();
@ -76,7 +79,7 @@ public abstract class BaseSource {
}
// Get the URL of the first page that contains a source image and the page list
protected String overrideChapterPageUrl(String defaultPageUrl) {
protected String overrideChapterUrl(String defaultPageUrl) {
return defaultPageUrl;
}

View file

@ -25,14 +25,14 @@ import rx.schedulers.Schedulers;
public abstract class Source extends BaseSource {
@Inject protected NetworkHelper mNetworkService;
@Inject protected CacheManager mCacheManager;
@Inject protected NetworkHelper networkService;
@Inject protected CacheManager cacheManager;
@Inject protected PreferencesHelper prefs;
protected Headers mRequestHeaders;
protected Headers requestHeaders;
public Source(Context context) {
App.get(context).getComponent().inject(this);
mRequestHeaders = headersBuilder().build();
requestHeaders = headersBuilder().build();
}
// Get the most popular mangas from the source
@ -40,8 +40,8 @@ public abstract class Source extends BaseSource {
if (page.page == 1)
page.url = getInitialPopularMangasUrl();
return mNetworkService
.getStringResponse(page.url, mRequestHeaders, null)
return networkService
.getStringResponse(page.url, requestHeaders, null)
.map(Jsoup::parse)
.doOnNext(doc -> page.mangas = parsePopularMangasFromHtml(doc))
.doOnNext(doc -> page.nextPageUrl = parseNextPopularMangasUrl(doc, page))
@ -53,8 +53,8 @@ public abstract class Source extends BaseSource {
if (page.page == 1)
page.url = getInitialSearchUrl(query);
return mNetworkService
.getStringResponse(page.url, mRequestHeaders, null)
return networkService
.getStringResponse(page.url, requestHeaders, null)
.map(Jsoup::parse)
.doOnNext(doc -> page.mangas = parseSearchFromHtml(doc))
.doOnNext(doc -> page.nextPageUrl = parseNextSearchUrl(doc, page, query))
@ -63,21 +63,21 @@ public abstract class Source extends BaseSource {
// Get manga details from the source
public Observable<Manga> pullMangaFromNetwork(final String mangaUrl) {
return mNetworkService
.getStringResponse(overrideMangaUrl(mangaUrl), mRequestHeaders, null)
return networkService
.getStringResponse(getBaseUrl() + overrideMangaUrl(mangaUrl), requestHeaders, null)
.flatMap(unparsedHtml -> Observable.just(parseHtmlToManga(mangaUrl, unparsedHtml)));
}
// Get chapter list of a manga from the source
public Observable<List<Chapter>> pullChaptersFromNetwork(String mangaUrl) {
return mNetworkService
.getStringResponse(mangaUrl, mRequestHeaders, null)
public Observable<List<Chapter>> pullChaptersFromNetwork(final String mangaUrl) {
return networkService
.getStringResponse(getBaseUrl() + mangaUrl, requestHeaders, null)
.flatMap(unparsedHtml ->
Observable.just(parseHtmlToChapters(unparsedHtml)));
}
public Observable<List<Page>> getCachedPageListOrPullFromNetwork(final String chapterUrl) {
return mCacheManager.getPageUrlsFromDiskCache(chapterUrl)
return cacheManager.getPageUrlsFromDiskCache(getChapterCacheKey(chapterUrl))
.onErrorResumeNext(throwable -> {
return pullPageListFromNetwork(chapterUrl);
})
@ -85,8 +85,8 @@ public abstract class Source extends BaseSource {
}
public Observable<List<Page>> pullPageListFromNetwork(final String chapterUrl) {
return mNetworkService
.getStringResponse(overrideChapterPageUrl(chapterUrl), mRequestHeaders, null)
return networkService
.getStringResponse(getBaseUrl() + overrideChapterUrl(chapterUrl), requestHeaders, null)
.flatMap(unparsedHtml -> {
List<String> pageUrls = parseHtmlToPageUrls(unparsedHtml);
return Observable.just(getFirstImageFromPageUrls(pageUrls, unparsedHtml));
@ -102,8 +102,8 @@ public abstract class Source extends BaseSource {
public Observable<Page> getImageUrlFromPage(final Page page) {
page.setStatus(Page.LOAD_PAGE);
return mNetworkService
.getStringResponse(overridePageUrl(page.getUrl()), mRequestHeaders, null)
return networkService
.getStringResponse(overridePageUrl(page.getUrl()), requestHeaders, null)
.flatMap(unparsedHtml -> Observable.just(parseHtmlToImageUrl(unparsedHtml)))
.onErrorResumeNext(e -> {
page.setStatus(Page.ERROR);
@ -123,13 +123,13 @@ public abstract class Source extends BaseSource {
return pageObservable
.flatMap(p -> {
if (!mCacheManager.isImageInCache(page.getImageUrl())) {
if (!cacheManager.isImageInCache(page.getImageUrl())) {
return cacheImage(page);
}
return Observable.just(page);
})
.flatMap(p -> {
page.setImagePath(mCacheManager.getImagePath(page.getImageUrl()));
page.setImagePath(cacheManager.getImagePath(page.getImageUrl()));
page.setStatus(Page.READY);
return Observable.just(page);
})
@ -143,7 +143,7 @@ public abstract class Source extends BaseSource {
page.setStatus(Page.DOWNLOAD_IMAGE);
return getImageProgressResponse(page)
.flatMap(resp -> {
if (!mCacheManager.putImageToDiskCache(page.getImageUrl(), resp)) {
if (!cacheManager.putImageToDiskCache(page.getImageUrl(), resp)) {
throw new IllegalStateException("Unable to save image");
}
return Observable.just(page);
@ -151,12 +151,12 @@ public abstract class Source extends BaseSource {
}
public Observable<Response> getImageProgressResponse(final Page page) {
return mNetworkService.getProgressResponse(page.getImageUrl(), mRequestHeaders, page);
return networkService.getProgressResponse(page.getImageUrl(), requestHeaders, page);
}
public void savePageList(String chapterUrl, List<Page> pages) {
if (pages != null)
mCacheManager.putPageUrlsToDiskCache(chapterUrl, pages);
cacheManager.putPageUrlsToDiskCache(getChapterCacheKey(chapterUrl), pages);
}
protected List<Page> convertToPages(List<String> pageUrls) {
@ -174,4 +174,8 @@ public abstract class Source extends BaseSource {
return pages;
}
protected String getChapterCacheKey(String chapteUrl) {
return getSourceId() + chapteUrl;
}
}

View file

@ -1,6 +1,7 @@
package eu.kanade.mangafeed.data.source.online.english;
import android.content.Context;
import android.net.Uri;
import com.squareup.okhttp.FormEncodingBuilder;
import com.squareup.okhttp.Headers;
@ -33,13 +34,11 @@ public class Batoto extends Source {
public static final String NAME = "Batoto (EN)";
public static final String BASE_URL = "http://bato.to";
public static final String INITIAL_POPULAR_MANGAS_URL =
"http://bato.to/search_ajax?order_cond=views&order=desc&p=";
public static final String INITIAL_SEARCH_URL = "http://bato.to/search_ajax?";
public static final String INITIAL_PAGE_URL = "http://bato.to/areader?";
public static final String LOGIN_URL =
"https://bato.to/forums/index.php?app=core&module=global&section=login";
public static final String POPULAR_MANGAS_URL = BASE_URL + "/search_ajax?order_cond=views&order=desc&p=%d";
public static final String SEARCH_URL = BASE_URL + "/search_ajax?name=%s&p=%s";
public static final String PAGE_URL = BASE_URL + "/areader?id=%s&p=%s";
public static final String MANGA_URL = "/comic_pop?id=%s";
public static final String LOGIN_URL = BASE_URL + "/forums/index.php?app=core&module=global&section=login";
public Batoto(Context context) {
super(context);
@ -50,6 +49,16 @@ public class Batoto extends Source {
return NAME;
}
@Override
public int getSourceId() {
return SourceManager.BATOTO;
}
@Override
public String getBaseUrl() {
return BASE_URL;
}
@Override
protected Headers.Builder headersBuilder() {
Headers.Builder builder = super.headersBuilder();
@ -103,11 +112,6 @@ public class Batoto extends Source {
return Observable.just(genres);
}
@Override
public int getSourceId() {
return SourceManager.BATOTO;
}
@Override
public boolean isLoginRequired() {
return true;
@ -115,24 +119,24 @@ public class Batoto extends Source {
@Override
public String getInitialPopularMangasUrl() {
return INITIAL_POPULAR_MANGAS_URL + "1";
return String.format(POPULAR_MANGAS_URL, 1);
}
@Override
public String getInitialSearchUrl(String query) {
return INITIAL_SEARCH_URL + "name=" + query + "&p=1";
return String.format(SEARCH_URL, Uri.encode(query), 1);
}
@Override
protected String overrideMangaUrl(String defaultMangaUrl) {
String mangaId = defaultMangaUrl.substring(defaultMangaUrl.lastIndexOf("r") + 1);
return "http://bato.to/comic_pop?id=" + mangaId;
return String.format(MANGA_URL, mangaId);
}
@Override
protected String overrideChapterPageUrl(String defaultPageUrl) {
protected String overrideChapterUrl(String defaultPageUrl) {
String id = defaultPageUrl.substring(defaultPageUrl.indexOf("#") + 1);
return INITIAL_PAGE_URL + "id=" + id + "&p=1";
return String.format(PAGE_URL, id, 1);
}
@Override
@ -140,7 +144,7 @@ public class Batoto extends Source {
int start = defaultPageUrl.indexOf("#") + 1;
int end = defaultPageUrl.indexOf("_", start);
String id = defaultPageUrl.substring(start, end);
return INITIAL_PAGE_URL + "id=" + id + "&p=" + defaultPageUrl.substring(end+1);
return String.format(PAGE_URL, id, defaultPageUrl.substring(end+1));
}
@Override
@ -167,7 +171,7 @@ public class Batoto extends Source {
if (next == null)
return null;
return INITIAL_POPULAR_MANGAS_URL + (page.page + 1);
return String.format(POPULAR_MANGAS_URL, page.page + 1);
}
@Override
@ -192,22 +196,16 @@ public class Batoto extends Source {
Manga mangaFromHtmlBlock = new Manga();
Element urlElement = htmlBlock.select("a[href^=http://bato.to]").first();
Element nameElement = urlElement;
Element updateElement = htmlBlock.select("td").get(5);
mangaFromHtmlBlock.source = getSourceId();
if (urlElement != null) {
String fieldUrl = urlElement.attr("href");
mangaFromHtmlBlock.url = fieldUrl;
}
if (nameElement != null) {
String fieldName = nameElement.text().trim();
mangaFromHtmlBlock.title = fieldName;
mangaFromHtmlBlock.setUrl(urlElement.attr("href"));
mangaFromHtmlBlock.title = urlElement.text().trim();
}
if (updateElement != null) {
long fieldUpdate = parseUpdateFromElement(updateElement);
mangaFromHtmlBlock.last_update = fieldUpdate;
mangaFromHtmlBlock.last_update = parseUpdateFromElement(updateElement);
}
return mangaFromHtmlBlock;
@ -219,7 +217,7 @@ public class Batoto extends Source {
if (next == null)
return null;
return INITIAL_SEARCH_URL + "name=" + query + "&p=" + (page.page + 1);
return String.format(SEARCH_URL, query, page.page + 1);
}
private long parseUpdateFromElement(Element updateElement) {
@ -257,8 +255,7 @@ public class Batoto extends Source {
}
}
if (descriptionElement != null) {
String fieldDescription = descriptionElement.text().substring("Description:".length()).trim();
newManga.description = fieldDescription;
newManga.description = descriptionElement.text().substring("Description:".length()).trim();
}
if (genreElements != null) {
String fieldGenres = "";
@ -274,8 +271,7 @@ public class Batoto extends Source {
newManga.genre = fieldGenres;
}
if (thumbnailUrlElement != null) {
String fieldThumbnailUrl = thumbnailUrlElement.attr("src");
newManga.thumbnail_url = fieldThumbnailUrl;
newManga.thumbnail_url = thumbnailUrlElement.attr("src");
}
boolean fieldCompleted = unparsedHtml.contains("<td>Complete</td>");
@ -309,20 +305,16 @@ public class Batoto extends Source {
Chapter newChapter = Chapter.create();
Element urlElement = chapterElement.select("a[href^=http://bato.to/reader").first();
Element nameElement = urlElement;
Element dateElement = chapterElement.select("td").get(4);
if (urlElement != null) {
String fieldUrl = urlElement.attr("href");
newChapter.url = fieldUrl;
}
if (nameElement != null) {
String fieldName = nameElement.text().trim();
newChapter.name = fieldName;
newChapter.setUrl(fieldUrl);
newChapter.name = urlElement.text().trim();
}
if (dateElement != null) {
long fieldDate = parseDateFromElement(dateElement);
newChapter.date_upload = fieldDate;
newChapter.date_upload = parseDateFromElement(dateElement);
}
newChapter.date_fetch = new Date().getTime();
@ -401,7 +393,7 @@ public class Batoto extends Source {
@Override
public Observable<Boolean> login(String username, String password) {
return mNetworkService.getStringResponse(LOGIN_URL, mRequestHeaders, null)
return networkService.getStringResponse(LOGIN_URL, requestHeaders, null)
.flatMap(response -> doLogin(response, username, password))
.map(this::isAuthenticationSuccessful);
}
@ -420,7 +412,7 @@ public class Batoto extends Source {
formBody.add("invisible", "1");
formBody.add("rememberMe", "1");
return mNetworkService.postData(postUrl, formBody.build(), mRequestHeaders);
return networkService.postData(postUrl, formBody.build(), requestHeaders);
}
@Override
@ -431,7 +423,7 @@ public class Batoto extends Source {
@Override
public boolean isLogged() {
try {
for ( HttpCookie cookie : mNetworkService.getCookies().get(new URI(BASE_URL)) ) {
for ( HttpCookie cookie : networkService.getCookies().get(new URI(BASE_URL)) ) {
if (cookie.getName().equals("pass_hash"))
return true;
}

View file

@ -1,6 +1,7 @@
package eu.kanade.mangafeed.data.source.online.english;
import android.content.Context;
import android.net.Uri;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
@ -25,9 +26,9 @@ public class Mangafox extends Source {
public static final String NAME = "Mangafox (EN)";
public static final String BASE_URL = "http://mangafox.me";
public static final String INITIAL_POPULAR_MANGAS_URL = "http://mangafox.me/directory/";
public static final String INITIAL_SEARCH_URL =
"http://mangafox.me/search.php?name_method=cw&advopts=1&order=az&sort=name";
public static final String POPULAR_MANGAS_URL = BASE_URL + "/directory/%s";
public static final String SEARCH_URL =
BASE_URL + "/search.php?name_method=cw&advopts=1&order=az&sort=name&name=%s&page=%s";
public Mangafox(Context context) {
super(context);
@ -43,6 +44,11 @@ public class Mangafox extends Source {
return SourceManager.MANGAFOX;
}
@Override
public String getBaseUrl() {
return BASE_URL;
}
@Override
public boolean isLoginRequired() {
return false;
@ -50,12 +56,12 @@ public class Mangafox extends Source {
@Override
protected String getInitialPopularMangasUrl() {
return INITIAL_POPULAR_MANGAS_URL;
return String.format(POPULAR_MANGAS_URL, "");
}
@Override
protected String getInitialSearchUrl(String query) {
return INITIAL_SEARCH_URL + "&name=" + query + "&page=1";
return String.format(SEARCH_URL, Uri.encode(query), 1);
}
@Override
@ -78,7 +84,7 @@ public class Mangafox extends Source {
Element urlElement = htmlBlock.select("a.title").first();
if (urlElement != null) {
mangaFromHtmlBlock.url = urlElement.attr("href");
mangaFromHtmlBlock.setUrl(urlElement.attr("href"));
mangaFromHtmlBlock.title = urlElement.text();
}
@ -91,7 +97,7 @@ public class Mangafox extends Source {
if (next == null)
return null;
return INITIAL_POPULAR_MANGAS_URL + next.attr("href");
return String.format(POPULAR_MANGAS_URL, next.attr("href"));
}
@Override
@ -114,7 +120,7 @@ public class Mangafox extends Source {
Element urlElement = htmlBlock.select("a.series_preview").first();
if (urlElement != null) {
mangaFromHtmlBlock.url = urlElement.attr("href");
mangaFromHtmlBlock.setUrl(urlElement.attr("href"));
mangaFromHtmlBlock.title = urlElement.text();
}
@ -153,24 +159,19 @@ public class Mangafox extends Source {
newManga.title = title;
}
if (artistElement != null) {
String fieldArtist = artistElement.text();
newManga.artist = fieldArtist;
newManga.artist = artistElement.text();
}
if (authorElement != null) {
String fieldAuthor = authorElement.text();
newManga.author = fieldAuthor;
newManga.author = authorElement.text();
}
if (descriptionElement != null) {
String fieldDescription = descriptionElement.text();
newManga.description = fieldDescription;
newManga.description = descriptionElement.text();
}
if (genreElement != null) {
String fieldGenre = genreElement.text();
newManga.genre = fieldGenre;
newManga.genre = genreElement.text();
}
if (thumbnailUrlElement != null) {
String fieldThumbnailUrl = thumbnailUrlElement.attr("src");
newManga.thumbnail_url = fieldThumbnailUrl;
newManga.thumbnail_url = thumbnailUrlElement.attr("src");
}
// if (statusElement != null) {
// boolean fieldCompleted = statusElement.text().contains("Completed");
@ -206,7 +207,7 @@ public class Mangafox extends Source {
Element dateElement = chapterElement.select("span.date").first();
if (urlElement != null) {
newChapter.url = urlElement.attr("href");
newChapter.setUrl(urlElement.attr("href"));
}
if (nameElement != null) {
newChapter.name = nameElement.text();

View file

@ -1,6 +1,7 @@
package eu.kanade.mangafeed.data.source.online.english;
import android.content.Context;
import android.net.Uri;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
@ -26,9 +27,8 @@ public class Mangahere extends Source {
public static final String NAME = "Mangahere (EN)";
public static final String BASE_URL = "http://www.mangahere.co";
private static final String INITIAL_POPULAR_MANGAS_URL = "http://www.mangahere.co/directory/";
private static final String INITIAL_SEARCH_URL = "http://www.mangahere.co/search.php?";
public static final String POPULAR_MANGAS_URL = BASE_URL + "/directory/%s";
public static final String SEARCH_URL = BASE_URL + "/search.php?name=%s&page=%s";
public Mangahere(Context context) {
super(context);
@ -44,6 +44,11 @@ public class Mangahere extends Source {
return SourceManager.MANGAHERE;
}
@Override
public String getBaseUrl() {
return BASE_URL;
}
@Override
public boolean isLoginRequired() {
return false;
@ -51,12 +56,12 @@ public class Mangahere extends Source {
@Override
protected String getInitialPopularMangasUrl() {
return INITIAL_POPULAR_MANGAS_URL;
return String.format(POPULAR_MANGAS_URL, "");
}
@Override
protected String getInitialSearchUrl(String query) {
return INITIAL_SEARCH_URL + "name=" + query + "&page=1";
return String.format(SEARCH_URL, Uri.encode(query), 1);
}
public Observable<List<String>> getGenres() {
@ -117,7 +122,7 @@ public class Mangahere extends Source {
Element urlElement = htmlBlock.select("div.title > a").first();
if (urlElement != null) {
mangaFromHtmlBlock.url = urlElement.attr("href");
mangaFromHtmlBlock.setUrl(urlElement.attr("href"));
mangaFromHtmlBlock.title = urlElement.attr("title");
}
@ -130,7 +135,7 @@ public class Mangahere extends Source {
if (next == null)
return null;
return INITIAL_POPULAR_MANGAS_URL + next.attr("href");
return String.format(POPULAR_MANGAS_URL, next.attr("href"));
}
@Override
@ -153,7 +158,7 @@ public class Mangahere extends Source {
Element urlElement = htmlBlock.select("a.manga_info").first();
if (urlElement != null) {
mangaFromHtmlBlock.url = urlElement.attr("href");
mangaFromHtmlBlock.setUrl(urlElement.attr("href"));
mangaFromHtmlBlock.title = urlElement.text();
}
@ -231,20 +236,16 @@ public class Mangahere extends Source {
newManga.url = mangaUrl;
if (artistElement != null) {
String fieldArtist = artistElement.text();
newManga.artist = fieldArtist;
newManga.artist = artistElement.text();
}
if (authorElement != null) {
String fieldAuthor = authorElement.text();
newManga.author = fieldAuthor;
newManga.author = authorElement.text();
}
if (descriptionElement != null) {
String fieldDescription = descriptionElement.text().substring(0, descriptionElement.text().length() - "Show less".length());
newManga.description = fieldDescription;
newManga.description = descriptionElement.text().substring(0, descriptionElement.text().length() - "Show less".length());
}
if (genreElement != null) {
String fieldGenre = genreElement.text().substring("Genre(s):".length());
newManga.genre = fieldGenre;
newManga.genre = genreElement.text().substring("Genre(s):".length());
}
if (statusElement != null) {
boolean fieldCompleted = statusElement.text().contains("Completed");
@ -258,8 +259,7 @@ public class Mangahere extends Source {
Element thumbnailUrlElement = parsedDocument.select("img").first();
if (thumbnailUrlElement != null) {
String fieldThumbnailUrl = thumbnailUrlElement.attr("src");
newManga.thumbnail_url = fieldThumbnailUrl;
newManga.thumbnail_url = thumbnailUrlElement.attr("src");
}
newManga.initialized = true;
@ -295,16 +295,13 @@ public class Mangahere extends Source {
Element dateElement = chapterElement.select("span.right").first();
if (urlElement != null) {
String fieldUrl = urlElement.attr("href");
newChapter.url = fieldUrl;
newChapter.setUrl(urlElement.attr("href"));
}
if (nameElement != null) {
String fieldName = nameElement.text();
newChapter.name = fieldName;
newChapter.name = nameElement.text();
}
if (dateElement != null) {
long fieldDate = parseDateFromElement(dateElement);
newChapter.date_upload = fieldDate;
newChapter.date_upload = parseDateFromElement(dateElement);
}
newChapter.date_fetch = new Date().getTime();

View file

@ -0,0 +1,21 @@
package eu.kanade.mangafeed.util;
import java.net.URI;
import java.net.URISyntaxException;
public class UrlUtil {
public static String getPath(String s) {
try {
URI uri = new URI(s);
String out = uri.getPath();
if (uri.getQuery() != null)
out += "?" + uri.getQuery();
if (uri.getFragment() != null)
out += "#" + uri.getFragment();
return out;
} catch (URISyntaxException e) {
return s;
}
}
}

View file

@ -7,7 +7,7 @@
<TextView
android:layout_width="match_parent"
android:layout_height="40dp"
android:layout_height="50dp"
tools:text="New Text"
android:id="@+id/source_name"
android:gravity="center_vertical"