From a3463addc38c0a41beb4d6cd0f59e7653d065cdf Mon Sep 17 00:00:00 2001 From: inorichi Date: Tue, 6 Oct 2015 19:44:30 +0200 Subject: [PATCH] Get chapter list --- .../main/java/eu/kanade/mangafeed/App.java | 2 +- .../data/managers/ChapterManager.java | 65 +++++++++- .../kanade/mangafeed/data/models/Chapter.java | 26 ++-- .../kanade/mangafeed/data/models/Manga.java | 7 +- .../mangafeed/data/tables/ChaptersTable.java | 8 +- .../eu/kanade/mangafeed/sources/Batoto.java | 115 ++++-------------- 6 files changed, 109 insertions(+), 114 deletions(-) diff --git a/app/src/main/java/eu/kanade/mangafeed/App.java b/app/src/main/java/eu/kanade/mangafeed/App.java index 2b67fd6a5..6ea0fe4ce 100644 --- a/app/src/main/java/eu/kanade/mangafeed/App.java +++ b/app/src/main/java/eu/kanade/mangafeed/App.java @@ -29,7 +29,7 @@ public class App extends Application { .appModule(new AppModule(this)) .build(); - ACRA.init(this); + //ACRA.init(this); } public static App get(Context context) { diff --git a/app/src/main/java/eu/kanade/mangafeed/data/managers/ChapterManager.java b/app/src/main/java/eu/kanade/mangafeed/data/managers/ChapterManager.java index 162efa5d9..ab37c4dc4 100644 --- a/app/src/main/java/eu/kanade/mangafeed/data/managers/ChapterManager.java +++ b/app/src/main/java/eu/kanade/mangafeed/data/managers/ChapterManager.java @@ -1,7 +1,11 @@ package eu.kanade.mangafeed.data.managers; import com.pushtorefresh.storio.sqlite.StorIOSQLite; +import com.pushtorefresh.storio.sqlite.operations.delete.DeleteResult; +import com.pushtorefresh.storio.sqlite.operations.delete.DeleteResults; +import com.pushtorefresh.storio.sqlite.operations.get.PreparedGetListOfObjects; import com.pushtorefresh.storio.sqlite.operations.put.PutResult; +import com.pushtorefresh.storio.sqlite.operations.put.PutResults; import com.pushtorefresh.storio.sqlite.queries.Query; import java.util.List; @@ -17,7 +21,7 @@ public class ChapterManager extends BaseManager { super(db); } - public Observable> get(Manga manga) { + private PreparedGetListOfObjects prepareGet(Manga manga) { return db.get() .listOfObjects(Chapter.class) .withQuery(Query.builder() @@ -25,8 +29,11 @@ public class ChapterManager extends BaseManager { .where(ChaptersTable.COLUMN_MANGA_ID + "=?") .whereArgs(manga.id) .build()) - .prepare() - .createObservable(); + .prepare(); + } + + public Observable> get(Manga manga) { + return prepareGet(manga).createObservable(); } public Observable insert(Chapter chapter) { @@ -36,16 +43,66 @@ public class ChapterManager extends BaseManager { .createObservable(); } + public Observable> insert(List chapters) { + return db.put() + .objects(chapters) + .prepare() + .createObservable(); + } + + // Add new chapters or delete if the source deletes them + public Observable insertOrRemove(Manga manga, List chapters) { + // I don't know a better approach + return Observable.create(subscriber -> { + List dbGet = prepareGet(manga).executeAsBlocking(); + + Observable.just(dbGet) + .doOnNext(dbChapters -> { + Observable.from(chapters) + .filter(c -> !dbChapters.contains(c)) + .toList() + .subscribe(newChapters -> { + if (newChapters.size() > 0) + insert(newChapters).subscribe(); + }); + }) + .flatMap(Observable::from) + .filter(c -> !chapters.contains(c)) + .toList() + .subscribe(removedChapters -> { + if (removedChapters.size() > 0) + delete(removedChapters).subscribe(); + subscriber.onCompleted(); + }); + + }); + + } + public void createDummyChapters() { Chapter c; for (int i = 1; i < 100; i++) { c = new Chapter(); - c.manga_id = 1; + c.manga_id = 1L; c.name = "Chapter " + i; c.url = "http://example.com/1"; insert(c).subscribe(); } } + + public Observable> delete(List chapters) { + return db.delete() + .objects(chapters) + .prepare() + .createObservable(); + } + + public Observable delete(Chapter chapter) { + return db.delete() + .object(chapter) + .prepare() + .createObservable(); + } } diff --git a/app/src/main/java/eu/kanade/mangafeed/data/models/Chapter.java b/app/src/main/java/eu/kanade/mangafeed/data/models/Chapter.java index 030f7e8f4..5b3ad3655 100644 --- a/app/src/main/java/eu/kanade/mangafeed/data/models/Chapter.java +++ b/app/src/main/java/eu/kanade/mangafeed/data/models/Chapter.java @@ -17,7 +17,7 @@ public class Chapter { @NonNull @StorIOSQLiteColumn(name = ChaptersTable.COLUMN_MANGA_ID) - public int manga_id; + public Long manga_id; @NonNull @StorIOSQLiteColumn(name = ChaptersTable.COLUMN_URL) @@ -35,6 +35,10 @@ public class Chapter { @StorIOSQLiteColumn(name = ChaptersTable.COLUMN_DATE_FETCH) public long date_fetch; + @NonNull + @StorIOSQLiteColumn(name = ChaptersTable.COLUMN_DATE_UPLOAD) + public long date_upload; + public Chapter() {} @@ -45,23 +49,17 @@ public class Chapter { Chapter chapter = (Chapter) o; - if (manga_id != chapter.manga_id) return false; - if (read != chapter.read) return false; - if (date_fetch != chapter.date_fetch) return false; - if (id != null ? !id.equals(chapter.id) : chapter.id != null) return false; - if (!url.equals(chapter.url)) return false; - return name.equals(chapter.name); + return url.equals(chapter.url); } @Override public int hashCode() { - int result = id != null ? id.hashCode() : 0; - result = 31 * result + manga_id; - result = 31 * result + url.hashCode(); - result = 31 * result + name.hashCode(); - result = 31 * result + read; - result = 31 * result + (int) (date_fetch ^ (date_fetch >>> 32)); - return result; + return url.hashCode(); + } + + public static Chapter newChapter() { + Chapter c = new Chapter(); + return c; } } diff --git a/app/src/main/java/eu/kanade/mangafeed/data/models/Manga.java b/app/src/main/java/eu/kanade/mangafeed/data/models/Manga.java index d2a643016..252d57017 100644 --- a/app/src/main/java/eu/kanade/mangafeed/data/models/Manga.java +++ b/app/src/main/java/eu/kanade/mangafeed/data/models/Manga.java @@ -79,10 +79,9 @@ public class Manga { this.title = title; } - public Manga(long id, String title, String author, String artist, String url, + public Manga(String title, String author, String artist, String url, String description, String genre, String status, int rank, String thumbnail_url) { - this.id = id; this.title = title; this.author = author; this.artist = artist; @@ -94,10 +93,10 @@ public class Manga { this.thumbnail_url = thumbnail_url; } - public static Manga newManga(long id, String title, String author, String artist, String url, + public static Manga newManga(String title, String author, String artist, String url, String description, String genre, String status, int rank, String thumbnail_url) { - return new Manga(id, title, author, artist, url, description, genre, status, rank, thumbnail_url); + return new Manga(title, author, artist, url, description, genre, status, rank, thumbnail_url); } @Override diff --git a/app/src/main/java/eu/kanade/mangafeed/data/tables/ChaptersTable.java b/app/src/main/java/eu/kanade/mangafeed/data/tables/ChaptersTable.java index e11c40553..2145ac72e 100644 --- a/app/src/main/java/eu/kanade/mangafeed/data/tables/ChaptersTable.java +++ b/app/src/main/java/eu/kanade/mangafeed/data/tables/ChaptersTable.java @@ -2,9 +2,6 @@ package eu.kanade.mangafeed.data.tables; import android.support.annotation.NonNull; -/** - * Created by len on 23/09/2015. - */ public class ChaptersTable { @NonNull @@ -28,6 +25,9 @@ public class ChaptersTable { @NonNull public static final String COLUMN_DATE_FETCH = "date_fetch"; + @NonNull + public static final String COLUMN_DATE_UPLOAD = "date_upload"; + @NonNull public static String getCreateTableQuery() { return "CREATE TABLE " + TABLE + "(" @@ -37,8 +37,10 @@ public class ChaptersTable { + COLUMN_NAME + " TEXT NOT NULL, " + COLUMN_READ + " BOOLEAN NOT NULL, " + COLUMN_DATE_FETCH + " LONG NOT NULL, " + + COLUMN_DATE_UPLOAD + " LONG NOT NULL, " + "FOREIGN KEY(" + COLUMN_MANGA_ID + ") REFERENCES " + MangasTable.TABLE + "(" + MangasTable.COLUMN_ID + ") " + "ON DELETE CASCADE" + ");"; } + } diff --git a/app/src/main/java/eu/kanade/mangafeed/sources/Batoto.java b/app/src/main/java/eu/kanade/mangafeed/sources/Batoto.java index ead3da552..4916f0cf4 100644 --- a/app/src/main/java/eu/kanade/mangafeed/sources/Batoto.java +++ b/app/src/main/java/eu/kanade/mangafeed/sources/Batoto.java @@ -1,19 +1,27 @@ package eu.kanade.mangafeed.sources; import com.squareup.okhttp.Headers; +import com.squareup.okhttp.Response; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; +import java.text.ParseException; +import java.text.SimpleDateFormat; import java.util.ArrayList; +import java.util.Date; import java.util.List; +import java.util.Locale; import java.util.concurrent.atomic.AtomicInteger; import eu.kanade.mangafeed.data.caches.CacheManager; import eu.kanade.mangafeed.data.helpers.NetworkHelper; +import eu.kanade.mangafeed.data.models.Chapter; +import eu.kanade.mangafeed.data.models.Manga; import rx.Observable; +import rx.functions.Func1; import rx.schedulers.Schedulers; import timber.log.Timber; @@ -318,52 +326,35 @@ public class Batoto { return newManga; } + */ - public Observable> pullChaptersFromNetwork(final String mangaUrl, final String mangaName) { + public Observable> pullChaptersFromNetwork(String mangaUrl) { return mNetworkService - .getResponse(mangaUrl, NetworkModule.NULL_CACHE_CONTROL, REQUEST_HEADERS) - .flatMap(new Func1>() { - @Override - public Observable call(Response response) { - return mNetworkService.mapResponseToString(response); - } - }) - .flatMap(new Func1>>() { - @Override - public Observable> call(String unparsedHtml) { - return Observable.just(parseHtmlToChapters(mangaUrl, mangaName, unparsedHtml)); - } - }); + .getStringResponse(mangaUrl, mNetworkService.NULL_CACHE_CONTROL, REQUEST_HEADERS) + .flatMap(unparsedHtml -> + Observable.just(parseHtmlToChapters(unparsedHtml))); } - private List parseHtmlToChapters(String mangaUrl, String mangaName, String unparsedHtml) { + private List parseHtmlToChapters(String unparsedHtml) { Document parsedDocument = Jsoup.parse(unparsedHtml); - List chapterList = scrapeChaptersFromParsedDocument(parsedDocument); - chapterList = setSourceForChapterList(chapterList); - chapterList = setParentInfoForChapterList(chapterList, mangaUrl, mangaName); - chapterList = setNumberForChapterList(chapterList); - - saveChaptersToDatabase(chapterList, mangaUrl); - - return chapterList; - } - - private List scrapeChaptersFromParsedDocument(Document parsedDocument) { List chapterList = new ArrayList(); Elements chapterElements = parsedDocument.select("tr.row.lang_English.chapter_row"); for (Element chapterElement : chapterElements) { Chapter currentChapter = constructChapterFromHtmlBlock(chapterElement); - + System.out.println(currentChapter.name); chapterList.add(currentChapter); } + //saveChaptersToDatabase(chapterList, mangaUrl); + return chapterList; + } private Chapter constructChapterFromHtmlBlock(Element chapterElement) { - Chapter newChapter = DefaultFactory.Chapter.constructDefault(); + Chapter newChapter = Chapter.newChapter(); Element urlElement = chapterElement.select("a[href^=http://bato.to/read/").first(); Element nameElement = urlElement; @@ -371,16 +362,17 @@ public class Batoto { if (urlElement != null) { String fieldUrl = urlElement.attr("href"); - newChapter.setUrl(fieldUrl); + newChapter.url = fieldUrl; } if (nameElement != null) { String fieldName = nameElement.text().trim(); - newChapter.setName(fieldName); + newChapter.name = fieldName; } if (dateElement != null) { long fieldDate = parseDateFromElement(dateElement); - newChapter.setDate(fieldDate); + newChapter.date_upload = fieldDate; } + newChapter.date_fetch = new Date().getTime(); return newChapter; } @@ -396,77 +388,24 @@ public class Batoto { // Do Nothing. } - return DefaultFactory.Chapter.DEFAULT_DATE; + return 0; } - private List setSourceForChapterList(List chapterList) { - for (Chapter currentChapter : chapterList) { - currentChapter.setSource(NAME); - } - - return chapterList; - } - - private List setParentInfoForChapterList(List chapterList, String parentUrl, String parentName) { - for (Chapter currentChapter : chapterList) { - currentChapter.setParentUrl(parentUrl); - currentChapter.setParentName(parentName); - } - - return chapterList; - } - - private List setNumberForChapterList(List chapterList) { - Collections.reverse(chapterList); - for (int index = 0; index < chapterList.size(); index++) { - chapterList.get(index).setNumber(index + 1); - } - - return chapterList; - } - - private void saveChaptersToDatabase(List chapterList, String parentUrl) { - StringBuilder selection = new StringBuilder(); - List selectionArgs = new ArrayList(); - - selection.append(ApplicationContract.Chapter.COLUMN_SOURCE + " = ?"); - selectionArgs.add(NAME); - selection.append(" AND ").append(ApplicationContract.Chapter.COLUMN_PARENT_URL + " = ?"); - selectionArgs.add(parentUrl); - - mQueryManager.beginApplicationTransaction(); - try { - mQueryManager.deleteAllChapter(selection.toString(), selectionArgs.toArray(new String[selectionArgs.size()])) - .toBlocking() - .single(); - - for (Chapter currentChapter : chapterList) { - mQueryManager.createChapter(currentChapter) - .toBlocking() - .single(); - } - - mQueryManager.setApplicationTransactionSuccessful(); - } finally { - mQueryManager.endApplicationTransaction(); - } - } - */ - public Observable pullImageUrlsFromNetwork(final String chapterUrl) { final List temporaryCachedImageUrls = new ArrayList<>(); return mCacheManager.getImageUrlsFromDiskCache(chapterUrl) .onErrorResumeNext(throwable -> { return mNetworkService - .getStringResponse(chapterUrl, mNetworkService.NULL_CACHE_CONTROL, null) + .getStringResponse(chapterUrl, mNetworkService.NULL_CACHE_CONTROL, REQUEST_HEADERS) + .subscribeOn(Schedulers.io()) .flatMap(unparsedHtml -> Observable.from(parseHtmlToPageUrls(unparsedHtml))) .buffer(3) .concatMap(batchedPageUrls -> { List> imageUrlObservables = new ArrayList<>(); for (String pageUrl : batchedPageUrls) { Observable temporaryObservable = mNetworkService - .getStringResponse(pageUrl, mNetworkService.NULL_CACHE_CONTROL, null) + .getStringResponse(pageUrl, mNetworkService.NULL_CACHE_CONTROL, REQUEST_HEADERS) .flatMap(unparsedHtml -> Observable.just(parseHtmlToImageUrl(unparsedHtml))) .subscribeOn(Schedulers.io());