mirror of
https://github.com/mihonapp/mihon.git
synced 2024-11-21 20:47:03 -05:00
Get chapter list
This commit is contained in:
inorichi
parent
a78359e4a9
commit
a3463addc3
6 changed files with 109 additions and 114 deletions
|
|
@ -29,7 +29,7 @@ public class App extends Application {
|
|||
.appModule(new AppModule(this))
|
||||
.build();
|
||||
|
||||
ACRA.init(this);
|
||||
//ACRA.init(this);
|
||||
}
|
||||
|
||||
public static App get(Context context) {
|
||||
|
|
|
|||
|
|
@ -1,7 +1,11 @@
|
|||
package eu.kanade.mangafeed.data.managers;
|
||||
|
||||
import com.pushtorefresh.storio.sqlite.StorIOSQLite;
|
||||
import com.pushtorefresh.storio.sqlite.operations.delete.DeleteResult;
|
||||
import com.pushtorefresh.storio.sqlite.operations.delete.DeleteResults;
|
||||
import com.pushtorefresh.storio.sqlite.operations.get.PreparedGetListOfObjects;
|
||||
import com.pushtorefresh.storio.sqlite.operations.put.PutResult;
|
||||
import com.pushtorefresh.storio.sqlite.operations.put.PutResults;
|
||||
import com.pushtorefresh.storio.sqlite.queries.Query;
|
||||
|
||||
import java.util.List;
|
||||
|
|
@ -17,7 +21,7 @@ public class ChapterManager extends BaseManager {
|
|||
super(db);
|
||||
}
|
||||
|
||||
public Observable<List<Chapter>> get(Manga manga) {
|
||||
private PreparedGetListOfObjects<Chapter> prepareGet(Manga manga) {
|
||||
return db.get()
|
||||
.listOfObjects(Chapter.class)
|
||||
.withQuery(Query.builder()
|
||||
|
|
@ -25,8 +29,11 @@ public class ChapterManager extends BaseManager {
|
|||
.where(ChaptersTable.COLUMN_MANGA_ID + "=?")
|
||||
.whereArgs(manga.id)
|
||||
.build())
|
||||
.prepare()
|
||||
.createObservable();
|
||||
.prepare();
|
||||
}
|
||||
|
||||
public Observable<List<Chapter>> get(Manga manga) {
|
||||
return prepareGet(manga).createObservable();
|
||||
}
|
||||
|
||||
public Observable<PutResult> insert(Chapter chapter) {
|
||||
|
|
@ -36,16 +43,66 @@ public class ChapterManager extends BaseManager {
|
|||
.createObservable();
|
||||
}
|
||||
|
||||
public Observable<PutResults<Chapter>> insert(List<Chapter> chapters) {
|
||||
return db.put()
|
||||
.objects(chapters)
|
||||
.prepare()
|
||||
.createObservable();
|
||||
}
|
||||
|
||||
// Add new chapters or delete if the source deletes them
|
||||
public Observable insertOrRemove(Manga manga, List<Chapter> chapters) {
|
||||
// I don't know a better approach
|
||||
return Observable.create(subscriber -> {
|
||||
List<Chapter> dbGet = prepareGet(manga).executeAsBlocking();
|
||||
|
||||
Observable.just(dbGet)
|
||||
.doOnNext(dbChapters -> {
|
||||
Observable.from(chapters)
|
||||
.filter(c -> !dbChapters.contains(c))
|
||||
.toList()
|
||||
.subscribe(newChapters -> {
|
||||
if (newChapters.size() > 0)
|
||||
insert(newChapters).subscribe();
|
||||
});
|
||||
})
|
||||
.flatMap(Observable::from)
|
||||
.filter(c -> !chapters.contains(c))
|
||||
.toList()
|
||||
.subscribe(removedChapters -> {
|
||||
if (removedChapters.size() > 0)
|
||||
delete(removedChapters).subscribe();
|
||||
subscriber.onCompleted();
|
||||
});
|
||||
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
public void createDummyChapters() {
|
||||
Chapter c;
|
||||
|
||||
for (int i = 1; i < 100; i++) {
|
||||
c = new Chapter();
|
||||
c.manga_id = 1;
|
||||
c.manga_id = 1L;
|
||||
c.name = "Chapter " + i;
|
||||
c.url = "http://example.com/1";
|
||||
insert(c).subscribe();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public Observable<DeleteResults<Chapter>> delete(List<Chapter> chapters) {
|
||||
return db.delete()
|
||||
.objects(chapters)
|
||||
.prepare()
|
||||
.createObservable();
|
||||
}
|
||||
|
||||
public Observable<DeleteResult> delete(Chapter chapter) {
|
||||
return db.delete()
|
||||
.object(chapter)
|
||||
.prepare()
|
||||
.createObservable();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ public class Chapter {
|
|||
|
||||
@NonNull
|
||||
@StorIOSQLiteColumn(name = ChaptersTable.COLUMN_MANGA_ID)
|
||||
public int manga_id;
|
||||
public Long manga_id;
|
||||
|
||||
@NonNull
|
||||
@StorIOSQLiteColumn(name = ChaptersTable.COLUMN_URL)
|
||||
|
|
@ -35,6 +35,10 @@ public class Chapter {
|
|||
@StorIOSQLiteColumn(name = ChaptersTable.COLUMN_DATE_FETCH)
|
||||
public long date_fetch;
|
||||
|
||||
@NonNull
|
||||
@StorIOSQLiteColumn(name = ChaptersTable.COLUMN_DATE_UPLOAD)
|
||||
public long date_upload;
|
||||
|
||||
|
||||
public Chapter() {}
|
||||
|
||||
|
|
@ -45,23 +49,17 @@ public class Chapter {
|
|||
|
||||
Chapter chapter = (Chapter) o;
|
||||
|
||||
if (manga_id != chapter.manga_id) return false;
|
||||
if (read != chapter.read) return false;
|
||||
if (date_fetch != chapter.date_fetch) return false;
|
||||
if (id != null ? !id.equals(chapter.id) : chapter.id != null) return false;
|
||||
if (!url.equals(chapter.url)) return false;
|
||||
return name.equals(chapter.name);
|
||||
return url.equals(chapter.url);
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
int result = id != null ? id.hashCode() : 0;
|
||||
result = 31 * result + manga_id;
|
||||
result = 31 * result + url.hashCode();
|
||||
result = 31 * result + name.hashCode();
|
||||
result = 31 * result + read;
|
||||
result = 31 * result + (int) (date_fetch ^ (date_fetch >>> 32));
|
||||
return result;
|
||||
return url.hashCode();
|
||||
}
|
||||
|
||||
public static Chapter newChapter() {
|
||||
Chapter c = new Chapter();
|
||||
return c;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -79,10 +79,9 @@ public class Manga {
|
|||
this.title = title;
|
||||
}
|
||||
|
||||
public Manga(long id, String title, String author, String artist, String url,
|
||||
public Manga(String title, String author, String artist, String url,
|
||||
String description, String genre, String status, int rank,
|
||||
String thumbnail_url) {
|
||||
this.id = id;
|
||||
this.title = title;
|
||||
this.author = author;
|
||||
this.artist = artist;
|
||||
|
|
@ -94,10 +93,10 @@ public class Manga {
|
|||
this.thumbnail_url = thumbnail_url;
|
||||
}
|
||||
|
||||
public static Manga newManga(long id, String title, String author, String artist, String url,
|
||||
public static Manga newManga(String title, String author, String artist, String url,
|
||||
String description, String genre, String status, int rank,
|
||||
String thumbnail_url) {
|
||||
return new Manga(id, title, author, artist, url, description, genre, status, rank, thumbnail_url);
|
||||
return new Manga(title, author, artist, url, description, genre, status, rank, thumbnail_url);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
|||
|
|
@ -2,9 +2,6 @@ package eu.kanade.mangafeed.data.tables;
|
|||
|
||||
import android.support.annotation.NonNull;
|
||||
|
||||
/**
|
||||
* Created by len on 23/09/2015.
|
||||
*/
|
||||
public class ChaptersTable {
|
||||
|
||||
@NonNull
|
||||
|
|
@ -28,6 +25,9 @@ public class ChaptersTable {
|
|||
@NonNull
|
||||
public static final String COLUMN_DATE_FETCH = "date_fetch";
|
||||
|
||||
@NonNull
|
||||
public static final String COLUMN_DATE_UPLOAD = "date_upload";
|
||||
|
||||
@NonNull
|
||||
public static String getCreateTableQuery() {
|
||||
return "CREATE TABLE " + TABLE + "("
|
||||
|
|
@ -37,8 +37,10 @@ public class ChaptersTable {
|
|||
+ COLUMN_NAME + " TEXT NOT NULL, "
|
||||
+ COLUMN_READ + " BOOLEAN NOT NULL, "
|
||||
+ COLUMN_DATE_FETCH + " LONG NOT NULL, "
|
||||
+ COLUMN_DATE_UPLOAD + " LONG NOT NULL, "
|
||||
+ "FOREIGN KEY(" + COLUMN_MANGA_ID + ") REFERENCES " + MangasTable.TABLE + "(" + MangasTable.COLUMN_ID + ") "
|
||||
+ "ON DELETE CASCADE"
|
||||
+ ");";
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,19 +1,27 @@
|
|||
package eu.kanade.mangafeed.sources;
|
||||
|
||||
import com.squareup.okhttp.Headers;
|
||||
import com.squareup.okhttp.Response;
|
||||
|
||||
import org.jsoup.Jsoup;
|
||||
import org.jsoup.nodes.Document;
|
||||
import org.jsoup.nodes.Element;
|
||||
import org.jsoup.select.Elements;
|
||||
|
||||
import java.text.ParseException;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import eu.kanade.mangafeed.data.caches.CacheManager;
|
||||
import eu.kanade.mangafeed.data.helpers.NetworkHelper;
|
||||
import eu.kanade.mangafeed.data.models.Chapter;
|
||||
import eu.kanade.mangafeed.data.models.Manga;
|
||||
import rx.Observable;
|
||||
import rx.functions.Func1;
|
||||
import rx.schedulers.Schedulers;
|
||||
import timber.log.Timber;
|
||||
|
||||
|
|
@ -318,52 +326,35 @@ public class Batoto {
|
|||
|
||||
return newManga;
|
||||
}
|
||||
*/
|
||||
|
||||
public Observable<List<Chapter>> pullChaptersFromNetwork(final String mangaUrl, final String mangaName) {
|
||||
public Observable<List<Chapter>> pullChaptersFromNetwork(String mangaUrl) {
|
||||
return mNetworkService
|
||||
.getResponse(mangaUrl, NetworkModule.NULL_CACHE_CONTROL, REQUEST_HEADERS)
|
||||
.flatMap(new Func1<Response, Observable<String>>() {
|
||||
@Override
|
||||
public Observable<String> call(Response response) {
|
||||
return mNetworkService.mapResponseToString(response);
|
||||
}
|
||||
})
|
||||
.flatMap(new Func1<String, Observable<List<Chapter>>>() {
|
||||
@Override
|
||||
public Observable<List<Chapter>> call(String unparsedHtml) {
|
||||
return Observable.just(parseHtmlToChapters(mangaUrl, mangaName, unparsedHtml));
|
||||
}
|
||||
});
|
||||
.getStringResponse(mangaUrl, mNetworkService.NULL_CACHE_CONTROL, REQUEST_HEADERS)
|
||||
.flatMap(unparsedHtml ->
|
||||
Observable.just(parseHtmlToChapters(unparsedHtml)));
|
||||
}
|
||||
|
||||
private List<Chapter> parseHtmlToChapters(String mangaUrl, String mangaName, String unparsedHtml) {
|
||||
private List<Chapter> parseHtmlToChapters(String unparsedHtml) {
|
||||
Document parsedDocument = Jsoup.parse(unparsedHtml);
|
||||
|
||||
List<Chapter> chapterList = scrapeChaptersFromParsedDocument(parsedDocument);
|
||||
chapterList = setSourceForChapterList(chapterList);
|
||||
chapterList = setParentInfoForChapterList(chapterList, mangaUrl, mangaName);
|
||||
chapterList = setNumberForChapterList(chapterList);
|
||||
|
||||
saveChaptersToDatabase(chapterList, mangaUrl);
|
||||
|
||||
return chapterList;
|
||||
}
|
||||
|
||||
private List<Chapter> scrapeChaptersFromParsedDocument(Document parsedDocument) {
|
||||
List<Chapter> chapterList = new ArrayList<Chapter>();
|
||||
|
||||
Elements chapterElements = parsedDocument.select("tr.row.lang_English.chapter_row");
|
||||
for (Element chapterElement : chapterElements) {
|
||||
Chapter currentChapter = constructChapterFromHtmlBlock(chapterElement);
|
||||
|
||||
System.out.println(currentChapter.name);
|
||||
chapterList.add(currentChapter);
|
||||
}
|
||||
|
||||
//saveChaptersToDatabase(chapterList, mangaUrl);
|
||||
|
||||
return chapterList;
|
||||
|
||||
}
|
||||
|
||||
private Chapter constructChapterFromHtmlBlock(Element chapterElement) {
|
||||
Chapter newChapter = DefaultFactory.Chapter.constructDefault();
|
||||
Chapter newChapter = Chapter.newChapter();
|
||||
|
||||
Element urlElement = chapterElement.select("a[href^=http://bato.to/read/").first();
|
||||
Element nameElement = urlElement;
|
||||
|
|
@ -371,16 +362,17 @@ public class Batoto {
|
|||
|
||||
if (urlElement != null) {
|
||||
String fieldUrl = urlElement.attr("href");
|
||||
newChapter.setUrl(fieldUrl);
|
||||
newChapter.url = fieldUrl;
|
||||
}
|
||||
if (nameElement != null) {
|
||||
String fieldName = nameElement.text().trim();
|
||||
newChapter.setName(fieldName);
|
||||
newChapter.name = fieldName;
|
||||
}
|
||||
if (dateElement != null) {
|
||||
long fieldDate = parseDateFromElement(dateElement);
|
||||
newChapter.setDate(fieldDate);
|
||||
newChapter.date_upload = fieldDate;
|
||||
}
|
||||
newChapter.date_fetch = new Date().getTime();
|
||||
|
||||
return newChapter;
|
||||
}
|
||||
|
|
@ -396,77 +388,24 @@ public class Batoto {
|
|||
// Do Nothing.
|
||||
}
|
||||
|
||||
return DefaultFactory.Chapter.DEFAULT_DATE;
|
||||
return 0;
|
||||
}
|
||||
|
||||
private List<Chapter> setSourceForChapterList(List<Chapter> chapterList) {
|
||||
for (Chapter currentChapter : chapterList) {
|
||||
currentChapter.setSource(NAME);
|
||||
}
|
||||
|
||||
return chapterList;
|
||||
}
|
||||
|
||||
private List<Chapter> setParentInfoForChapterList(List<Chapter> chapterList, String parentUrl, String parentName) {
|
||||
for (Chapter currentChapter : chapterList) {
|
||||
currentChapter.setParentUrl(parentUrl);
|
||||
currentChapter.setParentName(parentName);
|
||||
}
|
||||
|
||||
return chapterList;
|
||||
}
|
||||
|
||||
private List<Chapter> setNumberForChapterList(List<Chapter> chapterList) {
|
||||
Collections.reverse(chapterList);
|
||||
for (int index = 0; index < chapterList.size(); index++) {
|
||||
chapterList.get(index).setNumber(index + 1);
|
||||
}
|
||||
|
||||
return chapterList;
|
||||
}
|
||||
|
||||
private void saveChaptersToDatabase(List<Chapter> chapterList, String parentUrl) {
|
||||
StringBuilder selection = new StringBuilder();
|
||||
List<String> selectionArgs = new ArrayList<String>();
|
||||
|
||||
selection.append(ApplicationContract.Chapter.COLUMN_SOURCE + " = ?");
|
||||
selectionArgs.add(NAME);
|
||||
selection.append(" AND ").append(ApplicationContract.Chapter.COLUMN_PARENT_URL + " = ?");
|
||||
selectionArgs.add(parentUrl);
|
||||
|
||||
mQueryManager.beginApplicationTransaction();
|
||||
try {
|
||||
mQueryManager.deleteAllChapter(selection.toString(), selectionArgs.toArray(new String[selectionArgs.size()]))
|
||||
.toBlocking()
|
||||
.single();
|
||||
|
||||
for (Chapter currentChapter : chapterList) {
|
||||
mQueryManager.createChapter(currentChapter)
|
||||
.toBlocking()
|
||||
.single();
|
||||
}
|
||||
|
||||
mQueryManager.setApplicationTransactionSuccessful();
|
||||
} finally {
|
||||
mQueryManager.endApplicationTransaction();
|
||||
}
|
||||
}
|
||||
*/
|
||||
|
||||
public Observable<String> pullImageUrlsFromNetwork(final String chapterUrl) {
|
||||
final List<String> temporaryCachedImageUrls = new ArrayList<>();
|
||||
|
||||
return mCacheManager.getImageUrlsFromDiskCache(chapterUrl)
|
||||
.onErrorResumeNext(throwable -> {
|
||||
return mNetworkService
|
||||
.getStringResponse(chapterUrl, mNetworkService.NULL_CACHE_CONTROL, null)
|
||||
.getStringResponse(chapterUrl, mNetworkService.NULL_CACHE_CONTROL, REQUEST_HEADERS)
|
||||
.subscribeOn(Schedulers.io())
|
||||
.flatMap(unparsedHtml -> Observable.from(parseHtmlToPageUrls(unparsedHtml)))
|
||||
.buffer(3)
|
||||
.concatMap(batchedPageUrls -> {
|
||||
List<Observable<String>> imageUrlObservables = new ArrayList<>();
|
||||
for (String pageUrl : batchedPageUrls) {
|
||||
Observable<String> temporaryObservable = mNetworkService
|
||||
.getStringResponse(pageUrl, mNetworkService.NULL_CACHE_CONTROL, null)
|
||||
.getStringResponse(pageUrl, mNetworkService.NULL_CACHE_CONTROL, REQUEST_HEADERS)
|
||||
.flatMap(unparsedHtml -> Observable.just(parseHtmlToImageUrl(unparsedHtml)))
|
||||
.subscribeOn(Schedulers.io());
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue