From 986572f6cb57bd6091576ff7ecfb5f97017a6ef1 Mon Sep 17 00:00:00 2001 From: inorichi Date: Fri, 1 Jan 2016 21:02:13 +0100 Subject: [PATCH] Create an util class to write less code on sources. Save status from sources. --- .../mangafeed/data/database/models/Manga.java | 11 + .../mangafeed/data/network/NetworkHelper.java | 4 +- .../data/source/base/LoginSource.java | 17 ++ .../mangafeed/data/source/base/Source.java | 12 +- .../data/source/online/english/Batoto.java | 234 +++++------------- .../data/source/online/english/Kissmanga.java | 137 ++++------ .../data/source/online/english/Mangafox.java | 124 +++------- .../data/source/online/english/Mangahere.java | 173 ++++--------- .../myanimelist/MyAnimeListPresenter.java | 1 + .../java/eu/kanade/mangafeed/util/Parser.java | 48 ++++ 10 files changed, 297 insertions(+), 464 deletions(-) create mode 100644 app/src/main/java/eu/kanade/mangafeed/data/source/base/LoginSource.java create mode 100644 app/src/main/java/eu/kanade/mangafeed/util/Parser.java diff --git a/app/src/main/java/eu/kanade/mangafeed/data/database/models/Manga.java b/app/src/main/java/eu/kanade/mangafeed/data/database/models/Manga.java index d76c60c94..7342262dd 100644 --- a/app/src/main/java/eu/kanade/mangafeed/data/database/models/Manga.java +++ b/app/src/main/java/eu/kanade/mangafeed/data/database/models/Manga.java @@ -60,8 +60,19 @@ public class Manga implements Serializable { public int category; + public static final int UNKNOWN = 0; + public static final int ONGOING = 1; + public static final int COMPLETED = 2; + public static final int LICENSED = 3; + public Manga() {} + public static Manga create(String pathUrl) { + Manga m = new Manga(); + m.url = pathUrl; + return m; + } + public void setUrl(String url) { this.url = UrlUtil.getPath(url); } diff --git a/app/src/main/java/eu/kanade/mangafeed/data/network/NetworkHelper.java b/app/src/main/java/eu/kanade/mangafeed/data/network/NetworkHelper.java index 8260209df..90a7d6b3e 100644 --- a/app/src/main/java/eu/kanade/mangafeed/data/network/NetworkHelper.java +++ b/app/src/main/java/eu/kanade/mangafeed/data/network/NetworkHelper.java @@ -2,6 +2,7 @@ package eu.kanade.mangafeed.data.network; import com.squareup.okhttp.CacheControl; +import com.squareup.okhttp.FormEncodingBuilder; import com.squareup.okhttp.Headers; import com.squareup.okhttp.OkHttpClient; import com.squareup.okhttp.Request; @@ -21,6 +22,7 @@ public final class NetworkHelper { public final CacheControl NULL_CACHE_CONTROL = new CacheControl.Builder().noCache().build(); public final Headers NULL_HEADERS = new Headers.Builder().build(); + public final RequestBody NULL_REQUEST_BODY = new FormEncodingBuilder().build(); public NetworkHelper() { client = new OkHttpClient(); @@ -65,7 +67,7 @@ public final class NetworkHelper { try { Request request = new Request.Builder() .url(url) - .post(formBody) + .post(formBody != null ? formBody : NULL_REQUEST_BODY) .headers(headers != null ? headers : NULL_HEADERS) .build(); return Observable.just(client.newCall(request).execute()); diff --git a/app/src/main/java/eu/kanade/mangafeed/data/source/base/LoginSource.java b/app/src/main/java/eu/kanade/mangafeed/data/source/base/LoginSource.java new file mode 100644 index 000000000..b4bd7474c --- /dev/null +++ b/app/src/main/java/eu/kanade/mangafeed/data/source/base/LoginSource.java @@ -0,0 +1,17 @@ +package eu.kanade.mangafeed.data.source.base; + +import android.content.Context; + +public abstract class LoginSource extends Source { + + public LoginSource() {} + + public LoginSource(Context context) { + super(context); + } + + @Override + public boolean isLoginRequired() { + return true; + } +} diff --git a/app/src/main/java/eu/kanade/mangafeed/data/source/base/Source.java b/app/src/main/java/eu/kanade/mangafeed/data/source/base/Source.java index 31792685b..d811e77a9 100644 --- a/app/src/main/java/eu/kanade/mangafeed/data/source/base/Source.java +++ b/app/src/main/java/eu/kanade/mangafeed/data/source/base/Source.java @@ -42,6 +42,11 @@ public abstract class Source extends BaseSource { glideHeaders = glideHeadersBuilder().build(); } + @Override + public boolean isLoginRequired() { + return false; + } + // Get the most popular mangas from the source public Observable pullPopularMangasFromNetwork(MangasPage page) { if (page.page == 1) @@ -95,8 +100,8 @@ public abstract class Source extends BaseSource { return networkService .getStringResponse(getBaseUrl() + overrideChapterUrl(chapterUrl), requestHeaders, null) .flatMap(unparsedHtml -> { - List pageUrls = parseHtmlToPageUrls(unparsedHtml); - return Observable.just(getFirstImageFromPageUrls(pageUrls, unparsedHtml)); + List pages = convertToPages(parseHtmlToPageUrls(unparsedHtml)); + return Observable.just(parseFirstPage(pages, unparsedHtml)); }); } @@ -182,8 +187,7 @@ public abstract class Source extends BaseSource { return pages; } - protected List getFirstImageFromPageUrls(List pageUrls, String unparsedHtml) { - List pages = convertToPages(pageUrls); + protected List parseFirstPage(List pages, String unparsedHtml) { String firstImage = parseHtmlToImageUrl(unparsedHtml); pages.get(0).setImageUrl(firstImage); return pages; diff --git a/app/src/main/java/eu/kanade/mangafeed/data/source/online/english/Batoto.java b/app/src/main/java/eu/kanade/mangafeed/data/source/online/english/Batoto.java index 5837534db..8b4b8e0fb 100644 --- a/app/src/main/java/eu/kanade/mangafeed/data/source/online/english/Batoto.java +++ b/app/src/main/java/eu/kanade/mangafeed/data/source/online/english/Batoto.java @@ -2,6 +2,7 @@ package eu.kanade.mangafeed.data.source.online.english; import android.content.Context; import android.net.Uri; +import android.text.TextUtils; import com.squareup.okhttp.FormEncodingBuilder; import com.squareup.okhttp.Headers; @@ -30,12 +31,13 @@ import java.util.regex.Pattern; import eu.kanade.mangafeed.data.database.models.Chapter; import eu.kanade.mangafeed.data.database.models.Manga; import eu.kanade.mangafeed.data.source.SourceManager; -import eu.kanade.mangafeed.data.source.base.Source; +import eu.kanade.mangafeed.data.source.base.LoginSource; import eu.kanade.mangafeed.data.source.model.MangasPage; import eu.kanade.mangafeed.data.source.model.Page; +import eu.kanade.mangafeed.util.Parser; import rx.Observable; -public class Batoto extends Source { +public class Batoto extends LoginSource { public static final String NAME = "Batoto (EN)"; public static final String BASE_URL = "http://bato.to"; @@ -87,56 +89,6 @@ public class Batoto extends Source { return builder; } - public Observable> getGenres() { - List genres = new ArrayList<>(38); - - genres.add("4-Koma"); - genres.add("Action"); - genres.add("Adventure"); - genres.add("Award Winning"); - genres.add("Comedy"); - genres.add("Cooking"); - genres.add("Doujinshi"); - genres.add("Drama"); - genres.add("Ecchi"); - genres.add("Fantasy"); - genres.add("Gender Bender"); - genres.add("Harem"); - genres.add("Historical"); - genres.add("Horror"); - genres.add("Josei"); - genres.add("Martial Arts"); - genres.add("Mecha"); - genres.add("Medical"); - genres.add("Music"); - genres.add("Mystery"); - genres.add("One Shot"); - genres.add("Psychological"); - genres.add("Romance"); - genres.add("School Life"); - genres.add("Sci-fi"); - genres.add("Seinen"); - genres.add("Shoujo"); - genres.add("Shoujo Ai"); - genres.add("Shounen"); - genres.add("Shounen Ai"); - genres.add("Slice of Life"); - genres.add("Smut"); - genres.add("Sports"); - genres.add("Supernatural"); - genres.add("Tragedy"); - genres.add("Webtoon"); - genres.add("Yaoi"); - genres.add("Yuri"); - - return Observable.just(genres); - } - - @Override - public boolean isLoginRequired() { - return true; - } - @Override public String getInitialPopularMangasUrl() { return String.format(POPULAR_MANGAS_URL, 1); @@ -167,140 +119,88 @@ public class Batoto extends Source { return String.format(PAGE_URL, id, defaultPageUrl.substring(end+1)); } - @Override - protected List parsePopularMangasFromHtml(Document parsedHtml) { - if (parsedHtml.text().contains("No (more) comics found!")) { - return new ArrayList<>(); - } - + private List parseMangasFromHtml(Document parsedHtml) { List mangaList = new ArrayList<>(); - Elements updatedHtmlBlocks = parsedHtml.select("tr:not([id]):not([class])"); - for (Element currentHtmlBlock : updatedHtmlBlocks) { - Manga currentlyUpdatedManga = constructMangaFromHtmlBlock(currentHtmlBlock); - - mangaList.add(currentlyUpdatedManga); + if (!parsedHtml.text().contains("No (more) comics found!")) { + for (Element currentHtmlBlock : parsedHtml.select("tr:not([id]):not([class])")) { + Manga manga = constructMangaFromHtmlBlock(currentHtmlBlock); + mangaList.add(manga); + } } - return mangaList; } + @Override + protected List parsePopularMangasFromHtml(Document parsedHtml) { + return parseMangasFromHtml(parsedHtml); + } + @Override protected String parseNextPopularMangasUrl(Document parsedHtml, MangasPage page) { - Element next = parsedHtml.select("#show_more_row").first(); - if (next == null) - return null; - - return String.format(POPULAR_MANGAS_URL, page.page + 1); + Element next = Parser.element(parsedHtml, "#show_more_row"); + return next != null ? String.format(POPULAR_MANGAS_URL, page.page + 1) : null; } @Override protected List parseSearchFromHtml(Document parsedHtml) { - if (parsedHtml.text().contains("No (more) comics found!")) { - return new ArrayList<>(); - } - - List mangaList = new ArrayList<>(); - - Elements updatedHtmlBlocks = parsedHtml.select("tr:not([id]):not([class])"); - for (Element currentHtmlBlock : updatedHtmlBlocks) { - Manga currentlyUpdatedManga = constructMangaFromHtmlBlock(currentHtmlBlock); - - mangaList.add(currentlyUpdatedManga); - } - - return mangaList; + return parseMangasFromHtml(parsedHtml); } private Manga constructMangaFromHtmlBlock(Element htmlBlock) { - Manga mangaFromHtmlBlock = new Manga(); - - Element urlElement = htmlBlock.select("a[href^=http://bato.to]").first(); - Element updateElement = htmlBlock.select("td").get(5); - - mangaFromHtmlBlock.source = getId(); + Manga manga = new Manga(); + manga.source = getId(); + Element urlElement = Parser.element(htmlBlock, "a[href^=http://bato.to]"); if (urlElement != null) { - mangaFromHtmlBlock.setUrl(urlElement.attr("href")); - mangaFromHtmlBlock.title = urlElement.text().trim(); + manga.setUrl(urlElement.attr("href")); + manga.title = urlElement.text().trim(); } - if (updateElement != null) { - mangaFromHtmlBlock.last_update = parseUpdateFromElement(updateElement); - } - - return mangaFromHtmlBlock; + return manga; } @Override protected String parseNextSearchUrl(Document parsedHtml, MangasPage page, String query) { - Element next = parsedHtml.select("#show_more_row").first(); - if (next == null) - return null; - - return String.format(SEARCH_URL, query, page.page + 1); - } - - private long parseUpdateFromElement(Element updateElement) { - String updatedDateAsString = updateElement.text(); - - try { - Date specificDate = new SimpleDateFormat("dd MMMMM yyyy - hh:mm a", Locale.ENGLISH).parse(updatedDateAsString); - - return specificDate.getTime(); - } catch (ParseException e) { - // Do Nothing. - } - - return 0; + Element next = Parser.element(parsedHtml, "#show_more_row"); + return next != null ? String.format(SEARCH_URL, query, page.page + 1) : null; } @Override protected Manga parseHtmlToManga(String mangaUrl, String unparsedHtml) { Document parsedDocument = Jsoup.parse(unparsedHtml); - Elements artistElements = parsedDocument.select("a[href^=http://bato.to/search?artist_name]"); - Element descriptionElement = parsedDocument.select("tr").get(5); - Elements genreElements = parsedDocument.select("img[src=http://bato.to/forums/public/style_images/master/bullet_black.png]"); - Element thumbnailUrlElement = parsedDocument.select("img[src^=http://img.bato.to/forums/uploads/]").first(); + Element tbody = parsedDocument.select("tbody").first(); + Element artistElement = tbody.select("tr:contains(Author/Artist:)").first(); + Elements genreElements = tbody.select("tr:contains(Genres:) img"); - Manga newManga = new Manga(); - newManga.url = mangaUrl; + Manga manga = Manga.create(mangaUrl); + manga.author = Parser.text(artistElement, "td:eq(1)"); + manga.artist = Parser.text(artistElement, "td:eq(2)", manga.author); + manga.description = Parser.text(tbody, "tr:contains(Description:) > td:eq(1)"); + manga.thumbnail_url = Parser.src(parsedDocument, "img[src^=http://img.bato.to/forums/uploads/]"); + manga.status = parseStatus(Parser.text(parsedDocument, "tr:contains(Status:) > td:eq(1)")); - if (artistElements != null) { - newManga.author = artistElements.get(0).text(); - if (artistElements.size() > 1) { - newManga.artist = artistElements.get(1).text(); - } else { - newManga.artist = newManga.author; + if (!genreElements.isEmpty()) { + List genres = new ArrayList<>(); + for (Element element : genreElements) { + genres.add(element.attr("alt")); } - } - if (descriptionElement != null) { - newManga.description = descriptionElement.text().substring("Description:".length()).trim(); - } - if (genreElements != null) { - String fieldGenres = ""; - for (int index = 0; index < genreElements.size(); index++) { - String currentGenre = genreElements.get(index).attr("alt"); - - if (index < genreElements.size() - 1) { - fieldGenres += currentGenre + ", "; - } else { - fieldGenres += currentGenre; - } - } - newManga.genre = fieldGenres; - } - if (thumbnailUrlElement != null) { - newManga.thumbnail_url = thumbnailUrlElement.attr("src"); + manga.genre = TextUtils.join(", ", genres); } - boolean fieldCompleted = unparsedHtml.contains("Complete"); - //TODO fix - newManga.status = 0; + manga.initialized = true; + return manga; + } - newManga.initialized = true; - - return newManga; + private int parseStatus(String status) { + switch (status) { + case "Ongoing": + return Manga.ONGOING; + case "Complete": + return Manga.COMPLETED; + default: + return Manga.UNKNOWN; + } } @Override @@ -311,34 +211,30 @@ public class Batoto extends Source { Elements chapterElements = parsedDocument.select("tr.row.lang_English.chapter_row"); for (Element chapterElement : chapterElements) { - Chapter currentChapter = constructChapterFromHtmlBlock(chapterElement); - chapterList.add(currentChapter); + Chapter chapter = constructChapterFromHtmlBlock(chapterElement); + chapterList.add(chapter); } - - //saveChaptersToDatabase(chapterList, mangaUrl); - return chapterList; } private Chapter constructChapterFromHtmlBlock(Element chapterElement) { - Chapter newChapter = Chapter.create(); + Chapter chapter = Chapter.create(); Element urlElement = chapterElement.select("a[href^=http://bato.to/reader").first(); Element dateElement = chapterElement.select("td").get(4); if (urlElement != null) { String fieldUrl = urlElement.attr("href"); - newChapter.setUrl(fieldUrl); - newChapter.name = urlElement.text().trim(); - + chapter.setUrl(fieldUrl); + chapter.name = urlElement.text().trim(); } if (dateElement != null) { - newChapter.date_upload = parseDateFromElement(dateElement); + chapter.date_upload = parseDateFromElement(dateElement); } - newChapter.date_fetch = new Date().getTime(); + chapter.date_fetch = new Date().getTime(); - return newChapter; + return chapter; } private long parseDateFromElement(Element dateElement) { @@ -372,8 +268,7 @@ public class Batoto extends Source { List pageUrlList = new ArrayList<>(); - Element selectElement = parsedDocument.select("#page_select").first(); - + Element selectElement = Parser.element(parsedDocument, "#page_select"); if (selectElement != null) { for (Element pageUrlElement : selectElement.select("option")) { pageUrlList.add(pageUrlElement.attr("value")); @@ -389,8 +284,7 @@ public class Batoto extends Source { } @Override - protected List getFirstImageFromPageUrls(List pageUrls, String unparsedHtml) { - List pages = convertToPages(pageUrls); + protected List parseFirstPage(List pages, String unparsedHtml) { if (!unparsedHtml.contains("Want to see this chapter per page instead?")) { String firstImage = parseHtmlToImageUrl(unparsedHtml); pages.get(0).setImageUrl(firstImage); @@ -412,9 +306,7 @@ public class Batoto extends Source { String trimmedHtml = unparsedHtml.substring(beginIndex, endIndex); Document parsedDocument = Jsoup.parse(trimmedHtml); - Element imageElement = parsedDocument.getElementById("comic_page"); - return imageElement.attr("src"); } @@ -431,7 +323,7 @@ public class Batoto extends Source { String postUrl = form.attr("action"); FormEncodingBuilder formBody = new FormEncodingBuilder(); - Element authKey = form.select("input[name=auth_key").first(); + Element authKey = form.select("input[name=auth_key]").first(); formBody.add(authKey.attr("name"), authKey.attr("value")); formBody.add("ips_username", username); diff --git a/app/src/main/java/eu/kanade/mangafeed/data/source/online/english/Kissmanga.java b/app/src/main/java/eu/kanade/mangafeed/data/source/online/english/Kissmanga.java index c69ca6829..e1e67a9f5 100644 --- a/app/src/main/java/eu/kanade/mangafeed/data/source/online/english/Kissmanga.java +++ b/app/src/main/java/eu/kanade/mangafeed/data/source/online/english/Kissmanga.java @@ -10,7 +10,6 @@ import com.squareup.okhttp.Response; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; import java.text.ParseException; import java.text.SimpleDateFormat; @@ -27,6 +26,7 @@ import eu.kanade.mangafeed.data.source.SourceManager; import eu.kanade.mangafeed.data.source.base.Source; import eu.kanade.mangafeed.data.source.model.MangasPage; import eu.kanade.mangafeed.data.source.model.Page; +import eu.kanade.mangafeed.util.Parser; import rx.Observable; public class Kissmanga extends Source { @@ -64,11 +64,6 @@ public class Kissmanga extends Source { return BASE_URL; } - @Override - public boolean isLoginRequired() { - return false; - } - @Override protected String getInitialPopularMangasUrl() { return String.format(POPULAR_MANGAS_URL, 1); @@ -83,36 +78,32 @@ public class Kissmanga extends Source { protected List parsePopularMangasFromHtml(Document parsedHtml) { List mangaList = new ArrayList<>(); - Elements mangaHtmlBlocks = parsedHtml.select("table.listing tr:gt(1)"); - for (Element currentHtmlBlock : mangaHtmlBlocks) { - Manga currentManga = constructPopularMangaFromHtmlBlock(currentHtmlBlock); - mangaList.add(currentManga); + for (Element currentHtmlBlock : parsedHtml.select("table.listing tr:gt(1)")) { + Manga manga = constructPopularMangaFromHtml(currentHtmlBlock); + mangaList.add(manga); } return mangaList; } - private Manga constructPopularMangaFromHtmlBlock(Element htmlBlock) { - Manga mangaFromHtmlBlock = new Manga(); - mangaFromHtmlBlock.source = getId(); + private Manga constructPopularMangaFromHtml(Element htmlBlock) { + Manga manga = new Manga(); + manga.source = getId(); - Element urlElement = htmlBlock.select("td a:eq(0)").first(); + Element urlElement = Parser.element(htmlBlock, "td a:eq(0)"); if (urlElement != null) { - mangaFromHtmlBlock.setUrl(urlElement.attr("href")); - mangaFromHtmlBlock.title = urlElement.text(); + manga.setUrl(urlElement.attr("href")); + manga.title = urlElement.text(); } - return mangaFromHtmlBlock; + return manga; } @Override protected String parseNextPopularMangasUrl(Document parsedHtml, MangasPage page) { - Element next = parsedHtml.select("li > a:contains(› Next)").first(); - if (next == null) - return null; - - return BASE_URL + next.attr("href"); + String path = Parser.href(parsedHtml, "li > a:contains(› Next)"); + return path != null ? BASE_URL + path : null; } public Observable searchMangasFromNetwork(MangasPage page, String query) { @@ -147,90 +138,75 @@ public class Kissmanga extends Source { @Override protected Manga parseHtmlToManga(String mangaUrl, String unparsedHtml) { Document parsedDocument = Jsoup.parse(unparsedHtml); - Element infoElement = parsedDocument.select("div.barContent").first(); - Element titleElement = infoElement.select("a.bigChar").first(); - Element authorElement = infoElement.select("p:has(span:contains(Author:)) > a").first(); - Elements genreElement = infoElement.select("p:has(span:contains(Genres:)) > *:gt(0)"); - Elements descriptionElement = infoElement.select("p:has(span:contains(Summary:)) ~ p"); - Element thumbnailUrlElement = parsedDocument.select(".rightBox:eq(0) img").first(); - Manga newManga = new Manga(); - newManga.url = mangaUrl; + Manga manga = Manga.create(mangaUrl); + manga.title = Parser.text(infoElement, "a.bigChar"); + manga.author = Parser.text(infoElement, "p:has(span:contains(Author:)) > a"); + manga.genre = Parser.allText(infoElement, "p:has(span:contains(Genres:)) > *:gt(0)"); + manga.description = Parser.allText(infoElement, "p:has(span:contains(Summary:)) ~ p"); + manga.status = parseStatus(Parser.text(infoElement, "p:has(span:contains(Status:))")); - if (titleElement != null) { - newManga.title = titleElement.text(); + String thumbnail = Parser.src(parsedDocument, ".rightBox:eq(0) img"); + if (thumbnail != null) { + manga.thumbnail_url = Uri.parse(thumbnail).buildUpon().authority(IP).toString(); } - if (authorElement != null) { - newManga.author = authorElement.text(); - } - if (descriptionElement != null) { - newManga.description = descriptionElement.text(); - } - if (genreElement != null) { - newManga.genre = genreElement.text(); - } - if (thumbnailUrlElement != null) { - newManga.thumbnail_url = Uri.parse(thumbnailUrlElement.attr("src")) - .buildUpon().authority(IP).toString(); - } -// if (statusElement != null) { -// boolean fieldCompleted = statusElement.text().contains("Completed"); -// newManga.status = fieldCompleted + ""; -// } - newManga.initialized = true; + manga.initialized = true; + return manga; + } - return newManga; + private int parseStatus(String status) { + if (status.contains("Ongoing")) { + return Manga.ONGOING; + } + if (status.contains("Completed")) { + return Manga.COMPLETED; + } + return Manga.UNKNOWN; } @Override protected List parseHtmlToChapters(String unparsedHtml) { Document parsedDocument = Jsoup.parse(unparsedHtml); - List chapterList = new ArrayList<>(); - Elements chapterElements = parsedDocument.select("table.listing tr:gt(1)"); - for (Element chapterElement : chapterElements) { - Chapter currentChapter = constructChapterFromHtmlBlock(chapterElement); - - chapterList.add(currentChapter); + for (Element chapterElement : parsedDocument.select("table.listing tr:gt(1)")) { + Chapter chapter = constructChapterFromHtmlBlock(chapterElement); + chapterList.add(chapter); } return chapterList; } private Chapter constructChapterFromHtmlBlock(Element chapterElement) { - Chapter newChapter = Chapter.create(); + Chapter chapter = Chapter.create(); - Element urlElement = chapterElement.select("a").first(); - Element dateElement = chapterElement.select("td:eq(1)").first(); + Element urlElement = Parser.element(chapterElement, "a"); + String date = Parser.text(chapterElement, "td:eq(1)"); if (urlElement != null) { - newChapter.setUrl(urlElement.attr("href")); - newChapter.name = urlElement.text(); + chapter.setUrl(urlElement.attr("href")); + chapter.name = urlElement.text(); } - if (dateElement != null) { + if (date != null) { try { - newChapter.date_upload = new SimpleDateFormat("MM/dd/yyyy", Locale.ENGLISH).parse(dateElement.text()).getTime(); - } catch (ParseException e) { - // Do Nothing. - } + chapter.date_upload = new SimpleDateFormat("MM/dd/yyyy", Locale.ENGLISH).parse(date).getTime(); + } catch (ParseException e) { /* Ignore */ } } - newChapter.date_fetch = new Date().getTime(); - - return newChapter; + chapter.date_fetch = new Date().getTime(); + return chapter; } + @Override public Observable> pullPageListFromNetwork(final String chapterUrl) { - FormEncodingBuilder builder = new FormEncodingBuilder(); return networkService - .postData(getBaseUrl() + overrideChapterUrl(chapterUrl), builder.build(), requestHeaders) + .postData(getBaseUrl() + overrideChapterUrl(chapterUrl), null, requestHeaders) .flatMap(networkService::mapResponseToString) .flatMap(unparsedHtml -> { - List pageUrls = parseHtmlToPageUrls(unparsedHtml); - return Observable.just(getFirstImageFromPageUrls(pageUrls, unparsedHtml)); + List pages = convertToPages(parseHtmlToPageUrls(unparsedHtml)); + return Observable.just(parseFirstPage(pages, unparsedHtml)); }); } @@ -248,18 +224,13 @@ public class Kissmanga extends Source { } @Override - protected List getFirstImageFromPageUrls(List pageUrls, String unparsedHtml) { - List pages = convertToPages(pageUrls); - + protected List parseFirstPage(List pages, String unparsedHtml) { Pattern p = Pattern.compile("lstImages.push\\(\"(.+?)\""); Matcher m = p.matcher(unparsedHtml); - List imageUrls = new ArrayList<>(); - while (m.find()) { - imageUrls.add(m.group(1)); - } - for (int i = 0; i < pages.size(); i++) { - pages.get(i).setImageUrl(imageUrls.get(i)); + int i = 0; + while (m.find()) { + pages.get(i++).setImageUrl(m.group(1)); } return pages; } diff --git a/app/src/main/java/eu/kanade/mangafeed/data/source/online/english/Mangafox.java b/app/src/main/java/eu/kanade/mangafeed/data/source/online/english/Mangafox.java index 8e6e39dfb..a7a148ef9 100644 --- a/app/src/main/java/eu/kanade/mangafeed/data/source/online/english/Mangafox.java +++ b/app/src/main/java/eu/kanade/mangafeed/data/source/online/english/Mangafox.java @@ -21,6 +21,7 @@ import eu.kanade.mangafeed.data.database.models.Manga; import eu.kanade.mangafeed.data.source.SourceManager; import eu.kanade.mangafeed.data.source.base.Source; import eu.kanade.mangafeed.data.source.model.MangasPage; +import eu.kanade.mangafeed.util.Parser; public class Mangafox extends Source { @@ -49,11 +50,6 @@ public class Mangafox extends Source { return BASE_URL; } - @Override - public boolean isLoginRequired() { - return false; - } - @Override protected String getInitialPopularMangasUrl() { return String.format(POPULAR_MANGAS_URL, ""); @@ -68,48 +64,39 @@ public class Mangafox extends Source { protected List parsePopularMangasFromHtml(Document parsedHtml) { List mangaList = new ArrayList<>(); - Elements mangaHtmlBlocks = parsedHtml.select("div#mangalist > ul.list > li"); - for (Element currentHtmlBlock : mangaHtmlBlocks) { + for (Element currentHtmlBlock : parsedHtml.select("div#mangalist > ul.list > li")) { Manga currentManga = constructPopularMangaFromHtmlBlock(currentHtmlBlock); mangaList.add(currentManga); } - return mangaList; } private Manga constructPopularMangaFromHtmlBlock(Element htmlBlock) { - Manga mangaFromHtmlBlock = new Manga(); - mangaFromHtmlBlock.source = getId(); - - Element urlElement = htmlBlock.select("a.title").first(); + Manga manga = new Manga(); + manga.source = getId(); + Element urlElement = Parser.element(htmlBlock, "a.title"); if (urlElement != null) { - mangaFromHtmlBlock.setUrl(urlElement.attr("href")); - mangaFromHtmlBlock.title = urlElement.text(); + manga.setUrl(urlElement.attr("href")); + manga.title = urlElement.text(); } - - return mangaFromHtmlBlock; + return manga; } @Override protected String parseNextPopularMangasUrl(Document parsedHtml, MangasPage page) { - Element next = parsedHtml.select("a:has(span.next)").first(); - if (next == null) - return null; - - return String.format(POPULAR_MANGAS_URL, next.attr("href")); + Element next = Parser.element(parsedHtml, "a:has(span.next)"); + return next != null ? String.format(POPULAR_MANGAS_URL, next.attr("href")) : null; } @Override protected List parseSearchFromHtml(Document parsedHtml) { List mangaList = new ArrayList<>(); - Elements mangaHtmlBlocks = parsedHtml.select("table#listing > tbody > tr:gt(0)"); - for (Element currentHtmlBlock : mangaHtmlBlocks) { + for (Element currentHtmlBlock : parsedHtml.select("table#listing > tbody > tr:gt(0)")) { Manga currentManga = constructSearchMangaFromHtmlBlock(currentHtmlBlock); mangaList.add(currentManga); } - return mangaList; } @@ -117,23 +104,18 @@ public class Mangafox extends Source { Manga mangaFromHtmlBlock = new Manga(); mangaFromHtmlBlock.source = getId(); - Element urlElement = htmlBlock.select("a.series_preview").first(); - + Element urlElement = Parser.element(htmlBlock, "a.series_preview"); if (urlElement != null) { mangaFromHtmlBlock.setUrl(urlElement.attr("href")); mangaFromHtmlBlock.title = urlElement.text(); } - return mangaFromHtmlBlock; } @Override protected String parseNextSearchUrl(Document parsedHtml, MangasPage page, String query) { - Element next = parsedHtml.select("a:has(span.next)").first(); - if (next == null) - return null; - - return BASE_URL + next.attr("href"); + Element next = Parser.element(parsedHtml, "a:has(span.next)"); + return next != null ? BASE_URL + next.attr("href") : null; } @Override @@ -141,84 +123,60 @@ public class Mangafox extends Source { Document parsedDocument = Jsoup.parse(unparsedHtml); Element infoElement = parsedDocument.select("div#title").first(); - Element titleElement = infoElement.select("h2 > a").first(); Element rowElement = infoElement.select("table > tbody > tr:eq(1)").first(); - Element authorElement = rowElement.select("td:eq(1)").first(); - Element artistElement = rowElement.select("td:eq(2)").first(); - Element genreElement = rowElement.select("td:eq(3)").first(); - Element descriptionElement = infoElement.select("p.summary").first(); - Element thumbnailUrlElement = parsedDocument.select("div.cover > img").first(); + Element sideInfoElement = parsedDocument.select("#series_info").first(); - Manga newManga = new Manga(); - newManga.url = mangaUrl; + Manga manga = Manga.create(mangaUrl); + manga.author = Parser.text(rowElement, "td:eq(1)"); + manga.artist = Parser.text(rowElement, "td:eq(2)"); + manga.description = Parser.text(infoElement, "p.summary"); + manga.genre = Parser.text(rowElement, "td:eq(3)"); + manga.thumbnail_url = Parser.src(sideInfoElement, "div.cover > img"); + manga.status = parseStatus(Parser.text(sideInfoElement, ".data")); - if (titleElement != null) { - String title = titleElement.text(); - // Strip the last word - title = title.substring(0, title.lastIndexOf(" ")); - newManga.title = title; - } - if (artistElement != null) { - newManga.artist = artistElement.text(); - } - if (authorElement != null) { - newManga.author = authorElement.text(); - } - if (descriptionElement != null) { - newManga.description = descriptionElement.text(); - } - if (genreElement != null) { - newManga.genre = genreElement.text(); - } - if (thumbnailUrlElement != null) { - newManga.thumbnail_url = thumbnailUrlElement.attr("src"); - } -// if (statusElement != null) { -// boolean fieldCompleted = statusElement.text().contains("Completed"); -// newManga.status = fieldCompleted + ""; -// } + manga.initialized = true; + return manga; + } - newManga.initialized = true; - - return newManga; + private int parseStatus(String status) { + if (status.contains("Ongoing")) { + return Manga.ONGOING; + } + if (status.contains("Completed")) { + return Manga.COMPLETED; + } + return Manga.UNKNOWN; } @Override protected List parseHtmlToChapters(String unparsedHtml) { Document parsedDocument = Jsoup.parse(unparsedHtml); - List chapterList = new ArrayList(); + List chapterList = new ArrayList<>(); - Elements chapterElements = parsedDocument.select("div#chapters li div"); - for (Element chapterElement : chapterElements) { + for (Element chapterElement : parsedDocument.select("div#chapters li div")) { Chapter currentChapter = constructChapterFromHtmlBlock(chapterElement); - chapterList.add(currentChapter); } - return chapterList; } private Chapter constructChapterFromHtmlBlock(Element chapterElement) { - Chapter newChapter = Chapter.create(); + Chapter chapter = Chapter.create(); Element urlElement = chapterElement.select("a.tips").first(); - Element nameElement = chapterElement.select("a.tips").first(); Element dateElement = chapterElement.select("span.date").first(); if (urlElement != null) { - newChapter.setUrl(urlElement.attr("href")); - } - if (nameElement != null) { - newChapter.name = nameElement.text(); + chapter.setUrl(urlElement.attr("href")); + chapter.name = urlElement.text(); } if (dateElement != null) { - newChapter.date_upload = parseUpdateFromElement(dateElement); + chapter.date_upload = parseUpdateFromElement(dateElement); } + chapter.date_fetch = new Date().getTime(); - newChapter.date_fetch = new Date().getTime(); - - return newChapter; + return chapter; } private long parseUpdateFromElement(Element updateElement) { diff --git a/app/src/main/java/eu/kanade/mangafeed/data/source/online/english/Mangahere.java b/app/src/main/java/eu/kanade/mangafeed/data/source/online/english/Mangahere.java index 425202c7e..829c8967f 100644 --- a/app/src/main/java/eu/kanade/mangafeed/data/source/online/english/Mangahere.java +++ b/app/src/main/java/eu/kanade/mangafeed/data/source/online/english/Mangahere.java @@ -21,7 +21,7 @@ import eu.kanade.mangafeed.data.database.models.Manga; import eu.kanade.mangafeed.data.source.SourceManager; import eu.kanade.mangafeed.data.source.base.Source; import eu.kanade.mangafeed.data.source.model.MangasPage; -import rx.Observable; +import eu.kanade.mangafeed.util.Parser; public class Mangahere extends Source { @@ -49,11 +49,6 @@ public class Mangahere extends Source { return BASE_URL; } - @Override - public boolean isLoginRequired() { - return false; - } - @Override protected String getInitialPopularMangasUrl() { return String.format(POPULAR_MANGAS_URL, ""); @@ -64,78 +59,33 @@ public class Mangahere extends Source { return String.format(SEARCH_URL, Uri.encode(query), 1); } - public Observable> getGenres() { - List genres = new ArrayList<>(30); - - genres.add("Action"); - genres.add("Adventure"); - genres.add("Comedy"); - genres.add("Drama"); - genres.add("Ecchi"); - genres.add("Fantasy"); - genres.add("Gender Bender"); - genres.add("Harem"); - genres.add("Historical"); - genres.add("Horror"); - genres.add("Josei"); - genres.add("Martial Arts"); - genres.add("Mature"); - genres.add("Mecha"); - genres.add("Mystery"); - genres.add("One Shot"); - genres.add("Psychological"); - genres.add("Romance"); - genres.add("School Life"); - genres.add("Sci-fi"); - genres.add("Seinen"); - genres.add("Shoujo"); - genres.add("Shoujo Ai"); - genres.add("Shounen"); - genres.add("Shounen Ai"); - genres.add("Slice of Life"); - genres.add("Sports"); - genres.add("Supernatural"); - genres.add("Tragedy"); - genres.add("Yaoi"); - genres.add("Yuri"); - - return Observable.just(genres); - } - @Override public List parsePopularMangasFromHtml(Document parsedHtml) { List mangaList = new ArrayList<>(); - Elements mangaHtmlBlocks = parsedHtml.select("div.directory_list > ul > li"); - for (Element currentHtmlBlock : mangaHtmlBlocks) { + for (Element currentHtmlBlock : parsedHtml.select("div.directory_list > ul > li")) { Manga currentManga = constructPopularMangaFromHtmlBlock(currentHtmlBlock); mangaList.add(currentManga); } - return mangaList; } private Manga constructPopularMangaFromHtmlBlock(Element htmlBlock) { - Manga mangaFromHtmlBlock = new Manga(); - mangaFromHtmlBlock.source = getId(); - - Element urlElement = htmlBlock.select("div.title > a").first(); + Manga manga = new Manga(); + manga.source = getId(); + Element urlElement = Parser.element(htmlBlock, "div.title > a"); if (urlElement != null) { - mangaFromHtmlBlock.setUrl(urlElement.attr("href")); - mangaFromHtmlBlock.title = urlElement.attr("title"); + manga.setUrl(urlElement.attr("href")); + manga.title = urlElement.attr("title"); } - - return mangaFromHtmlBlock; + return manga; } @Override protected String parseNextPopularMangasUrl(Document parsedHtml, MangasPage page) { - Element next = parsedHtml.select("div.next-page > a.next").first(); - if (next == null) - return null; - - return String.format(POPULAR_MANGAS_URL, next.attr("href")); + Element next = Parser.element(parsedHtml, "div.next-page > a.next"); + return next != null ? String.format(POPULAR_MANGAS_URL, next.attr("href")) : null; } @Override @@ -147,31 +97,25 @@ public class Mangahere extends Source { Manga currentManga = constructSearchMangaFromHtmlBlock(currentHtmlBlock); mangaList.add(currentManga); } - return mangaList; } private Manga constructSearchMangaFromHtmlBlock(Element htmlBlock) { - Manga mangaFromHtmlBlock = new Manga(); - mangaFromHtmlBlock.source = getId(); - - Element urlElement = htmlBlock.select("a.manga_info").first(); + Manga manga = new Manga(); + manga.source = getId(); + Element urlElement = Parser.element(htmlBlock, "a.manga_info"); if (urlElement != null) { - mangaFromHtmlBlock.setUrl(urlElement.attr("href")); - mangaFromHtmlBlock.title = urlElement.text(); + manga.setUrl(urlElement.attr("href")); + manga.title = urlElement.text(); } - - return mangaFromHtmlBlock; + return manga; } @Override protected String parseNextSearchUrl(Document parsedHtml, MangasPage page, String query) { - Element next = parsedHtml.select("div.next-page > a.next").first(); - if (next == null) - return null; - - return BASE_URL + next.attr("href"); + Element next = Parser.element(parsedHtml, "div.next-page > a.next"); + return next != null ? BASE_URL + next.attr("href") : null; } private long parseUpdateFromElement(Element updateElement) { @@ -223,49 +167,41 @@ public class Mangahere extends Source { String trimmedHtml = unparsedHtml.substring(beginIndex, endIndex); Document parsedDocument = Jsoup.parse(trimmedHtml); + Element detailElement = parsedDocument.select("ul.detail_topText").first(); - Elements detailElements = parsedDocument.select("ul.detail_topText li"); + Manga manga = Manga.create(mangaUrl); + manga.author = Parser.text(parsedDocument, "a[href^=http://www.mangahere.co/author/]"); + manga.artist = Parser.text(parsedDocument, "a[href^=http://www.mangahere.co/artist/]"); - Element artistElement = parsedDocument.select("a[href^=http://www.mangahere.co/artist/]").first(); - Element authorElement = parsedDocument.select("a[href^=http://www.mangahere.co/author/]").first(); - Element descriptionElement = detailElements.select("#show").first(); - Element genreElement = detailElements.get(3); - Element statusElement = detailElements.get(6); - - Manga newManga = new Manga(); - newManga.url = mangaUrl; - - if (artistElement != null) { - newManga.artist = artistElement.text(); + String description = Parser.text(detailElement, "#show"); + if (description != null) { + manga.description = description.substring(0, description.length() - "Show less".length()); } - if (authorElement != null) { - newManga.author = authorElement.text(); - } - if (descriptionElement != null) { - newManga.description = descriptionElement.text().substring(0, descriptionElement.text().length() - "Show less".length()); - } - if (genreElement != null) { - newManga.genre = genreElement.text().substring("Genre(s):".length()); - } - if (statusElement != null) { - boolean fieldCompleted = statusElement.text().contains("Completed"); - // TODO fix status -// newManga.status = fieldCompleted + ""; + String genres = Parser.text(detailElement, "li:eq(3)"); + if (genres != null) { + manga.genre = genres.substring("Genre(s):".length()); } + manga.status = parseStatus(Parser.text(detailElement, "li:eq(6)")); beginIndex = unparsedHtml.indexOf("", beginIndex); trimmedHtml = unparsedHtml.substring(beginIndex, endIndex + 2); + parsedDocument = Jsoup.parse(trimmedHtml); - Element thumbnailUrlElement = parsedDocument.select("img").first(); + manga.thumbnail_url = Parser.src(parsedDocument, "img"); - if (thumbnailUrlElement != null) { - newManga.thumbnail_url = thumbnailUrlElement.attr("src"); + manga.initialized = true; + return manga; + } + + private int parseStatus(String status) { + if (status.contains("Ongoing")) { + return Manga.ONGOING; } - - newManga.initialized = true; - - return newManga; + if (status.contains("Completed")) { + return Manga.COMPLETED; + } + return Manga.UNKNOWN; } @Override @@ -276,37 +212,31 @@ public class Mangahere extends Source { Document parsedDocument = Jsoup.parse(trimmedHtml); - List chapterList = new ArrayList(); + List chapterList = new ArrayList<>(); - Elements chapterElements = parsedDocument.getElementsByTag("li"); - for (Element chapterElement : chapterElements) { + for (Element chapterElement : parsedDocument.getElementsByTag("li")) { Chapter currentChapter = constructChapterFromHtmlBlock(chapterElement); - chapterList.add(currentChapter); } - return chapterList; } private Chapter constructChapterFromHtmlBlock(Element chapterElement) { - Chapter newChapter = Chapter.create(); + Chapter chapter = Chapter.create(); Element urlElement = chapterElement.select("a").first(); - Element nameElement = chapterElement.select("a").first(); Element dateElement = chapterElement.select("span.right").first(); if (urlElement != null) { - newChapter.setUrl(urlElement.attr("href")); - } - if (nameElement != null) { - newChapter.name = nameElement.text(); + chapter.setUrl(urlElement.attr("href")); + chapter.name = urlElement.text(); } if (dateElement != null) { - newChapter.date_upload = parseDateFromElement(dateElement); + chapter.date_upload = parseDateFromElement(dateElement); } - newChapter.date_fetch = new Date().getTime(); + chapter.date_fetch = new Date().getTime(); - return newChapter; + return chapter; } private long parseDateFromElement(Element dateElement) { @@ -348,7 +278,6 @@ public class Mangahere extends Source { // Do Nothing. } } - return 0; } @@ -360,7 +289,7 @@ public class Mangahere extends Source { Document parsedDocument = Jsoup.parse(trimmedHtml); - List pageUrlList = new ArrayList(); + List pageUrlList = new ArrayList<>(); Elements pageUrlElements = parsedDocument.select("select.wid60").first().getElementsByTag("option"); for (Element pageUrlElement : pageUrlElements) { diff --git a/app/src/main/java/eu/kanade/mangafeed/ui/manga/myanimelist/MyAnimeListPresenter.java b/app/src/main/java/eu/kanade/mangafeed/ui/manga/myanimelist/MyAnimeListPresenter.java index 96ca43828..fbb9874ae 100644 --- a/app/src/main/java/eu/kanade/mangafeed/ui/manga/myanimelist/MyAnimeListPresenter.java +++ b/app/src/main/java/eu/kanade/mangafeed/ui/manga/myanimelist/MyAnimeListPresenter.java @@ -69,6 +69,7 @@ public class MyAnimeListPresenter extends BasePresenter { for (MangaSync myManga : myList) { if (myManga.remote_id == mangaSync.remote_id) { mangaSync.copyPersonalFrom(myManga); + mangaSync.total_chapters = myManga.total_chapters; return Observable.just(mangaSync); } } diff --git a/app/src/main/java/eu/kanade/mangafeed/util/Parser.java b/app/src/main/java/eu/kanade/mangafeed/util/Parser.java new file mode 100644 index 000000000..f32c686ec --- /dev/null +++ b/app/src/main/java/eu/kanade/mangafeed/util/Parser.java @@ -0,0 +1,48 @@ +package eu.kanade.mangafeed.util; + +import android.support.annotation.Nullable; + +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; + +public class Parser { + + @Nullable + public static Element element(Element container, String pattern) { + return container.select(pattern).first(); + } + + @Nullable + public static String text(Element container, String pattern) { + return text(container, pattern, null); + } + + @Nullable + public static String text(Element container, String pattern, String defValue) { + Element element = container.select(pattern).first(); + return element != null ? element.text() : defValue; + } + + @Nullable + public static String allText(Element container, String pattern) { + Elements elements = container.select(pattern); + return !elements.isEmpty() ? elements.text() : null; + } + + @Nullable + public static String attr(Element container, String pattern, String attr) { + Element element = container.select(pattern).first(); + return element != null ? element.attr(attr) : null; + } + + @Nullable + public static String href(Element container, String pattern) { + return attr(container, pattern, "href"); + } + + @Nullable + public static String src(Element container, String pattern) { + return attr(container, pattern, "src"); + } + +}