From d7e6e812739919e47ee93d27b94b2f3bc3c3a4c7 Mon Sep 17 00:00:00 2001 From: Dragonatorul <109987846+Dragonatorul@users.noreply.github.com> Date: Sat, 7 Jun 2025 21:04:01 +0300 Subject: [PATCH 01/11] wip: adding wpmadara --- gallery_dl/extractor/__init__.py | 2 +- gallery_dl/extractor/common.py | 56 +++++++++++++++++++ .../extractor/{mangaread.py => wpmadara.py} | 37 ++++++++---- test/results/mangaread.py | 16 +++--- 4 files changed, 92 insertions(+), 19 deletions(-) rename gallery_dl/extractor/{mangaread.py => wpmadara.py} (80%) diff --git a/gallery_dl/extractor/__init__.py b/gallery_dl/extractor/__init__.py index fcf27c8a10..15483a48e7 100644 --- a/gallery_dl/extractor/__init__.py +++ b/gallery_dl/extractor/__init__.py @@ -106,7 +106,7 @@ "mangahere", "manganelo", "mangapark", - "mangaread", + "wpmadara", "mangoxo", "misskey", "motherless", diff --git a/gallery_dl/extractor/common.py b/gallery_dl/extractor/common.py index b7562a1ba3..b8b16db5d9 100644 --- a/gallery_dl/extractor/common.py +++ b/gallery_dl/extractor/common.py @@ -28,6 +28,7 @@ class Extractor(): + """Base class for all extractors""" category = "" subcategory = "" @@ -50,6 +51,18 @@ class Extractor(): request_timestamp = 0.0 def __init__(self, match): + """ + Initialize the CommonExtractor object. + + Args: + match (re.Match): The regular expression match object. + + Attributes: + log (logging.Logger): The logger object for logging messages. + url (str): The URL string. + _cfgpath (tuple): The configuration path. + _parentdir (str): The parent directory. + """ self.log = logging.getLogger(self.category) self.url = match.string self.match = match @@ -62,16 +75,31 @@ def __init__(self, match): @classmethod def from_url(cls, url): + """ + Create an instance of the class from a given URL. + + Args: + url (str): The URL to match against the class's pattern. + + Returns: + cls: An instance of the class if the URL matches the pattern, otherwise None. + """ if isinstance(cls.pattern, str): cls.pattern = util.re_compile(cls.pattern) match = cls.pattern.match(url) return cls(match) if match else None def __iter__(self): + """ + Returns an iterator object that iterates over the items in the extractor. + """ self.initialize() return self.items() def initialize(self): + """ + Initializes the extractor. + """ self._init_options() self._init_session() self._init_cookies() @@ -79,15 +107,43 @@ def initialize(self): self.initialize = util.noop def finalize(self): + """ + This method is called to perform any necessary finalization steps. + """ pass def items(self): + """ + Generator that yields a tuple of Message.Version and 1. + + Returns: + A tuple containing Message.Version and 1. + """ yield Message.Version, 1 def skip(self, num): + """ + Skips a specified number of items. + + Args: + num (int): The number of items to skip. + + Returns: + int: The number of items skipped. + """ return 0 def config(self, key, default=None): + """ + Retrieves the value of a configuration key from the specified configuration file. + + Args: + key (str): The configuration key to retrieve. + default (Any, optional): The default value to return if the key is not found. Defaults to None. + + Returns: + Any: The value of the configuration key, or the default value if the key is not found. + """ return config.interpolate(self._cfgpath, key, default) def config2(self, key, key2, default=None, sentinel=util.SENTINEL): diff --git a/gallery_dl/extractor/mangaread.py b/gallery_dl/extractor/wpmadara.py similarity index 80% rename from gallery_dl/extractor/mangaread.py rename to gallery_dl/extractor/wpmadara.py index 6444282e78..c54f9ed9c9 100644 --- a/gallery_dl/extractor/mangaread.py +++ b/gallery_dl/extractor/wpmadara.py @@ -6,13 +6,14 @@ """Extractors for https://mangaread.org/""" -from .common import ChapterExtractor, MangaExtractor -from .. import text, util, exception +from .common import BaseExtractor, ChapterExtractor, MangaExtractor +from .. import text, exception +import re -class MangareadBase(): +class WPMadaraBase(BaseExtractor): """Base class for Mangaread extractors""" - category = "mangaread" + basecategory = "wpmadara" root = "https://www.mangaread.org" @staticmethod @@ -29,13 +30,24 @@ def parse_chapter_string(chapter_string, data): data["lang"] = "en" data["language"] = "English" +BASE_PATTERN = WPMadaraBase.update({ + "mangaread": { + "root": "https://www.mangaread.org", + "pattern": r"(?:https?://)?(?:www\.)?mangaread\.org", + }, +}) -class MangareadChapterExtractor(MangareadBase, ChapterExtractor): + +class WPMadaraChapterExtractor(WPMadaraBase): """Extractor for manga-chapters from mangaread.org""" - pattern = (r"(?:https?://)?(?:www\.)?mangaread\.org" - r"(/manga/[^/?#]+/[^/?#]+)") + subcategory = "chapter" + pattern = BASE_PATTERN + r"(/(manga)/[^/?#]+/[^/?#]+)" example = "https://www.mangaread.org/manga/MANGA/chapter-01/" + def __init__(self, match): + WPMadaraBase.__init__(self, match) + self.chapter = match.group(match.lastindex) + def metadata(self, page): tags = text.extr(page, 'class="wp-manga-tags-list">', '') data = {"tags": list(text.split_html(tags)[::2])} @@ -54,12 +66,17 @@ def images(self, page): ] -class MangareadMangaExtractor(MangareadBase, MangaExtractor): +class WPMadaraMangaExtractor(WPMadaraBase): """Extractor for manga from mangaread.org""" - chapterclass = MangareadChapterExtractor - pattern = r"(?:https?://)?(?:www\.)?mangaread\.org(/manga/[^/?#]+)/?$" + chapterclass = WPMadaraChapterExtractor + subcategory = "manga" + pattern = BASE_PATTERN + r"(/(manga)/[^/?#]+)/?$" example = "https://www.mangaread.org/manga/MANGA" + def __init__(self, match): + WPMadaraBase.__init__(self, match) + self.manga = match.group(match.lastindex) + def chapters(self, page): if 'class="error404' in page: raise exception.NotFoundError("manga") diff --git a/test/results/mangaread.py b/test/results/mangaread.py index 1c5e9a356a..c31786f66a 100644 --- a/test/results/mangaread.py +++ b/test/results/mangaread.py @@ -4,7 +4,7 @@ # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import mangaread +from gallery_dl.extractor import wpmadara from gallery_dl import exception @@ -12,7 +12,7 @@ { "#url" : "https://www.mangaread.org/manga/one-piece/chapter-1053-3/", "#category": ("", "mangaread", "chapter"), - "#class" : mangaread.MangareadChapterExtractor, + "#class" : wpmadara.MangareadChapterExtractor, "#pattern" : r"https://www\.mangaread\.org/wp-content/uploads/WP-manga/data/manga_[^/]+/[^/]+/[^.]+\.\w+", "#count" : 11, @@ -28,14 +28,14 @@ { "#url" : "https://www.mangaread.org/manga/one-piece/chapter-1000000/", "#category": ("", "mangaread", "chapter"), - "#class" : mangaread.MangareadChapterExtractor, + "#class" : wpmadara.MangareadChapterExtractor, "#exception": exception.NotFoundError, }, { "#url" : "https://www.mangaread.org/manga/kanan-sama-wa-akumade-choroi/chapter-10/", "#category": ("", "mangaread", "chapter"), - "#class" : mangaread.MangareadChapterExtractor, + "#class" : wpmadara.MangareadChapterExtractor, "#pattern" : r"https://www\.mangaread\.org/wp-content/uploads/WP-manga/data/manga_[^/]+/[^/]+/[^.]+\.\w+", "#count" : 9, @@ -52,7 +52,7 @@ "#url" : "https://www.mangaread.org/manga/above-all-gods/chapter146-5/", "#comment" : "^^ no whitespace", "#category": ("", "mangaread", "chapter"), - "#class" : mangaread.MangareadChapterExtractor, + "#class" : wpmadara.MangareadChapterExtractor, "#pattern" : r"https://www\.mangaread\.org/wp-content/uploads/WP-manga/data/manga_[^/]+/[^/]+/[^.]+\.\w+", "#count" : 6, @@ -68,7 +68,7 @@ { "#url" : "https://www.mangaread.org/manga/kanan-sama-wa-akumade-choroi", "#category": ("", "mangaread", "manga"), - "#class" : mangaread.MangareadMangaExtractor, + "#class" : wordpressmadara.MangareadMangaExtractor, "#pattern" : r"https://www\.mangaread\.org/manga/kanan-sama-wa-akumade-choroi/chapter-\d+([_-].+)?/", "#count" : ">= 13", @@ -94,7 +94,7 @@ { "#url" : "https://www.mangaread.org/manga/one-piece", "#category": ("", "mangaread", "manga"), - "#class" : mangaread.MangareadMangaExtractor, + "#class" : wordpressmadara.MangareadMangaExtractor, "#pattern" : r"https://www\.mangaread\.org/manga/one-piece/chapter-\d+(-.+)?/", "#count" : ">= 1066", @@ -115,7 +115,7 @@ { "#url" : "https://www.mangaread.org/manga/doesnotexist", "#category": ("", "mangaread", "manga"), - "#class" : mangaread.MangareadMangaExtractor, + "#class" : wordpressmadara.MangareadMangaExtractor, "#exception": exception.NotFoundError, }, From 161c4a24626e00a83e50161b190cf48898ce4541 Mon Sep 17 00:00:00 2001 From: Dragonatorul <109987846+Dragonatorul@users.noreply.github.com> Date: Sat, 7 Jun 2025 21:04:35 +0300 Subject: [PATCH 02/11] undo comment changes --- gallery_dl/extractor/common.py | 48 +++------------------------------- 1 file changed, 4 insertions(+), 44 deletions(-) diff --git a/gallery_dl/extractor/common.py b/gallery_dl/extractor/common.py index b8b16db5d9..3bd7c77e1d 100644 --- a/gallery_dl/extractor/common.py +++ b/gallery_dl/extractor/common.py @@ -28,7 +28,6 @@ class Extractor(): - """Base class for all extractors""" category = "" subcategory = "" @@ -51,6 +50,10 @@ class Extractor(): request_timestamp = 0.0 def __init__(self, match): + self.log = logging.getLogger(self.category) + self.url = match.string + self._cfgpath = ("extractor", self.category, self.subcategory) + self._parentdir = "" """ Initialize the CommonExtractor object. @@ -75,31 +78,16 @@ def __init__(self, match): @classmethod def from_url(cls, url): - """ - Create an instance of the class from a given URL. - - Args: - url (str): The URL to match against the class's pattern. - - Returns: - cls: An instance of the class if the URL matches the pattern, otherwise None. - """ if isinstance(cls.pattern, str): cls.pattern = util.re_compile(cls.pattern) match = cls.pattern.match(url) return cls(match) if match else None def __iter__(self): - """ - Returns an iterator object that iterates over the items in the extractor. - """ self.initialize() return self.items() def initialize(self): - """ - Initializes the extractor. - """ self._init_options() self._init_session() self._init_cookies() @@ -107,43 +95,15 @@ def initialize(self): self.initialize = util.noop def finalize(self): - """ - This method is called to perform any necessary finalization steps. - """ pass def items(self): - """ - Generator that yields a tuple of Message.Version and 1. - - Returns: - A tuple containing Message.Version and 1. - """ yield Message.Version, 1 def skip(self, num): - """ - Skips a specified number of items. - - Args: - num (int): The number of items to skip. - - Returns: - int: The number of items skipped. - """ return 0 def config(self, key, default=None): - """ - Retrieves the value of a configuration key from the specified configuration file. - - Args: - key (str): The configuration key to retrieve. - default (Any, optional): The default value to return if the key is not found. Defaults to None. - - Returns: - Any: The value of the configuration key, or the default value if the key is not found. - """ return config.interpolate(self._cfgpath, key, default) def config2(self, key, key2, default=None, sentinel=util.SENTINEL): From d1f26e06c47a766fbebe737c9fb7414cd917bf64 Mon Sep 17 00:00:00 2001 From: Dragonatorul <109987846+Dragonatorul@users.noreply.github.com> Date: Sat, 7 Jun 2025 21:04:35 +0300 Subject: [PATCH 03/11] add support for toonily and webtoonxyz --- gallery_dl/extractor/wpmadara.py | 28 +++++++++++--- test/results/toonily.py | 64 ++++++++++++++++++++++++++++++++ test/results/webtoonxyz.py | 62 +++++++++++++++++++++++++++++++ 3 files changed, 148 insertions(+), 6 deletions(-) create mode 100644 test/results/toonily.py create mode 100644 test/results/webtoonxyz.py diff --git a/gallery_dl/extractor/wpmadara.py b/gallery_dl/extractor/wpmadara.py index c54f9ed9c9..ea0087d9e2 100644 --- a/gallery_dl/extractor/wpmadara.py +++ b/gallery_dl/extractor/wpmadara.py @@ -35,18 +35,31 @@ def parse_chapter_string(chapter_string, data): "root": "https://www.mangaread.org", "pattern": r"(?:https?://)?(?:www\.)?mangaread\.org", }, + "toonily": { + "root": "https://www.toonily.com", + "pattern": r"(?:https?://)?(?:www\.)?toonily\.com", + }, + "webtoonxyz": { + "root": "https://www.webtoon.xyz", + "pattern": r"(?:https?://)?(?:www\.)?webtoon\.xyz", + }, }) -class WPMadaraChapterExtractor(WPMadaraBase): +class WPMadaraChapterExtractor(WPMadaraBase, ChapterExtractor): """Extractor for manga-chapters from mangaread.org""" subcategory = "chapter" - pattern = BASE_PATTERN + r"(/(manga)/[^/?#]+/[^/?#]+)" + pattern = BASE_PATTERN + r"(/(manga|webtoon|read)/[^/?#]+/[^/?#]+)" example = "https://www.mangaread.org/manga/MANGA/chapter-01/" - def __init__(self, match): + def __init__(self, match, url=None): WPMadaraBase.__init__(self, match) self.chapter = match.group(match.lastindex) + self.log.debug("chapter: %s", self.chapter) + self.gallery_url = self.root + match.group(match.lastindex) + + if self.config("chapter-reverse", False): + self.reverse = not self.reverse def metadata(self, page): tags = text.extr(page, 'class="wp-manga-tags-list">', '') @@ -55,27 +68,30 @@ def metadata(self, page): if not info: raise exception.NotFoundError("chapter") self.parse_chapter_string(info, data) + self.log.debug("data: %s", data) return data def images(self, page): page = text.extr( page, '
', '
= 13", + + "manga" : "Such a Cute Spy", + "author" : ["Life of Ruin"], + "artist" : ["Ganghyeon Yeo"], + "genres" : [ + "Action", + "Comedy", + "Romance", + "School Life", + ], + "rating" : float, + "status" : "End", + "lang" : "en", + "language" : "English", + "manga_alt" : list, +}, + +{ + "#url" : "https://toonily.com/webtoon/doesnotexist", + "#category": ("", "toonily", "manga"), + "#class" : wpmadara.WPMadaraMangaExtractor, + "#exception": exception.HttpError, +}, + +) diff --git a/test/results/webtoonxyz.py b/test/results/webtoonxyz.py new file mode 100644 index 0000000000..62ca7d38d0 --- /dev/null +++ b/test/results/webtoonxyz.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import wpmadara +from gallery_dl import exception + + +__tests__ = ( +{ + "#url" : "https://www.webtoon.xyz/read/the-world-after-the-end/chapter-105/", + "#category": ("", "wpmadara", "chapter"), + "#class" : wpmadara.WPMadaraChapterExtractor, + "#pattern" : r"https://www\.webtoon\.xyz/wp-content/uploads/WP-manga/data/manga_[^/]+/[^/]+/[^.]+\.\w+", + "#count" : 11, + + "manga" : "The World After The End", + "title" : "", + "chapter" : 105, + "lang" : "en", + "language" : "English", +}, + +{ + "#url" : "https://www.webtoon.xyz/read/the-world-after-the-end/chapter-1000000/", + "#category": ("", "wpmadara", "chapter"), + "#class" : wpmadara.WPMadaraChapterExtractor, + "#exception": exception.NotFoundError, +}, + +{ + "#url" : "https://www.webtoon.xyz/read/the-world-after-the-end/", + "#category": ("", "wpmadara", "manga"), + "#class" : wpmadara.WPMadaraMangaExtractor, + "#pattern" : r"https://www\.webtoon\.xyz/read/such-a-cute-spy/chapter-\d+([_-].+)?/", + "#count" : ">= 13", + + "manga" : "The World After The End", + "author" : ["S-Cynaan", "Sing Shong"], + "artist" : ["Undead Potato"], + "genres" : [ + "Action", + "Adventure", + "Fantasy", + ], + "rating" : float, + "status" : "OnGoing", + "lang" : "en", + "language" : "English", + "manga_alt" : list, +}, + +{ + "#url" : "https://www.webtoon.xyz/read/doesnotexist", + "#category": ("", "wpmadara", "manga"), + "#class" : wpmadara.WPMadaraMangaExtractor, + "#exception": exception.HttpError, +}, + +) From 38d74b507a950e50e488d2eb294f6b0e7c85f81b Mon Sep 17 00:00:00 2001 From: Dragonatorul <109987846+Dragonatorul@users.noreply.github.com> Date: Sat, 7 Jun 2025 21:05:22 +0300 Subject: [PATCH 04/11] fix: correct tests --- test/results/mangaread.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/results/mangaread.py b/test/results/mangaread.py index c31786f66a..3560bc65d9 100644 --- a/test/results/mangaread.py +++ b/test/results/mangaread.py @@ -12,7 +12,7 @@ { "#url" : "https://www.mangaread.org/manga/one-piece/chapter-1053-3/", "#category": ("", "mangaread", "chapter"), - "#class" : wpmadara.MangareadChapterExtractor, + "#class" : wpmadara.WPMadaraChapterExtractor, "#pattern" : r"https://www\.mangaread\.org/wp-content/uploads/WP-manga/data/manga_[^/]+/[^/]+/[^.]+\.\w+", "#count" : 11, @@ -28,14 +28,14 @@ { "#url" : "https://www.mangaread.org/manga/one-piece/chapter-1000000/", "#category": ("", "mangaread", "chapter"), - "#class" : wpmadara.MangareadChapterExtractor, + "#class" : wpmadara.WPMadaraChapterExtractor, "#exception": exception.NotFoundError, }, { "#url" : "https://www.mangaread.org/manga/kanan-sama-wa-akumade-choroi/chapter-10/", "#category": ("", "mangaread", "chapter"), - "#class" : wpmadara.MangareadChapterExtractor, + "#class" : wpmadara.WPMadaraMangaExtractor, "#pattern" : r"https://www\.mangaread\.org/wp-content/uploads/WP-manga/data/manga_[^/]+/[^/]+/[^.]+\.\w+", "#count" : 9, @@ -52,7 +52,7 @@ "#url" : "https://www.mangaread.org/manga/above-all-gods/chapter146-5/", "#comment" : "^^ no whitespace", "#category": ("", "mangaread", "chapter"), - "#class" : wpmadara.MangareadChapterExtractor, + "#class" : wpmadara.WPMadaraMangaExtractor, "#pattern" : r"https://www\.mangaread\.org/wp-content/uploads/WP-manga/data/manga_[^/]+/[^/]+/[^.]+\.\w+", "#count" : 6, @@ -68,7 +68,7 @@ { "#url" : "https://www.mangaread.org/manga/kanan-sama-wa-akumade-choroi", "#category": ("", "mangaread", "manga"), - "#class" : wordpressmadara.MangareadMangaExtractor, + "#class" : wpmadara.WPMadaraMangaExtractor, "#pattern" : r"https://www\.mangaread\.org/manga/kanan-sama-wa-akumade-choroi/chapter-\d+([_-].+)?/", "#count" : ">= 13", @@ -94,7 +94,7 @@ { "#url" : "https://www.mangaread.org/manga/one-piece", "#category": ("", "mangaread", "manga"), - "#class" : wordpressmadara.MangareadMangaExtractor, + "#class" : wpmadara.WPMadaraMangaExtractor, "#pattern" : r"https://www\.mangaread\.org/manga/one-piece/chapter-\d+(-.+)?/", "#count" : ">= 1066", From 569e7f2963fab0723476747a777c81574b3757a6 Mon Sep 17 00:00:00 2001 From: Dragonatorul <109987846+Dragonatorul@users.noreply.github.com> Date: Sat, 7 Jun 2025 21:05:39 +0300 Subject: [PATCH 05/11] updated documentation --- docs/supportedsites.md | 28 ++++++++++++++++++++++------ gallery_dl/extractor/wpmadara.py | 8 ++++---- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/docs/supportedsites.md b/docs/supportedsites.md index 17f7d3c655..9af0d944a3 100644 --- a/docs/supportedsites.md +++ b/docs/supportedsites.md @@ -571,12 +571,6 @@ Consider all listed sites to potentially be NSFW. Chapters, Manga - - MangaRead - https://mangaread.org/ - Chapters, Manga - - Mangoxo https://www.mangoxo.com/ @@ -1875,5 +1869,27 @@ Consider all listed sites to potentially be NSFW. Albums + + + WordPressMadara based websites + + + MangaRead + https://mangaread.org/ + Chapters, Manga + + + + Toonily + https://toonily.com/ + Chapters, Manga + + + + WebtoonXYZ + https://www.webtoon.xyz/ + Chapters, Manga + + diff --git a/gallery_dl/extractor/wpmadara.py b/gallery_dl/extractor/wpmadara.py index ea0087d9e2..3df4037e13 100644 --- a/gallery_dl/extractor/wpmadara.py +++ b/gallery_dl/extractor/wpmadara.py @@ -4,7 +4,7 @@ # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -"""Extractors for https://mangaread.org/""" +"""Extractors for WordPressMadara based websites.""" from .common import BaseExtractor, ChapterExtractor, MangaExtractor from .. import text, exception @@ -12,7 +12,7 @@ class WPMadaraBase(BaseExtractor): - """Base class for Mangaread extractors""" + """Base class for WordPressMadara based extractors""" basecategory = "wpmadara" root = "https://www.mangaread.org" @@ -47,7 +47,7 @@ def parse_chapter_string(chapter_string, data): class WPMadaraChapterExtractor(WPMadaraBase, ChapterExtractor): - """Extractor for manga-chapters from mangaread.org""" + """Extractor for manga-chapters from WordPressMadara based websites.""" subcategory = "chapter" pattern = BASE_PATTERN + r"(/(manga|webtoon|read)/[^/?#]+/[^/?#]+)" example = "https://www.mangaread.org/manga/MANGA/chapter-01/" @@ -82,7 +82,7 @@ def images(self, page): class WPMadaraMangaExtractor(WPMadaraBase, MangaExtractor): - """Extractor for manga from mangaread.org""" + """Extractor for manga from WordPressMadara based websites.""" chapterclass = WPMadaraChapterExtractor subcategory = "manga" pattern = BASE_PATTERN + r"(/(manga|webtoon|read)/[^/?#]+)/?$" From 2c53c68c2ff03c098d28ba9ea4d830646ec329f6 Mon Sep 17 00:00:00 2001 From: Dragonatorul <109987846+Dragonatorul@users.noreply.github.com> Date: Sat, 7 Jun 2025 21:05:39 +0300 Subject: [PATCH 06/11] fix linting error --- gallery_dl/extractor/wpmadara.py | 1 + 1 file changed, 1 insertion(+) diff --git a/gallery_dl/extractor/wpmadara.py b/gallery_dl/extractor/wpmadara.py index 3df4037e13..c079893bba 100644 --- a/gallery_dl/extractor/wpmadara.py +++ b/gallery_dl/extractor/wpmadara.py @@ -30,6 +30,7 @@ def parse_chapter_string(chapter_string, data): data["lang"] = "en" data["language"] = "English" + BASE_PATTERN = WPMadaraBase.update({ "mangaread": { "root": "https://www.mangaread.org", From 7345da31df16c55cf013147ddc78efbfb3877f1e Mon Sep 17 00:00:00 2001 From: Dragonatorul <109987846+Dragonatorul@users.noreply.github.com> Date: Sat, 7 Jun 2025 21:06:00 +0300 Subject: [PATCH 07/11] fix: corrected tests --- test/results/mangaread.py | 16 ++++++++-------- test/results/toonily.py | 8 ++++---- test/results/webtoonxyz.py | 8 ++++---- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/test/results/mangaread.py b/test/results/mangaread.py index 3560bc65d9..aa76da7bc7 100644 --- a/test/results/mangaread.py +++ b/test/results/mangaread.py @@ -11,7 +11,7 @@ __tests__ = ( { "#url" : "https://www.mangaread.org/manga/one-piece/chapter-1053-3/", - "#category": ("", "mangaread", "chapter"), + "#category": ("wpmadara", "mangaread", "chapter"), "#class" : wpmadara.WPMadaraChapterExtractor, "#pattern" : r"https://www\.mangaread\.org/wp-content/uploads/WP-manga/data/manga_[^/]+/[^/]+/[^.]+\.\w+", "#count" : 11, @@ -27,15 +27,15 @@ { "#url" : "https://www.mangaread.org/manga/one-piece/chapter-1000000/", - "#category": ("", "mangaread", "chapter"), + "#category": ("wpmadara", "mangaread", "chapter"), "#class" : wpmadara.WPMadaraChapterExtractor, "#exception": exception.NotFoundError, }, { "#url" : "https://www.mangaread.org/manga/kanan-sama-wa-akumade-choroi/chapter-10/", - "#category": ("", "mangaread", "chapter"), - "#class" : wpmadara.WPMadaraMangaExtractor, + "#category": ("wpmadara", "mangaread", "chapter"), + "#class" : wpmadara.WPMadaraChapterExtractor, "#pattern" : r"https://www\.mangaread\.org/wp-content/uploads/WP-manga/data/manga_[^/]+/[^/]+/[^.]+\.\w+", "#count" : 9, @@ -51,8 +51,8 @@ { "#url" : "https://www.mangaread.org/manga/above-all-gods/chapter146-5/", "#comment" : "^^ no whitespace", - "#category": ("", "mangaread", "chapter"), - "#class" : wpmadara.WPMadaraMangaExtractor, + "#category": ("wpmadara", "mangaread", "chapter"), + "#class" : wpmadara.WPMadaraChapterExtractor, "#pattern" : r"https://www\.mangaread\.org/wp-content/uploads/WP-manga/data/manga_[^/]+/[^/]+/[^.]+\.\w+", "#count" : 6, @@ -67,7 +67,7 @@ { "#url" : "https://www.mangaread.org/manga/kanan-sama-wa-akumade-choroi", - "#category": ("", "mangaread", "manga"), + "#category": ("wpmadara", "mangaread", "manga"), "#class" : wpmadara.WPMadaraMangaExtractor, "#pattern" : r"https://www\.mangaread\.org/manga/kanan-sama-wa-akumade-choroi/chapter-\d+([_-].+)?/", "#count" : ">= 13", @@ -93,7 +93,7 @@ { "#url" : "https://www.mangaread.org/manga/one-piece", - "#category": ("", "mangaread", "manga"), + "#category": ("wpmadara", "mangaread", "manga"), "#class" : wpmadara.WPMadaraMangaExtractor, "#pattern" : r"https://www\.mangaread\.org/manga/one-piece/chapter-\d+(-.+)?/", "#count" : ">= 1066", diff --git a/test/results/toonily.py b/test/results/toonily.py index a309281aaa..b2f29b98b6 100644 --- a/test/results/toonily.py +++ b/test/results/toonily.py @@ -11,7 +11,7 @@ __tests__ = ( { "#url" : "https://toonily.com/webtoon/such-a-cute-spy/chapter-36/", - "#category": ("", "toonily", "chapter"), + "#category": ("wpmadara", "toonily", "chapter"), "#class" : wpmadara.WPMadaraChapterExtractor, "#pattern" : r"https://toonily\.com/wp-content/uploads/WP-manga/data/manga_[^/]+/[^/]+/[^.]+\.\w+", "#count" : 11, @@ -26,14 +26,14 @@ { "#url" : "https://toonily.com/webtoon/such-a-cute-spy/chapter-1000000/", - "#category": ("", "toonily", "chapter"), + "#category": ("wpmadara", "toonily", "chapter"), "#class" : wpmadara.WPMadaraChapterExtractor, "#exception": exception.NotFoundError, }, { "#url" : "https://toonily.com/webtoon/such-a-cute-spy", - "#category": ("", "toonily", "manga"), + "#category": ("wpmadara", "toonily", "manga"), "#class" : wpmadara.WPMadaraMangaExtractor, "#pattern" : r"https://toonily\.com/webtoon/such-a-cute-spy/chapter-\d+([_-].+)?/", "#count" : ">= 13", @@ -56,7 +56,7 @@ { "#url" : "https://toonily.com/webtoon/doesnotexist", - "#category": ("", "toonily", "manga"), + "#category": ("wpmadara", "toonily", "manga"), "#class" : wpmadara.WPMadaraMangaExtractor, "#exception": exception.HttpError, }, diff --git a/test/results/webtoonxyz.py b/test/results/webtoonxyz.py index 62ca7d38d0..18aa89faf9 100644 --- a/test/results/webtoonxyz.py +++ b/test/results/webtoonxyz.py @@ -11,7 +11,7 @@ __tests__ = ( { "#url" : "https://www.webtoon.xyz/read/the-world-after-the-end/chapter-105/", - "#category": ("", "wpmadara", "chapter"), + "#category": ("wpmadara", "webtoonxyz", "chapter"), "#class" : wpmadara.WPMadaraChapterExtractor, "#pattern" : r"https://www\.webtoon\.xyz/wp-content/uploads/WP-manga/data/manga_[^/]+/[^/]+/[^.]+\.\w+", "#count" : 11, @@ -25,14 +25,14 @@ { "#url" : "https://www.webtoon.xyz/read/the-world-after-the-end/chapter-1000000/", - "#category": ("", "wpmadara", "chapter"), + "#category": ("wpmadara", "webtoonxyz", "chapter"), "#class" : wpmadara.WPMadaraChapterExtractor, "#exception": exception.NotFoundError, }, { "#url" : "https://www.webtoon.xyz/read/the-world-after-the-end/", - "#category": ("", "wpmadara", "manga"), + "#category": ("wpmadara", "webtoonxyz", "manga"), "#class" : wpmadara.WPMadaraMangaExtractor, "#pattern" : r"https://www\.webtoon\.xyz/read/such-a-cute-spy/chapter-\d+([_-].+)?/", "#count" : ">= 13", @@ -54,7 +54,7 @@ { "#url" : "https://www.webtoon.xyz/read/doesnotexist", - "#category": ("", "wpmadara", "manga"), + "#category": ("wpmadara", "webtoonxyz", "manga"), "#class" : wpmadara.WPMadaraMangaExtractor, "#exception": exception.HttpError, }, From c1e8d36a2ab3a08255ebe131a91043e152cd0fc0 Mon Sep 17 00:00:00 2001 From: Dragonatorul <109987846+Dragonatorul@users.noreply.github.com> Date: Sat, 7 Jun 2025 21:06:00 +0300 Subject: [PATCH 08/11] fix rating for toonily --- gallery_dl/extractor/wpmadara.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/gallery_dl/extractor/wpmadara.py b/gallery_dl/extractor/wpmadara.py index c079893bba..599249bcaf 100644 --- a/gallery_dl/extractor/wpmadara.py +++ b/gallery_dl/extractor/wpmadara.py @@ -110,12 +110,19 @@ def chapters(self, page): def metadata(self, page): extr = text.extract_from(text.extr( page, 'class="summary_content">', 'class="manga-action"')) + # rating = 0.0 + if len(text.extr(page, 'total_votes">', "").strip()) > 0: + rating = text.parse_float(text.extr(page, 'total_votes">', "").strip()) + elif len(text.extr(page, 'property="ratingValue" id="averagerate">', "").strip()) > 0: + rating = text.parse_float(text.extr(page, 'property="ratingValue" id="averagerate">', "").strip()) + else: + rating = 0.0 + return { "manga" : text.extr(page, "

", "

").strip(), "description": text.unescape(text.remove_html(text.extract( page, ">", "
", page.index("summary__content"))[0])), - "rating" : text.parse_float( - extr('total_votes">', "").strip()), + "rating" : rating, "manga_alt" : text.remove_html( extr("Alternative\t\t\n\t
", "")).split("; "), "author" : list(text.extract_iter( From 62b74c82e416cefba2f036f6fedcb33683efee05 Mon Sep 17 00:00:00 2001 From: Dragonatorul <109987846+Dragonatorul@users.noreply.github.com> Date: Sat, 7 Jun 2025 21:06:00 +0300 Subject: [PATCH 09/11] fix line length --- gallery_dl/extractor/wpmadara.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/gallery_dl/extractor/wpmadara.py b/gallery_dl/extractor/wpmadara.py index 599249bcaf..9094a4e390 100644 --- a/gallery_dl/extractor/wpmadara.py +++ b/gallery_dl/extractor/wpmadara.py @@ -112,9 +112,13 @@ def metadata(self, page): page, 'class="summary_content">', 'class="manga-action"')) # rating = 0.0 if len(text.extr(page, 'total_votes">', "").strip()) > 0: - rating = text.parse_float(text.extr(page, 'total_votes">', "").strip()) - elif len(text.extr(page, 'property="ratingValue" id="averagerate">', "").strip()) > 0: - rating = text.parse_float(text.extr(page, 'property="ratingValue" id="averagerate">', "").strip()) + rating = text.parse_float(text.extr( + page, 'total_votes">', "").strip()) + elif len(text.extr(page, 'property="ratingValue" id="averagerate">', + "").strip()) > 0: + rating = text.parse_float(text.extr(page, + 'property="ratingValue" id="averagerate">', + "").strip()) else: rating = 0.0 From c66e5d76ab1d781b62603a408052b139da2ad0d9 Mon Sep 17 00:00:00 2001 From: Dragonatorul <109987846+Dragonatorul@users.noreply.github.com> Date: Sat, 7 Jun 2025 21:06:00 +0300 Subject: [PATCH 10/11] fix line indent --- gallery_dl/extractor/wpmadara.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/gallery_dl/extractor/wpmadara.py b/gallery_dl/extractor/wpmadara.py index 9094a4e390..2ad140e8b5 100644 --- a/gallery_dl/extractor/wpmadara.py +++ b/gallery_dl/extractor/wpmadara.py @@ -115,10 +115,11 @@ def metadata(self, page): rating = text.parse_float(text.extr( page, 'total_votes">', "").strip()) elif len(text.extr(page, 'property="ratingValue" id="averagerate">', - "").strip()) > 0: - rating = text.parse_float(text.extr(page, - 'property="ratingValue" id="averagerate">', - "").strip()) + "").strip()) > 0: + rating = text.parse_float( + text.extr(page, + 'property="ratingValue" id="averagerate">', + "").strip()) else: rating = 0.0 From 957e94b7f3404b59e8444d40b674a9763e350c8c Mon Sep 17 00:00:00 2001 From: Dragonatorul <109987846+Dragonatorul@users.noreply.github.com> Date: Sat, 7 Jun 2025 21:06:00 +0300 Subject: [PATCH 11/11] fix line indent --- gallery_dl/extractor/wpmadara.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gallery_dl/extractor/wpmadara.py b/gallery_dl/extractor/wpmadara.py index 2ad140e8b5..929f7e884c 100644 --- a/gallery_dl/extractor/wpmadara.py +++ b/gallery_dl/extractor/wpmadara.py @@ -115,7 +115,7 @@ def metadata(self, page): rating = text.parse_float(text.extr( page, 'total_votes">', "").strip()) elif len(text.extr(page, 'property="ratingValue" id="averagerate">', - "").strip()) > 0: + "").strip()) > 0: rating = text.parse_float( text.extr(page, 'property="ratingValue" id="averagerate">',