diff --git a/.github/release.py b/.github/release.py index 4493df47..8cc8eaf0 100644 --- a/.github/release.py +++ b/.github/release.py @@ -18,7 +18,7 @@ def parse_body(body): e: str = e.strip() if e == '': continue - points.append(f'{i}. {e}') + points.append(f'{i + 1}. {e}') return '\n'.join(points) diff --git a/src/jmcomic/__init__.py b/src/jmcomic/__init__.py index 0a3ab30c..45806004 100644 --- a/src/jmcomic/__init__.py +++ b/src/jmcomic/__init__.py @@ -2,7 +2,7 @@ # 被依赖方 <--- 使用方 # config <--- entity <--- toolkit <--- client <--- option <--- downloader -__version__ = '2.3.15' +__version__ = '2.3.16' from .api import * from .jm_plugin import * diff --git a/src/jmcomic/jm_client_impl.py b/src/jmcomic/jm_client_impl.py index 38474559..f610fb36 100644 --- a/src/jmcomic/jm_client_impl.py +++ b/src/jmcomic/jm_client_impl.py @@ -493,13 +493,13 @@ def fetch_scramble_id(self, photo_id): } ) - match = JmcomicText.pattern_html_album_scramble_id.search(resp.text) - - if match is not None: - scramble_id = match[1] - else: - jm_debug('api.scramble', '未从响应中匹配到scramble_id,返回默认值220980') - scramble_id = '220980' + scramble_id = PatternTool.match_or_default(resp.text, + JmcomicText.pattern_html_album_scramble_id, + None, + ) + if scramble_id is None: + jm_debug('api.scramble', f'未匹配到scramble_id,响应文本:{resp.text}') + scramble_id = str(JmModuleConfig.SCRAMBLE_220980) return scramble_id @@ -610,15 +610,12 @@ def require_resp_success(cls, resp: JmApiResp, orig_req_url: str): # 2. 是否是特殊的内容 # 暂无 - @classmethod - @field_cache('__init_cookies__') - def fetch_init_cookies(cls, client: 'JmApiClient'): - resp = client.setting() - return dict(resp.resp.cookies) - def after_init(self): - cookies = self.__class__.fetch_init_cookies(self) - self.get_root_postman().get_meta_data()['cookies'] = cookies + # cookies = self.__class__.fetch_init_cookies(self) + # self.get_root_postman().get_meta_data()['cookies'] = cookies + + self.get_root_postman().get_meta_data()['cookies'] = JmModuleConfig.get_cookies(self) + pass class FutureClientProxy(JmcomicClient): diff --git a/src/jmcomic/jm_config.py b/src/jmcomic/jm_config.py index 467aa8bb..1c3e14af 100644 --- a/src/jmcomic/jm_config.py +++ b/src/jmcomic/jm_config.py @@ -3,7 +3,7 @@ def field_cache(*args, **kwargs): return field_cache(*args, **kwargs) -def default_jm_debug(topic: str, msg: str): +def default_jm_debug_logging(topic: str, msg: str): from common import format_ts print(f'{format_ts()}:【{topic}】{msg}') @@ -62,13 +62,14 @@ class JmModuleConfig: SCRAMBLE_421926 = 421926 # 2023-02-08后改了图片切割算法 SCRAMBLE_CACHE = {} - # 移动端API的相关配置 - # API密钥 - APP_SECRET = '18comicAPP' + # 移动端API密钥 + APP_SECRET = '18comicAPPContent' - # 域名配置 - 移动端 - # 图片域名 - DOMAIN_API_IMAGE_LIST = str_to_list(''' + # cookies,目前只在移动端使用,因为移动端请求接口须携带,但不会校验cookies的内容。 + APP_COOKIES = None + + # 移动端图片域名 + DOMAIN_IMAGE_LIST = str_to_list(''' cdn-msp.jmapiproxy1.monster cdn-msp2.jmapiproxy1.monster cdn-msp.jmapiproxy1.cc @@ -78,7 +79,7 @@ class JmModuleConfig: ''') - # API域名 + # 移动端API域名 DOMAIN_API_LIST = str_to_list(''' www.jmapinode1.top www.jmapinode2.top @@ -88,8 +89,11 @@ class JmModuleConfig: ''') - # 域名配置 - 网页端 + # 网页端域名配置 # 无需配置,默认为None,需要的时候会发起请求获得 + # 使用优先级: + # 1. DOMAIN_HTML_LIST + # 2. [DOMAIN_HTML] DOMAIN_HTML = None DOMAIN_HTML_LIST = None @@ -106,7 +110,7 @@ class JmModuleConfig: REGISTRY_PLUGIN = {} # 执行debug的函数 - debug_executor = default_jm_debug + debug_executor = default_jm_debug_logging # postman构造函数 postman_constructor = default_postman_constructor # 网页正则表达式解析失败时,执行抛出异常的函数,可以替换掉用于debug @@ -190,7 +194,7 @@ def get_html_url(cls, postman=None): postman = postman or cls.new_postman(session=True) url = postman.with_redirect_catching().get(cls.JM_REDIRECT_URL) - cls.jm_debug('获取禁漫网页URL', f'[{cls.JM_REDIRECT_URL}] → [{url}]') + cls.jm_debug('module.html_url', f'获取禁漫网页URL: [{cls.JM_REDIRECT_URL}] → [{url}]') return url @classmethod @@ -211,9 +215,22 @@ def get_html_domain_all(cls, postman=None): from .jm_toolkit import JmcomicText domain_list = JmcomicText.analyse_jm_pub_html(resp.text) - cls.jm_debug('获取禁漫网页全部域名', f'[{resp.url}] → {domain_list}') + cls.jm_debug('module.html_domain_all', f'获取禁漫网页全部域名: [{resp.url}] → {domain_list}') return domain_list + @classmethod + @field_cache("APP_COOKIES") + def get_cookies(cls, postman=None): + from .jm_toolkit import JmcomicText + url = JmcomicText.format_url('/setting', cls.DOMAIN_API_LIST[0]) + postman = postman or cls.new_postman() + + resp = postman.get(url) + cookies = dict(resp.cookies) + + cls.jm_debug('module.cookies', f'获取cookies: [{url}] → {cookies}') + return cookies + @classmethod def new_html_headers(cls, domain='18comic.vip'): """ @@ -247,7 +264,7 @@ def new_api_headers(cls, key_ts): key_ts = time_stamp() import hashlib - token = hashlib.md5(f"{key_ts}{cls.APP_SECRET}".encode()).hexdigest() + token = hashlib.md5(f"{key_ts}{cls.APP_SECRET}".encode("utf-8")).hexdigest() return { 'token': token, diff --git a/src/jmcomic/jm_option.py b/src/jmcomic/jm_option.py index 72501a47..73406a06 100644 --- a/src/jmcomic/jm_option.py +++ b/src/jmcomic/jm_option.py @@ -314,6 +314,7 @@ def build_jm_client(self, **kwargs): def new_jm_client(self, domain=None, impl=None, cache=None, **kwargs) -> JmcomicClient: # 所有需要用到的 self.client 配置项如下 postman_conf: dict = self.client.postman.src_dict # postman dsl 配置 + meta_data: dict = postman_conf['meta_data'] # 请求元信息 impl: str = impl or self.client.impl # client_key retry_times: int = self.client.retry_times # 重试次数 cache: str = cache or self.client.cache # 启用缓存 @@ -335,15 +336,11 @@ def decide_domain(): # support kwargs overwrite meta_data if len(kwargs) != 0: - postman_conf['meta_data'].update(kwargs) + meta_data.update(kwargs) # headers - meta_data = postman_conf['meta_data'] if meta_data['headers'] is None: - headers = self.decide_postman_headers(impl, domain[0]) - # if headers is None: - # postman_conf['type'] = 'requests' - meta_data['headers'] = headers + meta_data['headers'] = self.decide_postman_headers(impl, domain[0]) # postman postman = Postmans.create(data=postman_conf) @@ -375,6 +372,9 @@ def decide_client_domain(self, client_key: str) -> List[str]: if is_client_type(JmHtmlClient): # 网页端 + domain_list = JmModuleConfig.DOMAIN_HTML_LIST + if domain_list is not None: + return domain_list return [JmModuleConfig.get_html_domain()] ExceptionTool.raises(f'没有配置域名,且是无法识别的client类型: {client_key}') diff --git a/src/jmcomic/jm_plugin.py b/src/jmcomic/jm_plugin.py index d936a61d..765a6e15 100644 --- a/src/jmcomic/jm_plugin.py +++ b/src/jmcomic/jm_plugin.py @@ -249,26 +249,32 @@ def invoke(self, mkdir_if_not_exists(zip_dir) # 原文件夹 -> zip文件 - dir_zip_dict = {} + dir_zip_dict: Dict[str, Optional[str]] = {} photo_dict = downloader.all_downloaded[album] if level == 'album': zip_path = self.get_zip_path(album, None, filename_rule, suffix, zip_dir) dir_path = self.zip_album(album, photo_dict, zip_path) - dir_zip_dict[dir_path] = zip_path + if dir_path is not None: + # 要删除这个album文件夹 + dir_zip_dict[dir_path] = zip_path + # 也要删除album下的photo文件夹 + for d in files_of_dir(dir_path): + dir_zip_dict[d] = None elif level == 'photo': for photo, image_list in photo_dict.items(): zip_path = self.get_zip_path(None, photo, filename_rule, suffix, zip_dir) dir_path = self.zip_photo(photo, image_list, zip_path) - dir_zip_dict[dir_path] = zip_path + if dir_path is not None: + dir_zip_dict[dir_path] = zip_path else: ExceptionTool.raises(f'Not Implemented Zip Level: {level}') self.after_zip(dir_zip_dict) - def zip_photo(self, photo, image_list: list, zip_path: str): + def zip_photo(self, photo, image_list: list, zip_path: str) -> Optional[str]: """ 压缩photo文件夹 :returns: photo文件夹路径 @@ -277,46 +283,54 @@ def zip_photo(self, photo, image_list: list, zip_path: str): if len(image_list) == 0 \ else os.path.dirname(image_list[0][0]) - all_filepath = set(map(lambda t: t[0], image_list)) + all_filepath = set(map(lambda t: self.unified_path(t[0]), image_list)) - if len(all_filepath) == 0: - self.debug('无下载文件,无需压缩', 'skip') - return - - from common import backup_dir_to_zip - backup_dir_to_zip(photo_dir, zip_path, acceptor=lambda f: f in all_filepath) - self.debug(f'压缩章节[{photo.photo_id}]成功 → {zip_path}', 'finish') + return self.do_zip(photo_dir, + zip_path, + all_filepath, + f'压缩章节[{photo.photo_id}]成功 → {zip_path}', + ) - return photo_dir + @staticmethod + def unified_path(f): + return fix_filepath(f, os.path.isdir(f)) - def zip_album(self, album, photo_dict: dict, zip_path): + def zip_album(self, album, photo_dict: dict, zip_path) -> Optional[str]: """ 压缩album文件夹 :returns: album文件夹路径 """ - album_dir = self.option.decide_album_dir(album) all_filepath: Set[str] = set() - for image_list in photo_dict.values(): - image_list: List[Tuple[str, JmImageDetail]] - for path, _ in image_list: - all_filepath.add(path) + def addpath(f): + all_filepath.update(set(f)) + + album_dir = self.option.decide_album_dir(album) + # addpath(self.option.decide_image_save_dir(photo) for photo in photo_dict.keys()) + addpath(path for ls in photo_dict.values() for path, _ in ls) + + return self.do_zip(album_dir, + zip_path, + all_filepath, + msg=f'压缩本子[{album.album_id}]成功 → {zip_path}', + ) + def do_zip(self, source_dir, zip_path, all_filepath, msg): if len(all_filepath) == 0: self.debug('无下载文件,无需压缩', 'skip') - return + return None from common import backup_dir_to_zip backup_dir_to_zip( - album_dir, + source_dir, zip_path, - acceptor=lambda f: f in all_filepath - ) + acceptor=lambda f: os.path.isdir(f) or self.unified_path(f) in all_filepath + ).close() - self.debug(f'压缩本子[{album.album_id}]成功 → {zip_path}', 'finish') - return album_dir + self.debug(msg, 'finish') + return self.unified_path(source_dir) - def after_zip(self, dir_zip_dict: Dict[str, str]): + def after_zip(self, dir_zip_dict: Dict[str, Optional[str]]): # 是否要删除所有原文件 if self.delete_original_file is True: self.delete_all_files_and_empty_dir( @@ -352,10 +366,10 @@ def delete_all_files_and_empty_dir(self, all_downloaded: dict, dir_list: List[st os.remove(f) self.debug(f'删除原文件: {f}', 'remove') - for d in dir_list: + for d in sorted(dir_list, reverse=True): # check exist - if file_exists(d) and len(os.listdir(d)) == 0: - os.removedirs(d) + if file_exists(d): + os.rmdir(d) self.debug(f'删除文件夹: {d}', 'remove') diff --git a/src/jmcomic/jm_toolkit.py b/src/jmcomic/jm_toolkit.py index a6e6321a..23536884 100644 --- a/src/jmcomic/jm_toolkit.py +++ b/src/jmcomic/jm_toolkit.py @@ -489,7 +489,7 @@ def post_adapt_photo(cls, data: dict, _clazz: type, fields: dict): fields['sort'] = sort import random - fields['data_original_domain'] = random.choice(JmModuleConfig.DOMAIN_API_IMAGE_LIST) + fields['data_original_domain'] = random.choice(JmModuleConfig.DOMAIN_IMAGE_LIST) class JmImageTool: diff --git a/tests/test_jmcomic/test_jm_custom.py b/tests/test_jmcomic/test_jm_custom.py index f3cfeebb..9e6015e1 100644 --- a/tests/test_jmcomic/test_jm_custom.py +++ b/tests/test_jmcomic/test_jm_custom.py @@ -55,8 +55,10 @@ class MyClient(JmHtmlClient): JmModuleConfig.register_client(MyClient) html_domain = self.client.get_html_domain() + JmModuleConfig.DOMAIN_HTML_LIST = [html_domain] + self.assertListEqual( - [html_domain], + JmModuleConfig.DOMAIN_HTML_LIST, self.option.new_jm_client(domain=[], impl=MyClient.client_key).get_domain_list() )