diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 00000000..0f04a246 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,23 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: '' +assignees: '' + +--- + +**需知** + +升级feapder，保证feapder是最新版，若BUG仍然存在，则详细描述问题 +> pip install --upgrade feapder + +**问题** + +**截图** + +**代码** + +```python + +``` diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 00000000..9ab3c9b8 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,6 @@ +# https://docs.github.com/en/github/building-a-strong-community/configuring-issue-templates-for-your-repository#configuring-the-template-chooser +blank_issues_allowed: false # We have a blank template which assigns labels +contact_links: + - name: Questions about using feapder? + url: "https://github.com/Boris-code/feapder/discussions" + about: Please see our guide on how to ask questions \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 00000000..bbcbbe7d --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: '' +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/.github/scripts/expire_sponsor.py b/.github/scripts/expire_sponsor.py new file mode 100644 index 00000000..8c3e6f34 --- /dev/null +++ b/.github/scripts/expire_sponsor.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import os +from datetime import date, datetime +from pathlib import Path +from zoneinfo import ZoneInfo + + +README = Path("README.md") +START_MARKER = "" +END_MARKER = "" +EXPIRE_ON = date.fromisoformat(os.environ.get("SPONSOR_EXPIRE_ON", "2026-08-28")) +TIMEZONE = ZoneInfo(os.environ.get("SPONSOR_TIMEZONE", "Asia/Shanghai")) + + +def current_date() -> date: + override = os.environ.get("SPONSOR_TODAY") + if override: + return date.fromisoformat(override) + return datetime.now(TIMEZONE).date() + + +def main() -> None: + today = current_date() + if today < EXPIRE_ON: + print(f"Sponsor block is still active until {EXPIRE_ON}; today is {today}.") + return + + text = README.read_text(encoding="utf-8") + start = text.find(START_MARKER) + end = text.find(END_MARKER) + + if start == -1 and end == -1: + print("Sponsor block markers are absent; nothing to remove.") + return + if start == -1 or end == -1 or end < start: + raise SystemExit("Sponsor block markers are incomplete or out of order.") + + end += len(END_MARKER) + updated = text[:start].rstrip() + "\n\n" + text[end:].lstrip() + + if updated == text: + print("Sponsor block is already removed.") + return + + README.write_text(updated, encoding="utf-8") + print(f"Removed sponsor block on {today}.") + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/expire-sponsor.yml b/.github/workflows/expire-sponsor.yml new file mode 100644 index 00000000..5e88f5c1 --- /dev/null +++ b/.github/workflows/expire-sponsor.yml @@ -0,0 +1,33 @@ +name: Expire sponsor block + +on: + schedule: + # 00:05 in Asia/Shanghai. + - cron: "5 16 * * *" + workflow_dispatch: + +permissions: + contents: write + +jobs: + remove-sponsor: + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Remove expired sponsor block + run: python3 .github/scripts/expire_sponsor.py + + - name: Commit README update + run: | + if git diff --quiet; then + echo "No README changes to commit." + exit 0 + fi + + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add README.md + git commit -m "Remove expired sponsor block" + git push diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml new file mode 100644 index 00000000..e69de29b diff --git a/.gitignore b/.gitignore index b584a9b9..721f067a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ files/* +docs/superpowers/ .DS_Store .idea/* */.idea/* @@ -14,3 +15,5 @@ dist/ .vscode/ media/ .MWebMetaData/ +push.sh +assets/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..63d42cb0 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,15 @@ +# 贡献指南 +感谢你的宝贵时间。你的贡献将使这个项目变得更好！在提交贡献之前，请务必花点时间阅读下面的入门指南。 + +## 提交 Pull Request +1. Fork [此仓库](https://github.com/Boris-code/feapder.git)， +2. clone到本地，从 `develop` 创建分支，对代码进行更改。 +3. 请确保进行了相应的测试。 +4. 推送代码到自己Fork的仓库中。 +5. 在Fork的仓库中点击 Pull request 链接 +6. 点击「New pull request」按钮。 +7. 填写提交说明后，「Create pull request」。提交到`develop`分支。 + +## License + +[MIT](./LICENSE) diff --git a/LICENSE b/LICENSE index ceeaa8af..9ce08381 100644 --- a/LICENSE +++ b/LICENSE @@ -2,7 +2,7 @@ MIT License Modifications: -Copyright (c) 2020 Boris +Copyright (c) 2020 Boris Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/MANIFEST.in b/MANIFEST.in index 6550b730..6649336f 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,8 +4,8 @@ include LICENSE include feapder/requirements.txt include feapder/VERSION +recursive-include feapder/utils/js * recursive-include feapder/templates * - recursive-include tests * global-exclude __pycache__ *.py[cod] \ No newline at end of file diff --git a/README.md b/README.md index faaaa083..210733b2 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,30 @@ # FEAPDER ![](https://img.shields.io/badge/python-3.6-brightgreen) +![](https://img.shields.io/github/watchers/Boris-code/feapder?style=social) +![](https://img.shields.io/github/stars/Boris-code/feapder?style=social) +![](https://img.shields.io/github/forks/Boris-code/feapder?style=social) +[![Downloads](https://pepy.tech/badge/feapder)](https://pepy.tech/project/feapder) +[![Downloads](https://pepy.tech/badge/feapder/month)](https://pepy.tech/project/feapder) +[![Downloads](https://pepy.tech/badge/feapder/week)](https://pepy.tech/project/feapder) ## 简介 -**feapder** 是一款简单、快速、轻量级的爬虫框架。起名源于 fast、easy、air、pro、spider的缩写，以开发快速、抓取快速、使用简单、功能强大为宗旨，历时4年倾心打造。支持轻量爬虫、分布式爬虫、批次爬虫、爬虫集成，以及完善的爬虫报警机制。 - -之前一直在公司内部使用，已使用本框架采集100+数据源，日采千万数据。现在开源，供大家学习交流！ +1. feapder是一款上手简单，功能强大的Python爬虫框架，内置AirSpider、Spider、TaskSpider、BatchSpider四种爬虫解决不同场景的需求。 +2. 支持断点续爬、监控报警、浏览器渲染、海量数据去重等功能。 +3. 更有功能强大的爬虫管理系统feaplat为其提供方便的部署及调度 读音: `[ˈfiːpdə]` -官方文档：http://boris.org.cn/feapder/ +![feapder](http://markdown-media.oss-cn-beijing.aliyuncs.com/2023/09/04/feapder.jpg) + + +## 文档地址 + +- 官方文档：https://feapder.com +- github：https://github.com/Boris-code/feapder +- 更新日志：https://github.com/Boris-code/feapder/releases +- 爬虫管理系统：http://feapder.com/#/feapder_platform/feaplat ## 环境要求： @@ -22,209 +36,136 @@ From PyPi: - pip3 install feapder - -From Git: - - pip3 install git+https://github.com/Boris-code/feapder.git - -若安装出错，请参考[安装问题](https://boris.org.cn/feapder/#/question/%E5%AE%89%E8%A3%85%E9%97%AE%E9%A2%98) - -## 小试一下 - -创建爬虫 - - feapder create -s first_spider - -创建后的爬虫代码如下： - - - import feapder - - - class FirstSpider(feapder.AirSpider): - def start_requests(self): - yield feapder.Request("https://www.baidu.com") - - def parse(self, request, response): - print(response) - - - if __name__ == "__main__": - FirstSpider().start() - -直接运行，打印如下： - - Thread-2|2021-02-09 14:55:11,373|request.py|get_response|line:283|DEBUG| - -------------- FirstSpider.parser request for ---------------- - url = https://www.baidu.com - method = GET - body = {'timeout': 22, 'stream': True, 'verify': False, 'headers': {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36'}} - - - Thread-2|2021-02-09 14:55:11,610|parser_control.py|run|line:415|INFO| parser 等待任务 ... - FirstSpider|2021-02-09 14:55:14,620|air_spider.py|run|line:80|DEBUG| 无任务，爬虫结束 - -代码解释如下： - -1. start_requests： 生产任务 -2. parser： 解析数据 - -## 为什么不使用scrapy - -scrapy给我的印象： - -1. 重，框架中的许多东西都用不到，如CrawlSpider、XMLFeedSpider -2. 中间件不灵活 -3. 不支持从数据库中取任务作为种子抓取 -4. 数据入库不支持批量，需要自己写批量逻辑 -5. 启动方式需要用scrapy命令行，打断点调试不方便 - -### 举例说明 - -本文以某东的商品爬虫为例，假如我们有1亿个商品，需要每7天全量更新一次，如何做呢？ +精简版 -#### 1. 准备种子任务 - -首先需要个种子任务表来存储这些商品id，设计表如下： - -![-w1028](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/03/09/16152931277517.jpg?x-oss-process=style/markdown-media) - -```sql -CREATE TABLE `jd_item_task` ( - `id` int(11) NOT NULL AUTO_INCREMENT, - `item_id` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL COMMENT '商品id', - `state` int(11) DEFAULT '0' COMMENT '任务状态 0 待抓取 1 抓取成功 2 抓取中 -1 抓取失败', - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; +```shell +pip install feapder ``` -然后将这1亿个商品id录入进来，作为种子任务 +浏览器渲染版： +```shell +pip install "feapder[render]" +``` -![-w357](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/03/09/16152932156268.jpg?x-oss-process=style/markdown-media) +完整版： -#### 2. 准备数据表 +```shell +pip install "feapder[all]" +``` -![-w808](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/03/09/16152934374807.jpg?x-oss-process=style/markdown-media) +三个版本区别： -```sql -CREATE TABLE `jd_item` ( - `id` int(11) NOT NULL AUTO_INCREMENT, - `title` varchar(255) DEFAULT NULL, - `batch_date` date DEFAULT NULL COMMENT '批次时间', - `crawl_time` datetime DEFAULT NULL COMMENT '采集时间', - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; -``` +1. 精简版：不支持浏览器渲染、不支持基于内存去重、不支持入库mongo +2. 浏览器渲染版：不支持基于内存去重、不支持入库mongo +3. 完整版：支持所有功能 -需求是每7天全量更新一次，即数据要以7天为维度划分，因此设置个`batch_date`字段，表示每条数据所属的批次。 +完整版可能会安装出错，若安装出错，请参考[安装问题](docs/question/安装问题.md) -这里只是演示，因此只采集标题字段 +## 小试一下 -#### 3. 采集 +创建爬虫 -若使用`scrapy`，需要手动将这些种子任务分批取出来发给爬虫，还需要维护种子任务的状态，以及上面提及的批次信息`batch_date`。并且为了保证数据的时效性，需要对采集进度进行监控，写个爬虫十分繁琐。 +```shell +feapder create -s first_spider +``` -而`feapder`内置了批次爬虫，可以很方便的应对这个需求。完整的爬虫写法如下： +创建后的爬虫代码如下： ```python import feapder -from feapder import Item -from feapder.utils import tools - - -class JdSpider(feapder.BatchSpider): - # 自定义数据库，若项目中有setting.py文件，此自定义可删除 - __custom_setting__ = dict( - REDISDB_IP_PORTS="localhost:6379", - REDISDB_DB=0, - MYSQL_IP="localhost", - MYSQL_PORT=3306, - MYSQL_DB="feapder", - MYSQL_USER_NAME="feapder", - MYSQL_USER_PASS="feapder123", - ) - - def start_requests(self, task): - task_id, item_id = task - url = "https://item.jd.com/{}.html".format(item_id) - yield feapder.Request(url, task_id=task_id) # 携带task_id字段 - def parse(self, request, response): - title = response.xpath("string(//div[@class='sku-name'])").extract_first(default="").strip() - item = Item() - item.table_name = "jd_item" # 指定入库的表名 - item.title = title - item.batch_date = self.batch_date # 获取批次信息，批次信息框架自己维护 - item.crawl_time = tools.get_current_date() # 获取当前时间 - yield item # 自动批量入库 - yield self.update_task_batch(request.task_id, 1) # 更新任务状态 +class FirstSpider(feapder.AirSpider): + def start_requests(self): + yield feapder.Request("https://www.baidu.com") + + def parse(self, request, response): + print(response) if __name__ == "__main__": - spider = JdSpider( - redis_key="feapder:jd_item", # redis中存放任务等信息key前缀 - task_table="jd_item_task", # mysql中的任务表 - task_keys=["id", "item_id"], # 需要获取任务表里的字段名，可添加多个 - task_state="state", # mysql中任务状态字段 - batch_record_table="jd_item_batch_record", # mysql中的批次记录表，自动生成 - batch_name="京东商品爬虫(周度全量)", # 批次名字 - batch_interval=7, # 批次周期 天为单位 若为小时 可写 1 / 24 - ) - - # 下面两个启动函数 相当于 master、worker。需要分开运行 - spider.start_monitor_task() # maser: 下发及监控任务 - # spider.start() # worker: 采集 + FirstSpider().start() + +``` +直接运行，打印如下： + +```shell +Thread-2|2021-02-09 14:55:11,373|request.py|get_response|line:283|DEBUG| + -------------- FirstSpider.parse request for ---------------- + url = https://www.baidu.com + method = GET + body = {'timeout': 22, 'stream': True, 'verify': False, 'headers': {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36'}} + + +Thread-2|2021-02-09 14:55:11,610|parser_control.py|run|line:415|DEBUG| parser 等待任务... +FirstSpider|2021-02-09 14:55:14,620|air_spider.py|run|line:80|INFO| 无任务，爬虫结束 ``` -我们分别运行`spider.start_monitor_task()`与`spider.start()`，待爬虫结束后，观察数据库 +代码解释如下： -**任务表**：`jd_item_task` +1. start_requests： 生产任务 +2. parse： 解析数据 -![-w282](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/03/09/16152953028811.jpg?x-oss-process=style/markdown-media) -任务均已完成了，框架有任务丢失重发机制，直到所有任务均已做完 +## 感谢以下代理赞助商 -**数据表**：`jd_item`: + -![-w569](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/03/09/16152952623851.jpg?x-oss-process=style/markdown-media) +### LokiProxy -数据里携带了批次时间信息，我们可以根据这个时间来对数据进行划分。当前批次为3月9号，若7天一批次，则下一批次为3月18号。 +> 支持免费测试，住宅代理低至 $0.48/GB，移动代理低至 $0.91/GB，不限量住宅代理仅 $12/小时即可体验 ⚡， 支持ASN精准定位及无限并发会话，高稳定、高性价比 + -**批次表**：`jd_item_batch_record` +![](https://markdown-media.oss-cn-beijing.aliyuncs.com/2026/07/21/17846416079896.jpg) -![-w901](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/03/09/16152953428596.jpg?x-oss-process=style/markdown-media) +链接： https://www.lokiproxy.com/?utm_t=1&utm_i=158 + -启动参数中指定，自动生成。批次表里详细记录了每个批次的抓取状态，如任务总量、已做量、失败量、是否已完成等信息 + -#### 4. 监控 -feapder会自动维护任务状态，每个批次（采集周期）的进度，并且内置丰富的报警，保证我们的数据时效性，如： +## 参与贡献 -1. 实时计算爬虫抓取速度，估算剩余时间，在指定的抓取周期内预判是否会超时 +贡献之前请先阅读 [贡献指南](./CONTRIBUTING.md) - ![-w657](http://markdown-media.oss-cn-beijing.aliyuncs.com/2020/12/20/16084718683378.jpg?x-oss-process=style/markdown-media) +感谢所有做过贡献的人! + + + -2. 爬虫卡死报警 +## 爬虫工具推荐 - ![-w501](http://markdown-media.oss-cn-beijing.aliyuncs.com/2020/12/20/16084718974597.jpg?x-oss-process=style/markdown-media) +1. 爬虫在线工具库：http://www.spidertools.cn +2. 爬虫管理系统：http://feapder.com/#/feapder_platform/feaplat +3. 验证码识别库：https://github.com/sml2h3/ddddocr -3. 爬虫任务失败数过多报警，可能是由于网站模板改动或封堵导致 +## 微信赞赏 - ![-w416](http://markdown-media.oss-cn-beijing.aliyuncs.com/2020/12/29/16092335882158.jpg?x-oss-process=style/markdown-media) +如果您觉得这个项目帮助到了您，您可以帮作者买一杯咖啡表示鼓励 🍹 -1. 下载情况监控 +也可和作者交个朋友，解决您在使用过程中遇到的问题 - ![-w1299](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/02/09/16128568548280.jpg?x-oss-process=style/markdown-media) +![赞赏码](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/03/16/zan-shang-ma.png) ## 学习交流 -知识星球： + + + + + + + + + + + +
知识星球：17321694 作者微信： boris_tm QQ群号：521494615
+
+ -![知识星球](http://markdown-media.oss-cn-beijing.aliyuncs.com/2020/02/16/zhi-shi-xing-qiu.jpeg) -星球会不定时分享爬虫技术干货，涉及的领域包括但不限于js逆向技巧、爬虫框架刨析、爬虫技术分享等 \ No newline at end of file + 加好友备注：feapder diff --git a/docs/README.md b/docs/README.md index 94f98b6f..08ccb6aa 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,16 +1,29 @@ # FEAPDER ![](https://img.shields.io/badge/python-3.6-brightgreen) +![](https://img.shields.io/github/watchers/Boris-code/feapder?style=social) +![](https://img.shields.io/github/stars/Boris-code/feapder?style=social) +![](https://img.shields.io/github/forks/Boris-code/feapder?style=social) +[![Downloads](https://pepy.tech/badge/feapder)](https://pepy.tech/project/feapder) +[![Downloads](https://pepy.tech/badge/feapder/month)](https://pepy.tech/project/feapder) +[![Downloads](https://pepy.tech/badge/feapder/week)](https://pepy.tech/project/feapder) ## 简介 -**feapder** 是一款简单、快速、轻量级的爬虫框架。起名源于 fast、easy、air、pro、spider的缩写，以开发快速、抓取快速、使用简单、功能强大为宗旨，历时4年倾心打造。支持轻量爬虫、分布式爬虫、批次爬虫、爬虫集成，以及完善的爬虫报警机制。 - -之前一直在公司内部使用，已使用本框架采集100+数据源，日采千万数据。现在开源，供大家学习交流！ +1. feapder是一款上手简单，功能强大的Python爬虫框架，内置AirSpider、Spider、TaskSpider、BatchSpider四种爬虫解决不同场景的需求。 +2. 支持断点续爬、监控报警、浏览器渲染、海量数据去重等功能。 +3. 更有功能强大的爬虫管理系统feaplat为其提供方便的部署及调度 读音: `[ˈfiːpdə]` -官方文档：http://boris.org.cn/feapder/ +![feapder](http://markdown-media.oss-cn-beijing.aliyuncs.com/2023/09/04/feapder.jpg) + +## 文档地址 + +- 官方文档：https://feapder.com +- github：https://github.com/Boris-code/feapder +- 更新日志：https://github.com/Boris-code/feapder/releases +- 爬虫管理系统：http://feapder.com/#/feapder_platform/feaplat ## 环境要求： @@ -22,209 +35,110 @@ From PyPi: - pip3 install feapder - -From Git: - - pip3 install git+https://github.com/Boris-code/feapder.git - -若安装出错，请参考[安装问题](question/安装问题) - -## 小试一下 - -创建爬虫 - - feapder create -s first_spider - -创建后的爬虫代码如下： - - - import feapder - - - class FirstSpider(feapder.AirSpider): - def start_requests(self): - yield feapder.Request("https://www.baidu.com") - - def parse(self, request, response): - print(response) - - - if __name__ == "__main__": - FirstSpider().start() - -直接运行，打印如下： - - Thread-2|2021-02-09 14:55:11,373|request.py|get_response|line:283|DEBUG| - -------------- FirstSpider.parser request for ---------------- - url = https://www.baidu.com - method = GET - body = {'timeout': 22, 'stream': True, 'verify': False, 'headers': {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36'}} - - - Thread-2|2021-02-09 14:55:11,610|parser_control.py|run|line:415|INFO| parser 等待任务 ... - FirstSpider|2021-02-09 14:55:14,620|air_spider.py|run|line:80|DEBUG| 无任务，爬虫结束 - -代码解释如下： - -1. start_requests： 生产任务 -2. parser： 解析数据 - -## 为什么不使用scrapy - -scrapy给我的印象： +精简版 -1. 重，框架中的许多东西都用不到，如CrawlSpider、XMLFeedSpider -2. 中间件不灵活 -3. 不支持从数据库中取任务作为种子抓取 -4. 数据入库不支持批量，需要自己写批量逻辑 -5. 启动方式需要用scrapy命令行，打断点调试不方便 - -### 举例说明 +```shell +pip install feapder +``` -本文以某东的商品爬虫为例，假如我们有1亿个商品，需要每7天全量更新一次，如何做呢？ +浏览器渲染版： +```shell +pip install "feapder[render]" +``` -#### 1. 准备种子任务 +完整版： -首先需要个种子任务表来存储这些商品id，设计表如下： +```shell +pip install "feapder[all]" +``` -![-w1028](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/03/09/16152931277517.jpg?x-oss-process=style/markdown-media) +三个版本区别： -```sql -CREATE TABLE `jd_item_task` ( - `id` int(11) NOT NULL AUTO_INCREMENT, - `item_id` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL COMMENT '商品id', - `state` int(11) DEFAULT '0' COMMENT '任务状态 0 待抓取 1 抓取成功 2 抓取中 -1 抓取失败', - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; -``` +1. 精简版：不支持浏览器渲染、不支持基于内存去重、不支持入库mongo +2. 浏览器渲染版：不支持基于内存去重、不支持入库mongo +3. 完整版：支持所有功能 -然后将这1亿个商品id录入进来，作为种子任务 -![-w357](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/03/09/16152932156268.jpg?x-oss-process=style/markdown-media) +完整版可能会安装出错，若安装出错，请参考[安装问题](question/安装问题) -#### 2. 准备数据表 +## 小试一下 -![-w808](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/03/09/16152934374807.jpg?x-oss-process=style/markdown-media) +创建爬虫 -```sql -CREATE TABLE `jd_item` ( - `id` int(11) NOT NULL AUTO_INCREMENT, - `title` varchar(255) DEFAULT NULL, - `batch_date` date DEFAULT NULL COMMENT '批次时间', - `crawl_time` datetime DEFAULT NULL COMMENT '采集时间', - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; +```shell +feapder create -s first_spider ``` -需求是每7天全量更新一次，即数据要以7天为维度划分，因此设置个`batch_date`字段，表示每条数据所属的批次。 - -这里只是演示，因此只采集标题字段 +创建后的爬虫代码如下： -#### 3. 采集 +```python -若使用`scrapy`，需要手动将这些种子任务分批取出来发给爬虫，还需要维护种子任务的状态，以及上面提及的批次信息`batch_date`。并且为了保证数据的时效性，需要对采集进度进行监控，写个爬虫十分繁琐。 +import feapder -而`feapder`内置了批次爬虫，可以很方便的应对这个需求。完整的爬虫写法如下： -```python -import feapder -from feapder import Item -from feapder.utils import tools - - -class JdSpider(feapder.BatchSpider): - # 自定义数据库，若项目中有setting.py文件，此自定义可删除 - __custom_setting__ = dict( - REDISDB_IP_PORTS="localhost:6379", - REDISDB_DB=0, - MYSQL_IP="localhost", - MYSQL_PORT=3306, - MYSQL_DB="feapder", - MYSQL_USER_NAME="feapder", - MYSQL_USER_PASS="feapder123", - ) - - def start_requests(self, task): - task_id, item_id = task - url = "https://item.jd.com/{}.html".format(item_id) - yield feapder.Request(url, task_id=task_id) # 携带task_id字段 +class FirstSpider(feapder.AirSpider): + def start_requests(self): + yield feapder.Request("https://www.baidu.com") def parse(self, request, response): - title = response.xpath("string(//div[@class='sku-name'])").extract_first(default="").strip() - - item = Item() - item.table_name = "jd_item" # 指定入库的表名 - item.title = title - item.batch_date = self.batch_date # 获取批次信息，批次信息框架自己维护 - item.crawl_time = tools.get_current_date() # 获取当前时间 - yield item # 自动批量入库 - yield self.update_task_batch(request.task_id, 1) # 更新任务状态 + print(response) if __name__ == "__main__": - spider = JdSpider( - redis_key="feapder:jd_item", # redis中存放任务等信息key前缀 - task_table="jd_item_task", # mysql中的任务表 - task_keys=["id", "item_id"], # 需要获取任务表里的字段名，可添加多个 - task_state="state", # mysql中任务状态字段 - batch_record_table="jd_item_batch_record", # mysql中的批次记录表，自动生成 - batch_name="京东商品爬虫(周度全量)", # 批次名字 - batch_interval=7, # 批次周期 天为单位 若为小时 可写 1 / 24 - ) - - # 下面两个启动函数 相当于 master、worker。需要分开运行 - spider.start_monitor_task() # maser: 下发及监控任务 - # spider.start() # worker: 采集 + FirstSpider().start() ``` -我们分别运行`spider.start_monitor_task()`与`spider.start()`，待爬虫结束后，观察数据库 - -**任务表**：`jd_item_task` - -![-w282](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/03/09/16152953028811.jpg?x-oss-process=style/markdown-media) - -任务均已完成了，框架有任务丢失重发机制，直到所有任务均已做完 - -**数据表**：`jd_item`: - -![-w569](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/03/09/16152952623851.jpg?x-oss-process=style/markdown-media) - -数据里携带了批次时间信息，我们可以根据这个时间来对数据进行划分。当前批次为3月9号，若7天一批次，则下一批次为3月18号。 - -**批次表**：`jd_item_batch_record` - -![-w901](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/03/09/16152953428596.jpg?x-oss-process=style/markdown-media) - -启动参数中指定，自动生成。批次表里详细记录了每个批次的抓取状态，如任务总量、已做量、失败量、是否已完成等信息 +直接运行，打印如下： -#### 4. 监控 +```shell +Thread-2|2021-02-09 14:55:11,373|request.py|get_response|line:283|DEBUG| + -------------- FirstSpider.parse request for ---------------- + url = https://www.baidu.com + method = GET + body = {'timeout': 22, 'stream': True, 'verify': False, 'headers': {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36'}} -feapder会自动维护任务状态，每个批次（采集周期）的进度，并且内置丰富的报警，保证我们的数据时效性，如： + +Thread-2|2021-02-09 14:55:11,610|parser_control.py|run|line:415|DEBUG| parser 等待任务... +FirstSpider|2021-02-09 14:55:14,620|air_spider.py|run|line:80|INFO| 无任务，爬虫结束 +``` -1. 实时计算爬虫抓取速度，估算剩余时间，在指定的抓取周期内预判是否会超时 +代码解释如下： - ![-w657](http://markdown-media.oss-cn-beijing.aliyuncs.com/2020/12/20/16084718683378.jpg?x-oss-process=style/markdown-media) +1. start_requests： 生产任务 +2. parse： 解析数据 +## 爬虫工具推荐 -2. 爬虫卡死报警 +1. 爬虫在线工具库：http://www.spidertools.cn +2. 爬虫管理系统：http://feapder.com/#/feapder_platform/feaplat +3. 验证码识别库：https://github.com/sml2h3/ddddocr - ![-w501](http://markdown-media.oss-cn-beijing.aliyuncs.com/2020/12/20/16084718974597.jpg?x-oss-process=style/markdown-media) -3. 爬虫任务失败数过多报警，可能是由于网站模板改动或封堵导致 + ## 学习交流 -知识星球： - -![知识星球](http://markdown-media.oss-cn-beijing.aliyuncs.com/2020/02/16/zhi-shi-xing-qiu.jpeg) - -星球会不定时分享爬虫技术干货，涉及的领域包括但不限于js逆向技巧、爬虫框架刨析、爬虫技术分享等 \ No newline at end of file + + + + + + + + + + + +
知识星球：17321694 作者微信： boris_tm QQ群号：521494615
+
+ + + 加好友备注：feapder \ No newline at end of file diff --git a/docs/_coverpage.md b/docs/_coverpage.md index aca7076a..4471ac93 100644 --- a/docs/_coverpage.md +++ b/docs/_coverpage.md @@ -1,4 +1,4 @@ -![feapder](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/02/08/feapder.png?x-oss-process=style/markdown-media) +![feapder](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/02/08/feapder.png) # feapder 爬虫框架文档 @@ -8,7 +8,7 @@ feapder 命名源于 fast-easy-air-pro-spider 缩写 秉承着开发快速、抓取快速、简单、轻量且功能强大的原则，倾心打造。 -支持轻量级爬虫、分布式爬虫、批次爬虫、多模板爬虫，以及完善的报警等。 +支持轻量级爬虫、分布式爬虫、批次爬虫、爬虫集成，以及完善的报警等。 [GitHub](https://github.com/Boris-code/feapder) diff --git a/docs/_navbar.md b/docs/_navbar.md index f30c3654..f5a7075a 100644 --- a/docs/_navbar.md +++ b/docs/_navbar.md @@ -1,5 +1,7 @@ + * [爬虫管理系统](feapder_platform/feaplat.md) + * [爬虫工具库](https://spidertools.cn) * [知识星球](https://t.zsxq.com/mmAmAuF) * [微信公众号](https://open.weixin.qq.com/qr/code?username=gh_870ffb1242a7) - * [个人博客](https://boris.org.cn/) + * [知乎](https://www.zhihu.com/people/boris-97-17/posts) * [讨论](https://gitter.im/feapder/community?utm_source=share-link&utm_medium=link&utm_campaign=share-link) \ No newline at end of file diff --git a/docs/_sidebar.md b/docs/_sidebar.md index 8f81adda..bef51b37 100644 --- a/docs/_sidebar.md +++ b/docs/_sidebar.md @@ -11,24 +11,42 @@ * [使用前必读](usage/使用前必读.md) * [轻量爬虫-AirSpider](usage/AirSpider.md) * [分布式爬虫-Spider](usage/Spider.md) + * [任务爬虫-TaskSpider](usage/TaskSpider.md) * [批次爬虫-BatchSpider](usage/BatchSpider.md) * [爬虫集成](usage/爬虫集成.md) * 使用进阶 * [请求-Request](source_code/Request.md) * [响应-Response](source_code/Response.md) + * [代理使用说明](source_code/proxy.md) + * [用户池说明](source_code/UserPool.md) + * [浏览器渲染-Selenium](source_code/浏览器渲染-Selenium.md) + * [浏览器渲染-Playwright](source_code/浏览器渲染-Playwright) * [解析器-BaseParser](source_code/BaseParser.md) * [批次解析器-BatchParser](source_code/BatchParser.md) * [Spider进阶](source_code/Spider进阶.md) * [BatchSpider进阶](source_code/BatchSpider进阶.md) + * [配置文件](source_code/配置文件.md) * [Item](source_code/Item.md) * [UpdateItem](source_code/UpdateItem.md) - * [ItemBuffer](source_code/ItemBuffer.md) - * [配置文件](source_code/配置文件.md) + * [数据管道-pipeline](source_code/pipeline.md) * [MysqlDB](source_code/MysqlDB.md) + * [MongoDB](source_code/MongoDB.md) * [RedisDB](source_code/RedisDB.md) * [工具库-tools](source_code/tools.md) + * [日志配置及使用](source_code/logger.md) * [海量数据去重-dedup](source_code/dedup.md) + * [报警及监控](source_code/报警及监控.md) + * [监控打点](source_code/监控打点.md) + * [自定义下载器](source_code/custom_downloader.md) + +* 爬虫管理系统 + * [简介及部署](feapder_platform/feaplat.md) + * [使用说明](feapder_platform/usage.md) + * [常见问题](feapder_platform/question.md) * 常见问题 - * [安装问题](question/安装问题.md) \ No newline at end of file + * [安装问题](question/安装问题.md) + * [运行问题](question/运行问题.md) + * [请求问题](question/请求问题.md) + * [setting不生效问题](question/setting不生效问题.md) \ No newline at end of file diff --git a/docs/command/cmdline.md b/docs/command/cmdline.md index 552a57f2..74691832 100644 --- a/docs/command/cmdline.md +++ b/docs/command/cmdline.md @@ -24,42 +24,39 @@ Available commands: create create project、feapder、item and so on shell debug response + zip zip project Use "feapder -h" to see more info about a command -可见feapder支持`create`及`shell`两种命令 +可见feapder支持`create`、`shell`及`zip`三种命令 ## 2. feapder create 使用feapder create 可快速创建项目、爬虫、item等，具体支持的命令可输入`feapder create -h` 查看使用帮助 > feapder create -h - usage: feapder [-h] [-p] [-s [...]] [-i [...]] [-t] [-init] [-j] [-sj] - [--host] [--port] [--username] [--password] [--db] + usage: cmdline.py [-h] [-p] [-s] [-i] [-t] [-init] [-j] [-sj] [-c] [--params] [--setting] [--host] [--port] [--username] [--password] [--db] 生成器 - + optional arguments: - -h, --help show this help message and exit - -p , --project 创建项目 如 feapder create -s - -s [ ...], --spider [ ...] - 创建爬虫 如 feapder create -s - spider_type=1 AirSpider; spider_type=2 Spider; - spider_type=3 BatchSpider; - -i [ ...], --item [ ...] - 创建item 如 feapder create -i test 则生成test表对应的item。 - 支持like语法模糊匹配所要生产的表。 若想生成支持字典方式赋值的item，则create -item - test 1 - -t , --table 根据json创建表 如 feapder create -t - -init 创建__init__.py 如 feapder create -init - -j, --json 创建json - -sj, --sort_json 创建有序json - --host mysql 连接地址 - --port mysql 端口 - --username mysql 用户名 - --password mysql 密码 - --db mysql 数据库名 + -h, --help show this help message and exit + -p , --project 创建项目 如 feapder create -p + -s , --spider 创建爬虫 如 feapder create -s + -i , --item 创建item 如 feapder create -i 支持模糊匹配 如 feapder create -i %table_name% + -t , --table 根据json创建表 如 feapder create -t + -init 创建__init__.py 如 feapder create -init + -j, --json 创建json + -sj, --sort_json 创建有序json + -c, --cookies 创建cookie + --params 解析地址中的参数 + --setting 创建全局配置文件feapder create --setting + --host mysql 连接地址 + --port mysql 端口 + --username mysql 用户名 + --password mysql 密码 + --db mysql 数据库名 具体使用方法如下： @@ -75,7 +72,7 @@ 生成如下： -![-w354](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/02/08/16127822246620.jpg?x-oss-process=style/markdown-media) +![-w354](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/02/08/16127822246620.jpg) * items： 文件夹存放与数据库表映射的item * spiders： 文件夹存放爬虫脚本 @@ -86,23 +83,23 @@ ### 2. 创建爬虫 -爬虫分为3中，分别为 轻量级爬虫（AirSpider）、分布式爬虫（Spider）以及 批次爬虫 （BatchSpider） - 命令 - feapder create -s - -* AirSpider 对应的 spider_type 值为 1 -* Spider 对应的 spider_type 值为 2 -* BatchSpider 对应的 spider_type 值为 3 -* 默认 spider_type 值为 1 - -AirSpider爬虫示例： - - feapder create -s first_spider 1 + feapder create -s + +示例：创建名为first_spider的爬虫 +```shell +feapder create -s first_spider -生成first_spider.py, 内容如下： +请选择爬虫模板 +> AirSpider + Spider + TaskSpider + BatchSpider +``` + +输入命令后，可以按上下键选择爬虫模板，如选择 AirSpider爬虫模板，生成first_spider.py, 内容如下： import feapder @@ -119,7 +116,7 @@ AirSpider爬虫示例： FirstSpider().start() -若为项目结构，建议先进入到spiders目录下，再创建爬虫 +若在项目下创建，建议先进入到spiders目录下，再创建爬虫 ### 3. 创建 item @@ -129,6 +126,16 @@ item为与数据库表的映射，与数据入库的逻辑相关。 命令 feapder create -i + +输出： + +``` +请选择Item类型 +> Item + Item 支持字典赋值 + UpdateItem + UpdateItem 支持字典赋值 +``` 示例 @@ -143,7 +150,7 @@ item为与数据库表的映射，与数据入库的逻辑相关。 2. 配置setting.py， 连接方式换成自己数据库的 - ![-w799](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/02/08/16127839359771.jpg?x-oss-process=style/markdown-media) + ![-w799](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/02/08/16127839359771.jpg) 3. 进入items目录，执行命令 @@ -154,20 +161,22 @@ item为与数据库表的映射，与数据入库的逻辑相关。 ``` from feapder import Item - - + + class SpiderDataItem(Item): """ This class was generated by feapder. command: feapder create -i spider_data. """ - + def __init__(self, *args, **kwargs): - # self.id = None # type : int(10) unsigned | allow_null : NO | key : PRI | default_value : None | extra : auto_increment | column_comment : - self.title = None # type : varchar(255) | allow_null : YES | key : | default_value : None | extra : | column_comment : - + # self.id = None + self.title = None + ``` +若字段有默认值或者自增，则默认注释掉，可按需打开 + 若不配置setting.py, 可在命令行中指定数据库连接信息 feapder create -i spider_data --host localhost --db feapder --username feapder --password feapder123 @@ -186,7 +195,36 @@ class SpiderDataItem(Item): 这样，以后所有的项目setting.py中均可不配置mysql连接信息 -### 4. 创建json 或 有序json +**若item字段过多，不想逐一赋值，可选择支持字典赋值的Item类型创建** + +![](http://markdown-media.oss-cn-beijing.aliyuncs.com/2022/09/09/16626945562298.jpg) + +生成： + +``` +from feapder import Item + + +class SpiderDataItem(Item): + """ + This class was generated by feapder. + command: feapder create -i spider_data 1. + """ + + def __init__(self, *args, **kwargs): + # self.id = kwargs.get('id') + self.title = kwargs.get('title') +``` + +这样当我们请求回来的json数据时，可直接赋值，如 + +``` +response_data = {"title":" 测试"} # 模拟请求回来的数据 +item = SpiderDataItem(**response_data) +``` + + +### 4. 创建json或有序json 此命令和快速将 `xxx:xxx` 这种字符串格式转为json格式，常用于将网页或者抓包工具抓取出来的header、cookie转为json @@ -200,11 +238,11 @@ class SpiderDataItem(Item): 1. copy 请求头，粘贴到提示下方 - ![](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/02/08/16127849396722.jpg?x-oss-process=style/markdown-media) + ![](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/02/08/16127849396722.jpg) 1. 输出如下： - ![-w1394](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/02/08/16127850065269.jpg?x-oss-process=style/markdown-media) + ![-w1394](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/02/08/16127850065269.jpg) **sort_json** 与json命令类似，只不过该命令生成的json是按照key排序的有序字典， 命令为 @@ -261,7 +299,7 @@ class SpiderDataItem(Item): feapder create -init 观察生成的\__init__.py文件，已自动包含当前目录下的py文件 -![-w880](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/02/08/16127859798201.jpg?x-oss-process=style/markdown-media) +![-w880](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/02/08/16127859798201.jpg) ## 3. feapder shell @@ -308,13 +346,13 @@ class SpiderDataItem(Item): 可以看出，我们可以直接使用response， response支持xpath表达式 - 若想查看都支持哪些函数，可输入`resonse.` 然后敲两次`tab键`，如下： - ![-w1738](http://markdown-media.oss-cn-beijing.aliyuncs.com/2020/06/21/15927532396490.jpg?x-oss-process=style/markdown-media) + 若想查看都支持哪些函数，可输入`response.` 然后敲两次`tab键`，如下： + ![-w1738](http://markdown-media.oss-cn-beijing.aliyuncs.com/2020/06/21/15927532396490.jpg) 1. 以curl为例，请求百度（通常用来测试post接口比较方便） 1. 打开浏览器检查工具，复制需要测试的接口为curl格式 - ![-w569](http://markdown-media.oss-cn-beijing.aliyuncs.com/2020/06/21/15927533333272.jpg?x-oss-process=style/markdown-media) + ![-w569](http://markdown-media.oss-cn-beijing.aliyuncs.com/2020/06/21/15927533333272.jpg) 1. 测试 输入`feapder shell --`, 粘贴刚刚复制的curl diff --git a/docs/favicon.ico b/docs/favicon.ico index 3a8ac43a..c9d0259e 100644 Binary files a/docs/favicon.ico and b/docs/favicon.ico differ diff --git a/docs/feapder_platform/feaplat.md b/docs/feapder_platform/feaplat.md new file mode 100644 index 00000000..405f3e0c --- /dev/null +++ b/docs/feapder_platform/feaplat.md @@ -0,0 +1,428 @@ +# 爬虫管理系统 - FEAPLAT + +> 生而为虫，不止于虫 + +**feaplat**命名源于 feapder 与 platform 的缩写 + +读音： `[ˈfiːplæt] ` + +![](http://markdown-media.oss-cn-beijing.aliyuncs.com/2022/10/12/16655602840534.jpg) + + +## 特性 + +1. 支持部署任何程序，包括不限于`feapder`、`scrapy` +2. 支持集群管理，部署分布式爬虫可一键扩展进程数 +3. 支持部署服务，且可自动实现服务负载均衡 +4. 支持程序异常报警、重启、保活 +5. 支持监控，监控内容可自定义 +6. 支持4种定时调度模式 +7. 自动从git仓库拉取最新的代码运行，支持指定分支 +8. 支持多人协同 +9. 支持浏览器渲染，支持有头模式。浏览器支持`playwright`、`selenium` +10. 支持弹性伸缩 +12. 支持自定义worker镜像，如自定义java的运行环境、node运行环境等，即根据自己的需求自定义（feaplat分为`master-调度端`和`worker-运行任务端`） +13. docker一键部署，架设在docker swarm集群上 + +## 功能概览 + +暂时不支持 苹果电脑的Apple芯片 + +### 1. 项目管理 + +添加/编辑项目 + +![](http://markdown-media.oss-cn-beijing.aliyuncs.com/2022/10/12/16655603474851.jpg) + +- 支持 git和zip两种方式上传项目 +- 根据requirements.txt自动安装依赖包 +- 可选择多个人参与项目 + +### 2. 任务管理 + +![](http://markdown-media.oss-cn-beijing.aliyuncs.com/2022/10/12/16655604191030.jpg) +![](http://markdown-media.oss-cn-beijing.aliyuncs.com/2022/10/12/16655604736752.jpg) + +- 支持一键启动多个任务实例（分布式爬虫场景或者需要启动多个进程的场景） +- 支持4种调度模式 +- 标签：给任务分类使用 +- 强制运行：（上一次任务没结束，本次是否运行，是则会停止上一次任务，然后运行本次调度） +- 异常重启：当部署的程序异常退出，是否自动重启，且会报警 + ![](http://markdown-media.oss-cn-beijing.aliyuncs.com/2022/10/12/16655607031254.jpg) +- 支持限制程序运行的CPU、内存等。 + + +### 3. 任务实例 + +一键部署了20份程序，每个程序独占一个进程，可从列表看每个进程部署到哪台服务器上了，运行状态是什么 + +![](http://markdown-media.oss-cn-beijing.aliyuncs.com/2022/10/12/16655608218525.jpg) + +实时查看日志 + +![](http://markdown-media.oss-cn-beijing.aliyuncs.com/2022/10/12/16655618630971.jpg) + +### 4. 爬虫监控 + +feaplat支持对feapder爬虫的运行情况进行监控，除了数据监控和请求监控外，用户还可自定义监控内容，详情参考[自定义监控](source_code/监控打点?id=自定义监控) + +若scrapy爬虫或其他python脚本使用监控功能，也可通过自定义监控的功能来支持，详情参考[自定义监控](source_code/监控打点?id=自定义监控) + +注：需 feapder>=1.6.6 + +![](http://markdown-media.oss-cn-beijing.aliyuncs.com/2022/10/12/16655595870715.jpg) + +### 5. 报警 + +调度异常、程序异常自动报警 +支持钉钉、企业微信、飞书、邮箱 + +![](http://markdown-media.oss-cn-beijing.aliyuncs.com/2022/10/12/16655607031254.jpg) + +## 为什么用feaplat爬虫管理系统 + +**稳！很稳！！相当稳！！！** + +### 市面上的爬虫管理系统 + +![feapderd](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/07/23/feapderd.png) + +worker节点常驻，且运行多个任务，不能弹性伸缩，任务之前会相互影响，稳定性得不到保障 + +### feaplat爬虫管理系统 + +![pic](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/07/23/pic.gif) + +worker节点根据任务动态生成，一个worker只运行一个任务实例，任务做完worker销毁，稳定性高；多个服务器间自动均衡分配，弹性伸缩 + +## 部署 + +> 安装方式参考docker官方文档：https://docs.docker.com/compose/install/ + +### 1. 安装docker + +#### 1.1 centos系统 + +> docker --version +> 作者的docker版本为 20.10.12，低于此版本的可能会存在问题 + +删除旧版本（可选，需要重装升级docker时执行） + +```shell +yum remove docker docker-common docker-selinux docker-engine +``` + +安装： +```shell +yum install -y yum-utils device-mapper-persistent-data lvm2 && python2 /usr/bin/yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo && yum install docker-ce -y +``` +国内用户推荐使用 +```shell +yum install -y yum-utils device-mapper-persistent-data lvm2 && python2 /usr/bin/yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo && yum install docker-ce -y +``` +或者使用国内 daocloud 一键安装命令 +``` +curl -sSL https://get.daocloud.io/docker | sh +``` + +启动docker服务 + +```shell +systemctl enable docker +systemctl start docker +``` + +验证: 打开终端，输入 + +```shell +docker ps +``` + +#### 1.2 ubuntu系统 + +``` +sudo apt update +sudo apt install docker.io docker-compose +``` + +启动docker服务 + +```shell +sudo systemctl enable docker +sudo systemctl start docker +``` + +验证: 打开终端，输入 + +```shell +sudo docker ps +``` + +#### 1.3 window系统 + +访问下面的链接，下载Docker Desktop, 然后安装即可 + +https://docs.docker.com/desktop/setup/install/windows-install/ + + +运行安装好的Docker Desktop + +验证: 打开cmd终端，输入 + +```shell +docker ps +``` + +#### 1.4 mac系统 + +访问下面的链接，下载Docker Desktop, 然后安装即可 + +https://docs.docker.com/desktop/setup/install/mac-install/ + + +运行安装好的Docker Desktop + +验证: 打开终端，输入 +```shell +docker ps +``` + + +### 2. 安装 docker swarm + + docker swarm init + + # 如果你的 Docker 主机有多个网卡，拥有多个 IP，必须使用 --advertise-addr 指定 IP + docker swarm init --advertise-addr 192.168.99.100 + +### 3. 安装docker-compose(非必须) +一般安装完docker后，会自带 docker compose。可先输入下面的命令验证是否有改环境，若有则不需要安装 +``` shell +docker compose +``` +若无`docker compose`命令，则按照下面的安装 + +```shell +sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose +sudo chmod +x /usr/local/bin/docker-compose +``` +国内用户推荐使用 +```shell +sudo curl -L "https://get.daocloud.io/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose +sudo chmod +x /usr/local/bin/docker-compose +``` +安装后输入`docker-compose`验证是否成功 + +注：`docker-compose` 与 `docker compose` 两种命令用法一样，是一个东西，只不过不同版本的docker可能叫法不一 + +### 4. 部署feaplat爬虫管理系统 +#### 预备项 +安装git(1.8.3的版本已够用) +```shell +yum -y install git +``` +#### 1. 下载项目 + +> 先按照下面命令拉取develop分支代码运行。 +> master分支不支持urllib3>=2.0版本，现在已经运行不起来了，但之前老用户不受影响。待后续测试好兼容性，不影响老用户后，会将develop分支合并到master + +gitub +```shell +git clone -b develop https://github.com/Boris-code/feaplat.git +``` +gitee +```shell +git clone -b develop https://gitee.com/Boris-code/feaplat.git +``` + +#### 2. 运行 + +首次运行需拉取镜像，时间比较久，且运行可能会报错，再次运行下就好了 + +```shell +cd feaplat +docker compose up -d +或者 +docker-compose up -d +``` + +- 若端口冲突，可修改.env文件，参考[常见问题](feapder_platform/question?id=修改端口) + +#### 3. 访问爬虫管理系统 + +默认地址：`http://localhost` +默认账密：admin / admin + +- 若未成功，参考[常见问题](feapder_platform/question) +- 使用说明，参考[使用说明](feapder_platform/usage) + +#### 4. 停止（可选） + +```shell +docker-compose stop +``` + +### 5. 添加服务器（可选） + +> 用于搭建集群，扩展爬虫（worker）节点服务器 + +#### 1. 安装docker + +参考部署步骤1 + +#### 2. 部署 + +在master服务器（feaplat爬虫管理系统所在服务器）执行下面命令，查看token + +```shell +docker swarm join-token worker +``` + +输出举例如下 + +```shell +docker swarm join --token SWMTKN-1-1mix1x7noormwig1pjqzmrvgnw2m8zxqdzctqa8t3o8s25fjgg-9ot0h1gatxfh0qrxiee38xxxx 172.17.5.110:2377 +``` + +**在需扩充的服务器上执行** + +```shell +docker swarm join --token [token] [ip] +``` + +若服务器彼此之间不是内网，为公网环境，则需要将ip改成公网，且开放端口2377 + +开启并检查2377端口 +```shell +firewall-cmd --zone=public --add-port=2377/tcp --permanent +firewall-cmd --reload +firewall-cmd --query-port=2377/tcp +``` + +#### 3. 验证是否成功 + +在master服务器（feaplat爬虫管理系统所在服务器）执行下面命令 + +```shell +docker node ls +``` + +若打印结果包含刚加入的服务器，则添加服务器成功 + +#### 4. 下线服务器（可选） + +在需要下线的服务器上执行 + +```shell +docker swarm leave +``` + +## 使用 + +见 [FEAPLAT使用说明](feapder_platform/usage) + +## 自定义爬虫镜像 + +默认的爬虫镜像只打包了`feapder`、`scrapy`框架，若需要其它环境，可基于`.env`文件里的`SPIDER_IMAGE`镜像自行构建 + +如自定义python版本，安装常用的库等，需修改feaplat下的`feapder_dockerfile` + +``` +# 基于最新的版本，若需要自定义python版本，则要求feapder版本号>=2.4 +FROM registry.cn-hangzhou.aliyuncs.com/feapderd/feapder:2.4 + +# 安装自定义的python版本，3.10.8 +RUN set -ex \ + && wget https://www.python.org/ftp/python/3.10.8/Python-3.10.8.tgz \ + && tar -zxvf Python-3.10.8.tgz \ + && cd Python-3.10.8 \ + && ./configure prefix=/usr/local/python-3.10.8 \ + && make \ + && make install \ + && make clean \ + && rm -rf /Python-3.10.8* \ + # 配置软链接 + && ln -s /usr/local/python-3.10.8/bin/python3 /usr/bin/python3.10.8 \ + && ln -s /usr/local/python-3.10.8/bin/pip3 /usr/bin/pip3.10.8 + +# 删除之前的默认python版本 +RUN set -ex \ + && rm -rf /usr/bin/python3 \ + && rm -rf /usr/bin/pip3 \ + && rm -rf /usr/bin/python \ + && rm -rf /usr/bin/pip + +# 设置默认为python3.10.8 +RUN set -ex \ + && ln -s /usr/local/python-3.10.8/bin/python3 /usr/bin/python \ + && ln -s /usr/local/python-3.10.8/bin/python3 /usr/bin/python3 \ + && ln -s /usr/local/python-3.10.8/bin/pip3 /usr/bin/pip \ + && ln -s /usr/local/python-3.10.8/bin/pip3 /usr/bin/pip3 + +# 将python3.10.8加入到环境变量 +ENV PATH=$PATH:/usr/local/python-3.10.8/bin/ + +# 安装依赖 +RUN pip3 install feapder \ + && pip3 install scrapy + +# 安装node依赖包，内置的node为v10.15.3版本 +# RUN npm install packageName -g + +``` + +改好后要打包镜像，打包命令： +``` +docker build -f feapder_dockerfile -t 镜像名:版本号 . +``` +如 +``` +docker build -f feapder_dockerfile -t my_feapder:1.0 . +``` + +打包好后修改下 `.env`文件里的 SPIDER_IMAGE 的值即可如： +``` +SPIDER_IMAGE=my_feapder:1.0 +``` + +注： +1. 若有多个worker服务器，且没将镜像传到镜像服务，则需要手动将镜像推到其他服务器上，否则无法拉取此镜像运行 +2. 若自定义了python版本，则需要添加挂载，否则feaplat上自动安装的依赖库不会保留。挂载方式：修改`docker-compose.yaml`的 SPIDER_RUN_ARGS参数。如 + ``` + SPIDER_RUN_ARGS=["--mount type=volume,source=feapder_python3.10,destination=/usr/local/python-3.10.8"] + ``` + +## 价格 + +可免费部署20个任务，超出额度时，需购买授权码，在授权有效期内不限额度，可换绑服务器 + +| 授权时长 | 价格 | 说明 | +|------|------|---------------------| +| 1个月 | 168元 | 无折扣| +| 6个月| 666元 | 原价1008元，减免342元| +| 1年 | 888元 | 原价2016元，减免1128元| +| 2年 | 1500元 | 原价4032元，减免2532元| + +**删除任务不可恢复额度** + +购买方式：添加微信 `boris_tm` + +随着功能的完善，价格会逐步调整 + +## 学习交流 + + + + + + + + + + + + +
知识星球：17321694 作者微信： boris_tm QQ群号：521494615
+
+ + 加好友备注：feapder diff --git a/docs/feapder_platform/feaplat_bak.md b/docs/feapder_platform/feaplat_bak.md new file mode 100644 index 00000000..87333075 --- /dev/null +++ b/docs/feapder_platform/feaplat_bak.md @@ -0,0 +1,288 @@ +# 爬虫管理系统 - FEAPLAT + +> 生而为虫，不止于虫 + +**feaplat**命名源于 feapder 与 platform 的缩写 + +读音： `[ˈfiːplæt] ` + +![](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/09/14/16316112326191.jpg) + +## 特性 + +1. 支持任何python脚本，包括不限于`feapder`、`scrapy` +2. 支持浏览器渲染，支持有头模式。浏览器支持`playwright`、`selenium` +3. 支持部署服务，可自动负载均衡 +4. 支持服务器集群管理 +5. 支持监控，监控内容可自定义 +6. 支持起多个实例，如分布式爬虫场景 +7. 支持弹性伸缩 +8. 支持4种定时启动方式 +9. 支持自定义worker镜像，如自定义java的运行环境、机器学习环境等，即根据自己的需求自定义（feaplat分为`master-调度端`和`worker-运行任务端`） +10. docker一键部署，架设在docker swarm集群上 + + +## 为什么用feaplat爬虫管理系统 + +**市面上的爬虫管理系统** + +![feapderd](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/07/23/feapderd.png) + +worker节点常驻，且运行多个任务，不能弹性伸缩，任务之前会相互影响，稳定性得不到保障 + +**feaplat爬虫管理系统** + +![pic](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/07/23/pic.gif) + +worker节点根据任务动态生成，一个worker只运行一个任务实例，任务做完worker销毁，稳定性高；多个服务器间自动均衡分配，弹性伸缩 + + +## 功能概览 + +### 1. 项目管理 + +添加/编辑项目 +![-w1785](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/07/06/16254968151490.jpg) + +### 2. 任务管理 + +![](http://markdown-media.oss-cn-beijing.aliyuncs.com/2022/03/03/16463109796998.jpg) + + +### 3. 任务实例 + +日志 +![](http://markdown-media.oss-cn-beijing.aliyuncs.com/2022/03/03/16463117042527.jpg) + + +### 4. 爬虫监控 + +feaplat支持对feapder爬虫的运行情况进行监控，除了数据监控和请求监控外，用户还可自定义监控内容，详情参考[自定义监控](source_code/监控打点?id=自定义监控) + +若scrapy爬虫或其他python脚本使用监控功能，也可通过自定义监控的功能来支持，详情参考[自定义监控](source_code/监控打点?id=自定义监控) + +注：需 feapder>=1.6.6 + +![](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/09/14/16316112326191.jpg) + + + +## 部署 + +> 下面部署以centos为例， 其他平台docker安装方式可参考docker官方文档：https://docs.docker.com/compose/install/ + +### 1. 安装docker + +删除旧版本（可选，需要重装升级时执行） + +```shell +yum remove docker docker-common docker-selinux docker-engine +``` + +安装： +```shell +yum install -y yum-utils device-mapper-persistent-data lvm2 && python2 /usr/bin/yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo && yum install docker-ce -y +``` +国内用户推荐使用 +```shell +yum install -y yum-utils device-mapper-persistent-data lvm2 && python2 /usr/bin/yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo && yum install docker-ce -y +``` +或者使用国内 daocloud 一键安装命令 +``` +curl -sSL https://get.daocloud.io/docker | sh +``` + + + +启动 +```shell +systemctl enable docker +systemctl start docker +``` + +### 2. 安装 docker swarm + + docker swarm init + + # 如果你的 Docker 主机有多个网卡，拥有多个 IP，必须使用 --advertise-addr 指定 IP + docker swarm init --advertise-addr 192.168.99.100 + +### 3. 安装docker-compose + +```shell +sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose +sudo chmod +x /usr/local/bin/docker-compose +``` +国内用户推荐使用 +```shell +sudo curl -L "https://get.daocloud.io/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose +sudo chmod +x /usr/local/bin/docker-compose +``` + +### 4. 部署feaplat爬虫管理系统 +#### 预备项 +安装git(1.8.3的版本已够用) +```shell +yum -y install git +``` +#### 1. 下载项目 + +gitub +```shell +git clone https://github.com/Boris-code/feaplat.git +``` +gitee +```shell +git clone https://gitee.com/Boris-code/feaplat.git +``` + +#### 2. 运行 + +首次运行需拉取镜像，时间比较久，且运行可能会报错，再次运行下就好了 + +```shell +cd feaplat +docker-compose up -d +``` + +- 若端口冲突，可修改.env文件，参考[常见问题](feapder_platform/question?id=修改端口) + +#### 3. 访问爬虫管理系统 + +默认地址：`http://localhost` +默认账密：admin / admin + +- 若未成功，参考[常见问题](feapder_platform/question) +- 使用说明，参考[使用说明](feapder_platform/usage) + +#### 4. 停止（可选） + +```shell +docker-compose stop +``` + +### 5. 添加服务器（可选） + +> 用于搭建集群，扩展爬虫（worker）节点服务器 + +#### 1. 安装docker + +参考部署步骤1 + +#### 2. 部署 + +在master服务器（feaplat爬虫管理系统所在服务器）执行下面命令，查看token + +```shell +docker swarm join-token worker +``` + +输出举例如下 + +```shell +docker swarm join --token SWMTKN-1-1mix1x7noormwig1pjqzmrvgnw2m8zxqdzctqa8t3o8s25fjgg-9ot0h1gatxfh0qrxiee38xxxx 172.17.5.110:2377 +``` + +**在需扩充的服务器上执行** + +```shell +docker swarm join --token [token] [ip] +``` + +若服务器彼此之间不是内网，为公网环境，则需要将ip改成公网，且开放端口2377 + +开启并检查2377端口 +```shell +firewall-cmd --zone=public --add-port=2377/tcp --permanent +firewall-cmd --reload +firewall-cmd --query-port=2377/tcp +``` + +#### 3. 验证是否成功 + +在master服务器（feaplat爬虫管理系统所在服务器）执行下面命令 + +```shell +docker node ls +``` + +若打印结果包含刚加入的服务器，则添加服务器成功 + +#### 4. 下线服务器（可选） + +在需要下线的服务器上执行 + +```shell +docker swarm leave +``` + +## 拉取私有项目 + +拉取私有项目需在git仓库里添加如下公钥 + +``` +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCd/k/tjbcMislEunjtYQNXxz5tgEDc/fSvuLHBNUX4PtfmMQ07TuUX2XJIIzLRPaqv3nsMn3+QZrV0xQd545FG1Cq83JJB98ATTW7k5Q0eaWXkvThdFeG5+n85KeVV2W4BpdHHNZ5h9RxBUmVZPpAZacdC6OUSBYTyCblPfX9DvjOk+KfwAZVwpJSkv4YduwoR3DNfXrmK5P+wrYW9z/VHUf0hcfWEnsrrHktCKgohZn9Fe8uS3B5wTNd9GgVrLGRk85ag+CChoqg80DjgFt/IhzMCArqwLyMn7rGG4Iu2Ie0TcdMc0TlRxoBhqrfKkN83cfQ3gDf41tZwp67uM9ZN feapder@qq.com +``` + +或在系统设置页面配置您的SSH私钥，然后在git仓库里添加您的公钥，例如： +![](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/10/19/16346353514967.jpg) + +注意，公私钥加密方式为RSA，其他的可能会有问题 + +生成RSA公私钥方式如下： +```shell +ssh-keygen -t rsa -C "备注" -f 生成路径/文件名 +``` +如： +`ssh-keygen -t rsa -C "feaplat" -f id_rsa` +然后一路回车，不要输密码 +![](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/11/17/16371210640228.jpg) +最终生成 `id_rsa`、`id_rsa.pub` 文件，复制`id_rsa.pub`文件内容到git仓库，复制`id_rsa`文件内容到feaplat爬虫管理系统 + +## 自定义爬虫镜像 + +默认的爬虫镜像只打包了`feapder`、`scrapy`框架，若需要其它环境，可基于`.env`文件里的`SPIDER_IMAGE`镜像自行构建 + +如将常用的python库打包到镜像 +``` +FROM registry.cn-hangzhou.aliyuncs.com/feapderd/feapder:[最新版本号] + +# 安装依赖 +RUN pip3 install feapder \ + && pip3 install scrapy + +``` + +自己随便搞事情，搞完修改下 `.env`文件里的 SPIDER_IMAGE 的值即可 + + +## 价格 + +| 类型 | 价格 | 说明 | +|------|------|---------------------| +| 试用版 | 0元 | 可部署20个任务，删除任务不可恢复额度 | +| 正式版 | 888元 | 有效期一年，可换绑服务器 | + +**部署后默认为试用版，购买授权码后配置到系统里即为正式版** + +购买方式：添加微信 `boris_tm` + +随着功能的完善，价格会逐步调整 + +## 学习交流 + + + + + + + + + + + + +
知识星球：17321694 作者微信： boris_tm QQ群号：750614606
+
+ + 加好友备注：feaplat diff --git a/docs/feapder_platform/question.md b/docs/feapder_platform/question.md new file mode 100644 index 00000000..78de0f2f --- /dev/null +++ b/docs/feapder_platform/question.md @@ -0,0 +1,155 @@ +# FEAPLAT常见问题 + +## 常用命令 + +1. 运行：`docker-compose up -d` +2. 停止：`docker-compose stop` +3. 查看后端日志：`docker logs -f feapder_backend` +4. 查看爬虫日志： + 1. 查看爬虫实例：`docker service ps task_任务id` + 2. 查看爬虫实例日志：`docker service logs -n 行数 -f ID` + + 举例： + + ``` + docker service ps task_9 + ``` + ![](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/09/17/16318829484192.jpg) + + ``` + docker service logs -n 20 -f u6qhyu2dauiu + ``` + +6. 查看正在运行的容器：`docker ps` +5. 进入容器：`docker exec -it 容器ID bash` + + +## 修改端口 + +默认端口如下： + +``` +# 前端端口 +FRONT_PORT=6385 +# 后端端口 +BACKEND_PORT=8000 +# MYSQL端口 +MYSQL_PORT=33306 +# REDIS 端口 +REDIS_PORT=6379 +# 监控系统端口配置 +INFLUXDB_PORT_TCP=8086 +INFLUXDB_PORT_UDP=8089 +``` + +通过 `vim .env` 敲击`i` 进入编辑模式，修改完按 `esc`退出编辑，敲击 `:wq` 保存 + + +## 启动失败 + +> 以下列的是几种可能原因，可按照这个顺序排查，但不是所有步骤都需要走一遍 + +1. 查看后端日志，观察报错 + 1. 若是docker版本问题，参考部署一节安装最新版本， + 2. 若是报 `This node is not a swarm manager`，则是部署环境没准备好，执行`docker swarm init`，可参考参考部署一节 +2. 查看worker状态： + ``` + docker service ps task_任务id --no-trunc + ``` + 看看error信息 + +4. 查看镜像`docker images`，若不存在爬虫镜像`registry.cn-hangzhou.aliyuncs.com/feapderd/feapder`，可能自动拉取失败了，可手动拉取，拉取命令：`docker pull registry.cn-hangzhou.aliyuncs.com/feapderd/feapder:版本号`，版本号在`.env`里查看 +5. 重启docker服务，Centos对应的命令为：`service docker restart`，其他自行查资料 + +## 依赖包安装失败，可手动安装包 + +1. 在项目配置处将 requirements.txt 一栏置空，使其不自动安装依赖 + + ![](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/09/17/16318840168908.jpg) + + +2. 添加一个常驻任务：执行命令可填写 `while true; do echo hello world; sleep 1; done` + + ![](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/09/17/16303761085876.jpg) + +1. 查看容器id`docker ps`（若您有多台worker服务器，该任务会被随机分配到一台机器上，您需要在对应的机器上查看） + + ![](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/09/17/16318842799082.jpg) +2. 进入容器 `docker exec -it 容器ID bash` + +3. 接来下就和在centos服务器上操作一样了，你可以`pip`安装依赖 + +## 授权问题 + +![](http://markdown-media.oss-cn-beijing.aliyuncs.com/2022/02/21/16454346779741.jpg) + +此问题为服务器时间和时区问题, 可以在服务器上输入 `date` ，命令检查时间及时区是否正确 + +正常应该为 `Mon Feb 21 17:03:11 CST 2022` 注意是 CST 不是 UTC + +修改时区及矫正时间命令 + +``` +# 改时区 +rm -f /etc/localtime +ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime + +# 校对时间 方式1 +clock --hctosys +# 校对时间 方式2 +ntpdate 0.asia.pool.ntp.org +``` + +## 我搭建了个集群，如何让主节点不跑任务 + +在主节点上执行下面命令，将其设置成drain状态即可 + + docker node update --availability drain 节点id + + ## Network 问题 + +attaching to network failed, make sure your network options are correct and check manager logs: context deadline exceeded + ![](http://markdown-media.oss-cn-beijing.aliyuncs.com/2023/02/16/16765140608308.jpg) + +1. 确定当前节点是不是Drain节点：docker node ls + + ![](http://markdown-media.oss-cn-beijing.aliyuncs.com/2023/02/16/16765145635622.jpg) + + 是则继续往下看，不是则在评论区留言 + +1. 修复 + + ``` + docker node update --availability active 节点id + docker node update --availability drain 节点id + ``` + +原因是Drain节点，不能为其分配网络资源，需要先改成active，然后启动，之后在改回drain + +**若不是以上情况，可能是network内的可分配的ip满了（老版本feaplat会有这个问题），那么可继续往下看** + +1. 先检查feaplat目录下的docker-compost.yaml，翻到最后，看network相关配置是否为如下。若不是，则改成下面这样的。若下面指定的11 ip段和主机有冲突，可以写12、13等 + + ``` + networks: + default: + name: feaplat + driver: overlay + attachable: true + ipam: + config: + - subnet: 11.0.0.0/8 + gateway: 11.0.0.1 + ``` + + 完整配置见：https://github.com/Boris-code/feaplat/blob/develop/docker-compose.yaml + + +2. 改完后，需要删除之前的network，使其重新创建，命令如下： + + ``` + docker service ls -q | xargs docker service rm # 注意 这个会停止掉所有任务。 + docker network rm feaplat # 删除网络 + docker compose rm # 删除之前feaplat运行环境 + docker compose up -d # 启动 + ``` \ No newline at end of file diff --git a/docs/feapder_platform/usage.md b/docs/feapder_platform/usage.md new file mode 100644 index 00000000..20e7bb12 --- /dev/null +++ b/docs/feapder_platform/usage.md @@ -0,0 +1,90 @@ +# FEAPLAT使用说明 + +## 首次运行须知 + +1. 管理系统默认账号密码：admin / admin + +## 添加项目 + +![](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/09/17/16318800747189.jpg) + +1. 使用git方式上传项目时，需要使用SSH协议，若拉取私有项目，可在feaplat的设置页面添加 SSH 密钥。使用git方式，每次运行前会拉取默认分支最新的代码 +2. 项目会被放到爬虫`worker`容器的根目录下 即 `/项目文件` +3. 工作路径：是指你的项目路径，比如下面的项目结构： + + ![](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/09/13/16315322995977.jpg) + + 工作路径为 `/spider-project`，feaplat会进入到这个目录，后续的代码执行命令都是在这个路径下运行的 + +1. requirements.txt：用于安装依赖包，填写依赖包的绝对路径 + +## 运行 + +1. 启动命令：启动命令是在您添加项目时配置的工作路径下执行的 +2. 定时类型： + 1. cron：crontab表达式，参考：https://tool.lu/crontab/ + 2. interval：时间间隔 + 3. date：指定日期 + 4. once：立即运行，且只运行一次 + +## 示例 + +1. 准备项目，项目结构如下： + ![](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/10/16/16343707944750.jpg) +2. 压缩后上传：（推荐使用 `feapder zip` 命令压缩） + ![](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/10/16/16343709590040.jpg) + - 工作路径：上传的项目会被放到docker里的根目录下（跟你本机项目路径没关系），然后解压运行。因`feapder_demo.zip`解压后为`feapder_demo`，所以工作路径配置`/feapder_demo` + - 本项目没依赖，可以不配置`requirements.txt` + - 若需要第三放库，则在项目下创建requirements.txt文件，把依赖库写进去，然后路径指向这个文件即可，如`/feaplat_demo/requirements.txt` +1. 点击项目进入任务列表，添加任务 + ![](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/10/16/16343712604864.jpg) + 启动命令的执行位置是在上面配置的工作路径下执行的，定时类型为once时点击确认添加会自动执行 +1. 查看任务实例： + ![](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/10/16/16343720658671.jpg) + ![](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/10/16/16343720862217.jpg) + + 可以看到已经运行完毕 + +## git方式拉取私有项目 + +拉取私有项目需在git仓库里添加如下公钥 + +``` +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCd/k/tjbcMislEunjtYQNXxz5tgEDc/fSvuLHBNUX4PtfmMQ07TuUX2XJIIzLRPaqv3nsMn3+QZrV0xQd545FG1Cq83JJB98ATTW7k5Q0eaWXkvThdFeG5+n85KeVV2W4BpdHHNZ5h9RxBUmVZPpAZacdC6OUSBYTyCblPfX9DvjOk+KfwAZVwpJSkv4YduwoR3DNfXrmK5P+wrYW9z/VHUf0hcfWEnsrrHktCKgohZn9Fe8uS3B5wTNd9GgVrLGRk85ag+CChoqg80DjgFt/IhzMCArqwLyMn7rGG4Iu2Ie0TcdMc0TlRxoBhqrfKkN83cfQ3gDf41tZwp67uM9ZN feapder@qq.com +``` + +或在系统设置页面配置您的SSH私钥，然后在git仓库里添加您的公钥，例如： +![](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/10/19/16346353514967.jpg) + +注意，公私钥加密方式为RSA，其他的可能会有问题 + +生成RSA公私钥方式如下： +```shell +ssh-keygen -t rsa -C "备注" -f 生成路径/文件名 +``` +如： +`ssh-keygen -t rsa -C "feaplat" -f id_rsa` +然后一路回车，不要输密码 +![](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/11/17/16371210640228.jpg) +最终生成 `id_rsa`、`id_rsa.pub` 文件，复制`id_rsa.pub`文件内容到git仓库，复制`id_rsa`文件内容到feaplat爬虫管理系统 + + + +## 爬虫监控 + +> 若您使用的是feapder爬虫或者使用了自定义打点，监控才会有对应的数据 + +1. 表名：以 task_id 命名 +2. 保留策略：这是influxdb的概念，监控数据默认保留180天，滚动更新，这个保留策略为`feapder_180d`，同时也被设置成了默认策略`default`。所以直接用`default`就可以。 + +## 系统设置 + +1. GIT_SSH_PRIVATE_KEY：可以在自己的笔记本上使用`cat .ssh/id_rsa`查看，然后把内容复制到进来。不了解git ssh协议的，自行查资料 + +## 更新版本 + +``` +git pull +docker-compose up -d +``` +依次执行以上命令即可 diff --git "a/docs/foreword/10\345\210\206\351\222\237\344\270\212\346\211\213.md" "b/docs/foreword/10\345\210\206\351\222\237\344\270\212\346\211\213.md" index 9985772e..63ac3874 100644 --- "a/docs/foreword/10\345\210\206\351\222\237\344\270\212\346\211\213.md" +++ "b/docs/foreword/10\345\210\206\351\222\237\344\270\212\346\211\213.md" @@ -12,13 +12,13 @@ https://www.qiushibaike.com/8hr/page/1/ -![-w843](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/01/09/16101267651625.jpg?x-oss-process=style/markdown-media) +![-w843](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/01/09/16101267651625.jpg) 我们以糗事百科抓取抓取为例，先抓推荐列表，然后抓详情，带领大家10分钟快速入门 ## 1. 抓列表 -![-w1485](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/01/09/16101239769142.jpg?x-oss-process=style/markdown-media) +![-w1485](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/01/09/16101239769142.jpg) 列表是html里直接返回的，我们打开检查工具，观察标题所在的标签，可写出如下xpath表达式 @@ -47,10 +47,10 @@ https://www.qiushibaike.com/8hr/page/1/ 3. 运行，打印如下： - ![-w1183](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/01/09/16101251846542.jpg?x-oss-process=style/markdown-media) + ![-w1183](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/01/09/16101251846542.jpg) 细心的你，会发现连接也自动补全了，网页源代码如下： - ![-w671](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/01/09/16101252789007.jpg?x-oss-process=style/markdown-media) + ![-w671](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/01/09/16101252789007.jpg) ## 2. 抓详情 @@ -66,9 +66,9 @@ https://www.qiushibaike.com/8hr/page/1/ 解析详情 -![-w1470](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/01/09/16101256627827.jpg?x-oss-process=style/markdown-media) +![-w1470](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/01/09/16101256627827.jpg) - def parser_detail(self, request, response): + def parse_detail(self, request, response): """ 解析详情 """ @@ -102,10 +102,10 @@ https://www.qiushibaike.com/8hr/page/1/ # print(title, url) yield feapder.Request( - url, callback=self.parser_detail, title=title + url, callback=self.parse_detail, title=title ) # callback 为回调函数 - def parser_detail(self, request, response): + def parse_detail(self, request, response): """ 解析详情 """ @@ -139,14 +139,15 @@ thread_count 为线程数 编码错误 -![-w1444](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/01/09/16101262462057.jpg?x-oss-process=style/markdown-media) +![-w1444](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/01/09/16101262462057.jpg) 这是因为框架解析解析text时默认使用了`strict`模式，我们可以加如下一行代码将其改成`ignore`模式，这样遇到不支持的字符便忽略 response.encoding_errors = 'ignore' -![-w743](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/01/09/16101264724039.jpg?x-oss-process=style/markdown-media) +![-w718](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/03/17/16159888418946.jpg) + 我们还可以指定编码，来解决。写法如下 - response.code = '网页编码' # 已将encoding简写为code \ No newline at end of file + response.code = '网页编码' # 已将encoding简写为code diff --git "a/docs/foreword/\345\212\237\350\203\275\346\246\202\350\247\210.md" "b/docs/foreword/\345\212\237\350\203\275\346\246\202\350\247\210.md" index 4efbb062..9c714a34 100644 --- "a/docs/foreword/\345\212\237\350\203\275\346\246\202\350\247\210.md" +++ "b/docs/foreword/\345\212\237\350\203\275\346\246\202\350\247\210.md" @@ -6,7 +6,7 @@ 本框架支持批次采集，引入了批次表的概念，详细记录了每一批次的抓取状态 -![-w899](http://markdown-media.oss-cn-beijing.aliyuncs.com/2020/12/20/16084680404224.jpg?x-oss-process=style/markdown-media) +![-w899](http://markdown-media.oss-cn-beijing.aliyuncs.com/2020/12/20/16084680404224.jpg) ## 2. 支持分布式采集 @@ -20,7 +20,7 @@ 框架内置3种去重机制，通过简单的配置可对任务及数据自动去重，也可拿出来单独作为模块使用，支持批量去重。 -1. 临时去重：处理一万条数据约0.26秒。 去重1亿条数据占用内存约1.43G，可指定去重的失效周期 +1. 临时去重：处理一万条数据约0.26秒。 去重一亿条数据占用内存约1.43G，可指定去重的失效周期 2. 内存去重：处理一万条数据约0.5秒。 去重一亿条数据占用内存约285MB 3. 永久去重：处理一万条数据约3.5秒。去重一亿条数据占用内存约285MB @@ -42,21 +42,21 @@ feapder对于每一条URL数据的抓取采取了强状态的控制，做到采 1. 实时计算爬虫抓取速度，估算剩余时间，在指定的抓取周期内预判是否会超时 - ![-w657](http://markdown-media.oss-cn-beijing.aliyuncs.com/2020/12/20/16084718683378.jpg?x-oss-process=style/markdown-media) + ![-w657](http://markdown-media.oss-cn-beijing.aliyuncs.com/2020/12/20/16084718683378.jpg) 2. 爬虫卡死报警 - ![-w501](http://markdown-media.oss-cn-beijing.aliyuncs.com/2020/12/20/16084718974597.jpg?x-oss-process=style/markdown-media) + ![-w501](http://markdown-media.oss-cn-beijing.aliyuncs.com/2020/12/20/16084718974597.jpg) 3. 爬虫任务失败数过多报警，可能是由于网站模板改动或封堵导致 - ![-w416](http://markdown-media.oss-cn-beijing.aliyuncs.com/2020/12/29/16092335882158.jpg?x-oss-process=style/markdown-media) + ![-w416](http://markdown-media.oss-cn-beijing.aliyuncs.com/2020/12/29/16092335882158.jpg) ## 9. 下载监控 框架对请求总数、成功数、失败数、解析异常数进行监控，将数据点打入到infuxdb，结合Grafana面板，可方便掌握抓取情况 -![-w1299](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/02/09/16128568548280.jpg?x-oss-process=style/markdown-media) +![-w1299](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/02/09/16128568548280.jpg) diff --git "a/docs/foreword/\346\236\266\346\236\204\350\256\276\350\256\241.md" "b/docs/foreword/\346\236\266\346\236\204\350\256\276\350\256\241.md" index 1e0039ad..0cbe9cfd 100644 --- "a/docs/foreword/\346\236\266\346\236\204\350\256\276\350\256\241.md" +++ "b/docs/foreword/\346\236\266\346\236\204\350\256\276\350\256\241.md" @@ -1,7 +1,7 @@ # 框架流程图 -![boris-spider -1-](http://markdown-media.oss-cn-beijing.aliyuncs.com/2020/06/08/borisspider-1.png?x-oss-process=style/markdown-media) +![boris-spider -1-](http://markdown-media.oss-cn-beijing.aliyuncs.com/2020/06/08/borisspider-1.png) ## 模块说明： diff --git a/docs/images/aliyun_sale.jpg b/docs/images/aliyun_sale.jpg deleted file mode 100644 index f7b42b1a..00000000 Binary files a/docs/images/aliyun_sale.jpg and /dev/null differ diff --git a/docs/images/qingguo.jpg b/docs/images/qingguo.jpg new file mode 100644 index 00000000..24331df2 Binary files /dev/null and b/docs/images/qingguo.jpg differ diff --git a/docs/index.html b/docs/index.html index 0101ea51..d1112896 100644 --- a/docs/index.html +++ b/docs/index.html @@ -1,118 +1,172 @@ - + - - feapder-document - - - - - - - - - + + feapder官方文档|feapder-document + + + + + + + + + + + + + + + + + + + + + + + + -
- + + + + + + + + + - - - - - - - - - - - - - - - - - - - - + } + } + // 由于docsify/lib/plugins/gitalk.min.js文件中已经有下面代码了，所以不需要在写一次了 + // gitalk.render('gitalk-container'); // 渲染Gitalk评论组件 + --> + + + + + + + + + + + + + + diff --git a/docs/lib/docsify-copy-code/docsify-copy-code.min.js b/docs/lib/docsify-copy-code/docsify-copy-code.min.js new file mode 100644 index 00000000..dee84c79 --- /dev/null +++ b/docs/lib/docsify-copy-code/docsify-copy-code.min.js @@ -0,0 +1,9 @@ +/*! + * docsify-copy-code + * v2.1.0 + * https://github.com/jperasmus/docsify-copy-code + * (c) 2017-2019 JP Erasmus + * MIT license + */ +!function(){"use strict";function r(o){return(r="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(o){return typeof o}:function(o){return o&&"function"==typeof Symbol&&o.constructor===Symbol&&o!==Symbol.prototype?"symbol":typeof o})(o)}!function(o,e){void 0===e&&(e={});var t=e.insertAt;if(o&&"undefined"!=typeof document){var n=document.head||document.getElementsByTagName("head")[0],c=document.createElement("style");c.type="text/css","top"===t&&n.firstChild?n.insertBefore(c,n.firstChild):n.appendChild(c),c.styleSheet?c.styleSheet.cssText=o:c.appendChild(document.createTextNode(o))}}(".docsify-copy-code-button,.docsify-copy-code-button span{cursor:pointer;transition:all .25s ease}.docsify-copy-code-button{position:absolute;z-index:1;top:0;right:0;overflow:visible;padding:.65em .8em;border:0;border-radius:0;outline:0;font-size:1em;background:grey;background:var(--theme-color,grey);color:#fff;opacity:0}.docsify-copy-code-button span{border-radius:3px;background:inherit;pointer-events:none}.docsify-copy-code-button .error,.docsify-copy-code-button .success{position:absolute;z-index:-100;top:50%;left:0;padding:.5em .65em;font-size:.825em;opacity:0;-webkit-transform:translateY(-50%);transform:translateY(-50%)}.docsify-copy-code-button.error .error,.docsify-copy-code-button.success .success{opacity:1;-webkit-transform:translate(-115%,-50%);transform:translate(-115%,-50%)}.docsify-copy-code-button:focus,pre:hover .docsify-copy-code-button{opacity:1}"),document.querySelector('link[href*="docsify-copy-code"]')&&console.warn("[Deprecation] Link to external docsify-copy-code stylesheet is no longer necessary."),window.DocsifyCopyCodePlugin={init:function(){return function(o,e){o.ready(function(){console.warn("[Deprecation] Manually initializing docsify-copy-code using window.DocsifyCopyCodePlugin.init() is no longer necessary.")})}}},window.$docsify=window.$docsify||{},window.$docsify.plugins=[function(o,s){o.doneEach(function(){var o=Array.apply(null,document.querySelectorAll("pre[data-lang]")),c={buttonText:"Copy to clipboard",errorText:"Error",successText:"Copied"};s.config.copyCode&&Object.keys(c).forEach(function(t){var n=s.config.copyCode[t];"string"==typeof n?c[t]=n:"object"===r(n)&&Object.keys(n).some(function(o){var e=-1',''.concat(c.buttonText,""),''.concat(c.errorText,""),''.concat(c.successText,""),""].join("");o.forEach(function(o){o.insertAdjacentHTML("beforeend",e)})}),o.mounted(function(){document.querySelector(".content").addEventListener("click",function(o){if(o.target.classList.contains("docsify-copy-code-button")){var e="BUTTON"===o.target.tagName?o.target:o.target.parentNode,t=document.createRange(),n=e.parentNode.querySelector("code"),c=window.getSelection();t.selectNode(n),c.removeAllRanges(),c.addRange(t);try{document.execCommand("copy")&&(e.classList.add("success"),setTimeout(function(){e.classList.remove("success")},1e3))}catch(o){console.error("docsify-copy-code: ".concat(o)),e.classList.add("error"),setTimeout(function(){e.classList.remove("error")},1e3)}"function"==typeof(c=window.getSelection()).removeRange?c.removeRange(t):"function"==typeof c.removeAllRanges&&c.removeAllRanges()}})})}].concat(window.$docsify.plugins||[])}(); +//# sourceMappingURL=docsify-copy-code.min.js.map diff --git a/docs/lib/docsify/lib/plugins/docsify-edit-on-github.js b/docs/lib/docsify/lib/plugins/docsify-edit-on-github.js new file mode 100644 index 00000000..d49b1b11 --- /dev/null +++ b/docs/lib/docsify/lib/plugins/docsify-edit-on-github.js @@ -0,0 +1,40 @@ +/** + * Minified by jsDelivr using Terser v3.14.1. + * Original file: /npm/docsify-edit-on-github@1.0.3/index.js + * + * Do NOT use SRI with dynamically generated files! More information: https://www.jsdelivr.com/using-sri-with-dynamic-files + */ +! function(t) { + t.EditOnGithubPlugin = {}, t.EditOnGithubPlugin.create = function(n, i, e) { + function u(t) { + return header = ['
', + '

', 'memo  ', t, "

", "
" + ].join("") + } + return e = e || "Edit on github", i = i || n.replace(/\/blob\//, "/edit/"), t.EditOnGithubPlugin.editDoc = + function(t, n) { + var e = n.route.file; + if (e) { + var u = i + e; + return window.open(u), t.preventDefault(), !1 + } + return !0 + }, + function(n, i) { + if (t.EditOnGithubPlugin.onClick = function(t) { + EditOnGithubPlugin.editDoc(t, i) + }, (r = e) && "[object Function]" === {}.toString.call(r)) n.afterEach(function(t) { + return u(e(i.route.file)) + t + }); + else { + var o = u(e); + n.afterEach(function(t) { + return o + t + }) + } + var r + } + } +}(window); +//# sourceMappingURL=/sm/eef821f4877f09e27be373326100cefe923735a9bb303de51b16f9079d063a86.map \ No newline at end of file diff --git "a/docs/question/setting\344\270\215\347\224\237\346\225\210\351\227\256\351\242\230.md" "b/docs/question/setting\344\270\215\347\224\237\346\225\210\351\227\256\351\242\230.md" new file mode 100644 index 00000000..0a443c97 --- /dev/null +++ "b/docs/question/setting\344\270\215\347\224\237\346\225\210\351\227\256\351\242\230.md" @@ -0,0 +1,38 @@ +# setting不生效问题 + +## 问题 + +以下面这个项目结构为例，在`spiders`目录下运行`spider_test.py`读取不到`setting.py`，所以`setting`的配置不生效。 + +![](http://markdown-media.oss-cn-beijing.aliyuncs.com/2022/11/01/16672715088563.jpg) + +读取不到是因为python的环境变量问题，在spiders目录下运行，只会找spides目录下的文件 + +## 解决方式 + +### 方法1：在setting同级目录下运行 + +在main.py中导入spider_test， 然后运行main.py + +### 方法2：设置工作区间 + +设置工作区间方式（以pycharm为例）：项目->右键->Mark Directory as -> Sources Root + +![](http://markdown-media.oss-cn-beijing.aliyuncs.com/2022/11/01/16672717483410.jpg) + +### 方法3：设置PYTHONPATH + +以mac或linux举例，执行如下命令 + +```shell +export PYTHONPATH=$PYTHONPATH:/绝对路径/spider-project +``` +注：这个命令设置的环境变量只在当前终端有效 + +然后即可在spiders目录下运行 + +```shell +python spider_test.py +``` + +window如何添加环境变量大家自行探索，搞定了可在评论区留言 \ No newline at end of file diff --git "a/docs/question/\345\256\211\350\243\205\351\227\256\351\242\230.md" "b/docs/question/\345\256\211\350\243\205\351\227\256\351\242\230.md" index baf3a148..ac48b2f9 100644 --- "a/docs/question/\345\256\211\350\243\205\351\227\256\351\242\230.md" +++ "b/docs/question/\345\256\211\350\243\205\351\227\256\351\242\230.md" @@ -1,11 +1,28 @@ -# 常见问题 +# 安装问题 -## 1. window下pip 安装报错 +## 1. bitarray问题 -报bitarray问题 +> window下pip 安装报错 -![](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/02/09/16128685646774.jpg?x-oss-process=style/markdown-media) + +![](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/02/09/16128685646774.jpg) 解决办法：安装 Microsoft Visual C++ 工具，工具下载地址如下所示： https://download.microsoft.com/download/5/f/7/5f7acaeb-8363-451f-9425-68a90f98b238/visualcppbuildtools_full.exe +## 2. AttributeError 'str' object has not attribute 'decode' + +> window下pip 安装报错 + +![670479264](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/03/16/670479264.jpg) + +下载bitarray离线包，版本要求`bitarray>=1.5.3` + +https://www.lfd.uci.edu/~gohlke/pythonlibs/#bitarray + +![-w722](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/03/16/16158992617537.jpg) + + +解压，进入目录下执行： + + python setup.py install diff --git "a/docs/question/\350\257\267\346\261\202\351\227\256\351\242\230.md" "b/docs/question/\350\257\267\346\261\202\351\227\256\351\242\230.md" new file mode 100644 index 00000000..4b6b0cec --- /dev/null +++ "b/docs/question/\350\257\267\346\261\202\351\227\256\351\242\230.md" @@ -0,0 +1,7 @@ +# 请求问题 + +## ValueError: check_hostname requires server_hostname + + pip install urllib3==1.25.8 + +参考：https://stackoverflow.com/questions/66642705/why-requests-raise-this-exception-check-hostname-requires-server-hostname \ No newline at end of file diff --git "a/docs/question/\350\277\220\350\241\214\351\227\256\351\242\230.md" "b/docs/question/\350\277\220\350\241\214\351\227\256\351\242\230.md" new file mode 100644 index 00000000..ade03f4d --- /dev/null +++ "b/docs/question/\350\277\220\350\241\214\351\227\256\351\242\230.md" @@ -0,0 +1,30 @@ +# 运行问题 + +## 1. 二次运行时卡住，不继续抓取 + +![1779423237](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/03/11/1779423237.jpg) + +**原因：** + +因爬虫支持分布式和任务防丢，为防止任务抢占和任务丢失，巧妙的利用了redis有序集合来存储任务。 + +策略：有序集合有个分数，爬虫取任务时，只取小于当前时间戳分数的任务，同时将任务分数修改为当前时间戳+10分钟，当任务做完时，再主动将任务删除。 + +目的：将取到的任务分数修改成10分钟后，可防止其他爬虫节点取到同样的任务，同时当爬虫意外退出后，任务也不会丢失，10分钟后还可以取到。但也会导致有时爬虫启动时，明明有任务，却处于等待任务的情况。 + +应对等待情况： + +1. 可将任务清空，重新抓取，可直接操作redis清空，或通过传参方式 + + spider = test_spider.TestSpider(redis_key="feapder:test_spider", delete_keys="*z_requsets") + spider.start() + + delete_keys为需要删除的key，类型: 元组/bool/string，支持正则; 常用于清空任务队列，否则重启时会断点续爬，如写成`delete_keys=True`也是可以的 + +1. 手动修改任务分数为小于当前时间戳的分数 + + ![-w917](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/03/11/16154327722622.jpg) + +1. 等10分钟就好了 + +2. 用debug模式开发 diff --git a/docs/robots.txt b/docs/robots.txt new file mode 100644 index 00000000..4f9540ba --- /dev/null +++ b/docs/robots.txt @@ -0,0 +1 @@ +User-agent: * \ No newline at end of file diff --git a/docs/source_code/BaseParser.md b/docs/source_code/BaseParser.md index 16466a4a..cb54bfb4 100644 --- a/docs/source_code/BaseParser.md +++ b/docs/source_code/BaseParser.md @@ -18,25 +18,41 @@ class BaseParser(object): pass - def parse(self, request, response): + def download_midware(self, request): """ - @summary: 默认的回调函数 + @summary: 下载中间件 可修改请求的一些参数, 或可自定义下载，然后返回 request, response + --------- + @param request: + --------- + @result: return request / request, response + """ + + pass + + def validate(self, request, response): + """ + @summary: 校验函数, 可用于校验response是否正确 + 若函数内抛出异常，则重试请求 + 若返回True 或 None，则进入解析函数 + 若返回False，则抛弃当前请求 + 可通过request.callback_name 区分不同的回调函数，编写不同的校验逻辑 --------- @param request: @param response: --------- - @result: + @result: True / None / False """ pass - def download_midware(self, request): + def parse(self, request, response): """ - @summary: 下载中间件 可修改请求的一些参数 + @summary: 默认的解析函数 --------- @param request: + @param response: --------- - @result: return request / None (不会修改原来的request) + @result: """ pass diff --git "a/docs/source_code/BatchSpider\350\277\233\351\230\266.md" "b/docs/source_code/BatchSpider\350\277\233\351\230\266.md" index 28552a7c..38813818 100644 --- "a/docs/source_code/BatchSpider\350\277\233\351\230\266.md" +++ "b/docs/source_code/BatchSpider\350\277\233\351\230\266.md" @@ -1,3 +1,223 @@ # BatchSpider -未完待续 \ No newline at end of file +## BatchSpider参数 + +```python +def __init__( + self, + task_table, + batch_record_table, + batch_name, + batch_interval, + task_keys, + task_state="state", + min_task_count=10000, + check_task_interval=5, + task_limit=10000, + related_redis_key=None, + related_batch_record=None, + task_condition="", + task_order_by="", + redis_key=None, + thread_count=None, + begin_callback=None, + end_callback=None, + delete_keys=(), + keep_alive=None, + send_run_time=False, +): + """ + @summary: 批次爬虫 + 必要条件 + 1、需有任务表 + 任务表中必须有id 及 任务状态字段 如 state。如指定parser_name字段，则任务会自动下发到对应的parser下, 否则会下发到所有的parser下。其他字段可根据爬虫需要的参数自行扩充 + + 参考建表语句如下： + CREATE TABLE `table_name` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `param` varchar(1000) DEFAULT NULL COMMENT '爬虫需要的抓取数据需要的参数', + `state` int(11) DEFAULT NULL COMMENT '任务状态', + `parser_name` varchar(255) DEFAULT NULL COMMENT '任务解析器的脚本类名', + PRIMARY KEY (`id`), + UNIQUE KEY `nui` (`param`) USING BTREE + ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8; + + 2、需有批次记录表 不存在自动创建 + --------- + @param task_table: mysql中的任务表 + @param batch_record_table: mysql 中的批次记录表 + @param batch_name: 批次采集程序名称 + @param batch_interval: 批次间隔 天为单位。 如想一小时一批次，可写成1/24 + @param task_keys: 需要获取的任务字段 列表 [] 如需指定解析的parser，则需将parser_name字段取出来。 + @param task_state: mysql中任务表的任务状态字段 + @param min_task_count: redis 中最少任务数, 少于这个数量会从mysql的任务表取任务 + @param check_task_interval: 检查是否还有任务的时间间隔； + @param task_limit: 从数据库中取任务的数量 + @param redis_key: 任务等数据存放在redis中的key前缀 + @param thread_count: 线程数，默认为配置文件中的线程数 + @param begin_callback: 爬虫开始回调函数 + @param end_callback: 爬虫结束回调函数 + @param delete_keys: 爬虫启动时删除的key，类型: 元组/bool/string。 支持正则; 常用于清空任务队列，否则重启时会断点续爬 + @param keep_alive: 爬虫是否常驻，默认否 + @param send_run_time: 发送运行时间 + @param related_redis_key: 有关联的其他爬虫任务表（redis）注意：要避免环路 如 A -> B & B -> A 。 + @param related_batch_record: 有关联的其他爬虫批次表（mysql）注意：要避免环路 如 A -> B & B -> A 。 + related_redis_key 与 related_batch_record 选其一配置即可；用于相关联的爬虫没结束时，本爬虫也不结束 + 若相关连的爬虫为批次爬虫，推荐以related_batch_record配置， + 若相关连的爬虫为普通爬虫，无批次表，可以以related_redis_key配置 + @param task_condition: 任务条件 用于从一个大任务表中挑选出数据自己爬虫的任务，即where后的条件语句 + @param task_order_by: 取任务时的排序条件 如 id desc + --------- + @result: + """ +``` + +下面介绍下理解起来可能有疑惑的参数 + +### 1. related_redis_key 与 related_batch_record + +这两个参数用于采集之间有关联的爬虫，比如列表爬虫和详情爬虫，详情的任务需依赖列表爬虫生产，列表爬虫没采集完毕，详情爬虫要处于等待状态。 + +举例说明：BatchSpider依赖Spider + +``` + +def crawl_list(): + """ + 普通爬虫 Spider + """ + spider = spider_test.SpiderTest(redis_key="feapder:list") + spider.start() + + +def crawl_detail(args): + """ + 批次爬虫 BatchSpider + @param args: 1 / 2 / init + """ + spider = batch_spider_test.BatchSpiderTest( + task_table="list_task", # mysql中的任务表 + batch_record_table="list_batch_record", # mysql中的批次记录表 + batch_name="详情爬虫(周全)", # 批次名字 + batch_interval=7, # 批次时间 天为单位 若为小时 可写 1 / 24 + task_keys=["id", "item_id"], # 需要获取任务表里的字段名，可添加多个 + redis_key="feapder:detail", # redis中存放request等信息的根key + task_state="state", # mysql中任务状态字段 + related_redis_key="feapder:list:z_requsets" + ) + + if args == 1: + spider.start_monitor_task() + elif args == 2: + spider.start() +``` + +若批次爬虫和批次爬虫之间有依赖，除了设置related_redis_key参数外，还支持设置related_batch_record参数，指定对方的批次记录表即可。两个参数二选一 + +### 2. task_condition + +取任务的条件，可以理解为sql后面的where条件。如获取url不为空的任务且id大于10的任务 + + task_condition="url is not null and id > 10" + + +## BatchSpider方法 + +BatchSpider继承至BatchParser，并且BatchParser是对开发者暴露的常用方法接口，因此推荐先看[BatchParser](source_code/BatchParser)，BatchSpider方法如下： + +### init_task 任务初始化 + +init_task函数会在每个批次开始时调用，用于将已完成的任务状态重置为0。 + +因此当本函数被重写为空时，可实现增量抓取 + +``` +def init_task(self): + pass +``` + +当手动调用本函数时，可将任务状态刷新为0，开发阶段经常使用 + + spider.init_task() + + +## 其他细节 + +### 1. 任务防丢 + +BatchSpider除了支持Spider的[任务防丢机制](source_code/Spider进阶?id=_1-任务防丢)外，还多了一层mysql任务表的保障，mysql任务表中每条任务都有任务状态，BatchSpider有任务丢失重发机制，直到所有任务都处于成功或者失败两种状态，才算采集结束。 + +### 2. 任务重试 + +> 与Spider相同 + +任务请求失败或解析函数抛出异常时，会自动重试，默认重试次数为100次，可通过配置文件`SPIDER_MAX_RETRY_TIMES`参数修改。当任务超过最大重试次数时，默认会将失败的任务存储到Redis的`{redis_key}:z_failed_requsets`里，供人工排查。 + +相关配置为： + +```python +# 每个请求最大重试次数 +SPIDER_MAX_RETRY_TIMES = 100 +# 重新尝试失败的requests 当requests重试次数超过允许的最大重试次数算失败 +RETRY_FAILED_REQUESTS = False +# 保存失败的request +SAVE_FAILED_REQUEST = True +# 任务失败数 超过WARNING_FAILED_COUNT则报警 +WARNING_FAILED_COUNT = 1000 +``` + +当`RETRY_FAILED_REQUESTS=True`时，爬虫再次启动时会将失败的任务重新下发到任务队列中，重新抓取 + +### 3. 去重 + +> 与Spider相同 + +支持任务去重和数据去重，任务默认是临时去重，去重库保留1个月，即只去重1个月内的任务，数据是永久去重。默认去重是关闭的，相关配置为： + +``` +ITEM_FILTER_ENABLE = False # item 去重 +REQUEST_FILTER_ENABLE = False # request 去重 +``` + +修改默认去重库： + +``` +from feapder.buffer.request_buffer import RequestBuffer +from feapder.buffer.item_buffer import ItemBuffer +from feapder.dedup import Dedup + +RequestBuffer.dedup = Dedup(filter_type=Dedup.MemoryFilter) +ItemBuffer.dedup = Dedup(filter_type=Dedup.MemoryFilter) +``` + +RequestBuffer 为任务入库前缓冲的buffer，ItemBuffer为数据入库前缓冲的buffer + +关于去重库详情见：[海量数据去重](source_code/dedup) + +### 4. 加速采集 + +> 与Spider相同 + +与爬虫采集速度的相关配置为： + +```python +# 爬虫相关 +# COLLECTOR +COLLECTOR_SLEEP_TIME = 1 # 从任务队列中获取任务到内存队列的间隔 +COLLECTOR_TASK_COUNT = 10 # 每次获取任务数量 + +# SPIDER +SPIDER_THREAD_COUNT = 1 # 爬虫并发数 +SPIDER_SLEEP_TIME = 0 # 下载时间间隔（解析完一个response后休眠时间） +SPIDER_MAX_RETRY_TIMES = 100 # 每个请求最大重试次数 +# 是否主动执行添加 设置为False 需要手动调用start_monitor_task，适用于多进程情况下 + +``` + +COLLECTOR 为从任务队列中取任务到内存队列的线程，SPIDER为实际采集的线程 + +`COLLECTOR_TASK_COUNT` 建议 >= `SPIDER_THREAD_COUNT`, 这样每个线程的爬虫才有任务可做。但COLLECTOR_TASK_COUNT不建议过大，不然分布式时，一个池子里的任务都被节点A取走了，其他节点取不到任务了。 + +### 了解更多 + +更多配置，详见[配置文件](source_code/配置文件) diff --git a/docs/source_code/Item.md b/docs/source_code/Item.md index e6ba510e..e48218b9 100644 --- a/docs/source_code/Item.md +++ b/docs/source_code/Item.md @@ -1,3 +1,140 @@ # Item -未完待续 \ No newline at end of file +有关Item的简介及创建，可参考[命令行工具](command/cmdline?id=_3-创建-item) + +## 数据入库 + +数据自动入库，除了根据mysql表生产item外，也可以直接给item赋值，示例如下： + +``` +from feapder import Item + +item = Item() +item.table_name = "spider_data" # 表名 +item.title = title +yield item +``` + +等价于： + +1. 生成item + + ``` + from feapder import Item + + class SpiderDataItem(Item): + """ + This class was generated by feapder. + command: feapder create -i spider_data. + """ + + def __init__(self, *args, **kwargs): + # self.id = None + self.title = None + ``` + +1. 使用 + + ``` + item = SpiderDataItem() + item.title = title + yield item + ``` + +## Item指纹 + +item指纹用于数据入库前的去重，默认为所有字段值排序后计算的md5，但当数据中有采集时间时，这种指纹计算方式明显不合理。因此我们可以通过如下方法指定参与去重的key + +``` +from feapder import Item + + +class SpiderDataItem(Item): + + __unique_key__ = ["title", "url"] # 指定去重的key为 title、url，最后的指纹为title与url值联合计算的md5 + + def __init__(self, *args, **kwargs): + # self.id = None + self.title = None + self.url = None + self.crawl_time = None +``` + +或可通过如下方式指定`__unique_key__` + +``` +item = SpiderDataItem() +item.unique_key = ["title", "url"] # 支持列表、元组、字符串 +``` + +或者重写指纹函数 + +``` +from feapder import Item + + +class SpiderDataItem(Item): + ... + + @property + def fingerprint(self): + return "我是指纹" +``` + +## 入库前对item进行处理 + +pre_to_db函数为每个item入库前的回调函数，可通过此函数对数据进行处理 + +```python +from feapder import Item + + +class SpiderDataItem(Item): + + def __init__(self, *args, **kwargs): + # self.id = None + self.title = None + + def pre_to_db(self): + """ + 入库前的处理 + """ + self.title = self.title.strip() +``` + +## 指定入库使用的pipelines + +```python + +from feapder import Item +from feapder.pipelines.csv_pipeline import CsvPipeline + + +class SpiderDataItem(Item): + + __pipelines__ = [CsvPipeline()] + + def __init__(self, *args, **kwargs): + # self.id = None + self.title = None +``` + +使用__pipelines__指定后，该item只会流经指定的pipelines处理 + + +## 更新数据 + +采集过程中，往往会有些数据漏采或解析出错，如果我们想更新已入库的数据，可将Item转为UpdateItem + + item = SpiderDataItem.to_UpdateItem() + +或直接修改继承类 + +``` +from feapder import Item, UpdateItem + +class SpiderDataItem(UpdateItem): + ... +``` + +关于UpdateItem使用，详见[UpdateItem](source_code/UpdateItem) diff --git a/docs/source_code/ItemBuffer.md b/docs/source_code/ItemBuffer.md deleted file mode 100644 index 264511d7..00000000 --- a/docs/source_code/ItemBuffer.md +++ /dev/null @@ -1,3 +0,0 @@ -# ItemBuffer - -未完待续 \ No newline at end of file diff --git a/docs/source_code/MongoDB.md b/docs/source_code/MongoDB.md new file mode 100644 index 00000000..1787e39b --- /dev/null +++ b/docs/source_code/MongoDB.md @@ -0,0 +1,152 @@ +# MongoDB + +## 数据自动入Mongo库使用须知 + +- 使用`MongoDb`存储数据，需要使用`MongoPipeline` + +示例: + +```python +import feapder +from feapder import Item + + +class TestMongo(feapder.AirSpider): + __custom_setting__ = dict( + ITEM_PIPELINES=["feapder.pipelines.mongo_pipeline.MongoPipeline"], + MONGO_IP="localhost", + MONGO_PORT=27017, + MONGO_DB="feapder", + MONGO_USER_NAME="", + MONGO_USER_PASS="", + ) + + def start_requests(self): + yield feapder.Request("https://www.baidu.com") + + def parse(self, request, response): + title = response.xpath("//title/text()").extract_first() # 取标题 + item = Item() # 声明一个item + item.table_name = "test_mongo" # 指定存储的表名 + item.title = title # 给item属性赋值 + yield item # 返回item， item会自动批量入库 + + +if __name__ == "__main__": + TestMongo().start() +``` + + +## 直接使用 + +### 连接 + +```python +from feapder.db.mongodb import MongoDB + + +db = MongoDB( + ip="localhost", port=27017, db="feapder", user_name="feapder", user_pass="feapder123" +) +``` + +若环境变量中配置了数据库连接方式或者setting中已配置，则可不传参 + +```python +db = MongoDB() +``` + +或者可以根据url连接 + +```python +db = MongoDB.from_url("mongodb://username:password@ip:port/db") +``` + +### 方法 + +> MongoDB封装了增删改查等方法，方便使用 + +#### 查 + +```python +def find(self, table, limit=0) -> List[Dict]: + """ + @summary: + 无数据： 返回() + 有数据： 若limit == 1 则返回 (data1, data2) + 否则返回 ((data1, data2),) + --------- + @param table: + @param limit: + --------- + @result: + """ +``` + + +#### 增 + +```python +def add(self, table, data, **kwargs): + """ + + Args: + table: + data: + kwargs: + auto_update: 覆盖更新，将替换唯一索引重复的数据，默认False + update_columns: 更新指定的列（如果数据的唯一索引存在，则更新指定字段，如 update_columns = ["name", "title"] + insert_ignore: 唯一索引冲突时是否忽略，默认为False + condition_fields: 用于条件查找的字段，默认以`_id`作为查找条件，默认：['_id'] + exception_callfunc: 异常回调 + + Returns: 添加行数 + + """ +``` + +```python +def add_batch(self, table: str, datas: List[Dict], **kwargs): + """ + @summary: 批量添加数据 + --------- + @param command: 字典 + @param datas: 列表 [[..], [...]] + @param **kwargs: + auto_update: 覆盖更新，将替换唯一索引重复的数据，默认False + update_columns: 更新指定的列（如果数据的唯一索引存在，则更新指定字段，如 update_columns = ["name", "title"] + update_columns_value: 指定更新的字段对应的值 + condition_fields: 用于条件查找的字段，默认以`_id`作为查找条件，默认：['_id'] + --------- + @result: 添加行数 + """ +``` + +#### 更新 + +```python +def update(self, coll_name, data: Dict, condition: Dict, upsert: bool = False): + """ + 更新 + Args: + coll_name: 集合名 + data: 单条数据 {"xxx":"xxx"} + condition: 更新条件 {"_id": "xxxx"} + upsert: 数据不存在则插入,默认为 False + + Returns: True / False + """ +``` + +#### 删除 + +```python +def delete(self, table, condition: Dict): + """ + 删除 + Args: + table: + condition: 查找条件 + Returns: True / False + """ +``` diff --git a/docs/source_code/MysqlDB.md b/docs/source_code/MysqlDB.md index 7f5a842f..d645653a 100644 --- a/docs/source_code/MysqlDB.md +++ b/docs/source_code/MysqlDB.md @@ -1,3 +1,146 @@ # MysqlDB -未完待续 \ No newline at end of file +MysqlDB具有断开自动重连特性，支持多线程下操作，内置连接池，最大连接数100 + +## 连接 + +```python +from feapder.db.mysqldb import MysqlDB + + +db = MysqlDB( + ip="localhost", port=3306, db="feapder", user_name="feapder", user_pass="feapder123" +) +``` + +若环境变量中配置了数据库连接方式或者setting中已配置，则可不传参 + +```python +db = MysqlDB() +``` + +或者可以根据url连接 + +```python +db = MysqlDB.from_url("mysql://username:password@ip:port/db?charset=utf8mb4") +``` + +## 方法 + +> MysqlDB封装了增删改查等方法，方便使用 + +### 查 + +```python +def find(self, sql, limit=0, to_json=False): + """ + @summary: + 无数据： 返回() + 有数据： 若limit == 1 则返回 (data1, data2) + 否则返回 ((data1, data2),) + --------- + @param sql: + @param limit: + @param to_json 是否将查询结果转为json + --------- + @result: + """ +``` + + +### 增 + +```python +def add(self, sql, exception_callfunc=None): + """ + Args: + sql: + exception_callfunc: 异常回调 + + Returns:添加行数 + + """ +``` + +```python +def add_smart(self, table, data: Dict, **kwargs): + """ + 添加数据, 直接传递json格式的数据，不用拼sql + Args: + table: 表名 + data: 字典 {"xxx":"xxx"} + **kwargs: + + Returns:添加行数 + + """ +``` + + +```python +def add_batch(self, sql, datas: List[Dict]): + """ + @summary: 批量添加数据 + --------- + @ param sql: insert ignore into (xxx, xxx) values (%s, %s, %s) + @ param datas: 列表 [{}, {}, {}] + --------- + @result:添加行数 + """ +``` + +```python +def add_batch_smart(self, table, datas: List[Dict], **kwargs): + """ + 批量添加数据, 直接传递list格式的数据，不用拼sql + Args: + table: 表名 + datas: 列表 [{}, {}, {}] + **kwargs: + + Returns: 添加行数 + + """ +``` + +### 更新 + +```python +def update(self, sql): + pass +``` + +```python +def update_smart(self, table, data: Dict, condition): + """ + 更新, 不用拼sql + Args: + table: 表名 + data: 数据 {"xxx":"xxx"} + condition: 更新条件 where后面的条件，如 condition='status=1' + + Returns: True / False + + """ +``` + +### 删除 + +```python +def delete(self, sql): + """ + 删除 + Args: + sql: + + Returns: True / False + + """ +``` + +### 执行其他sql + +```python +def execute(self, sql): + pass +``` \ No newline at end of file diff --git a/docs/source_code/RedisDB.md b/docs/source_code/RedisDB.md index 7f9bf9d1..39735de9 100644 --- a/docs/source_code/RedisDB.md +++ b/docs/source_code/RedisDB.md @@ -1,3 +1,64 @@ # RedisDB -未完待续 \ No newline at end of file +RedisDB支持**哨兵模式**、**集群模式**与单节点的**普通模式**，封装了操作redis的常用的方法 + +## 连接 + +> 若环境变量中配置了数据库连接方式或者setting中已配置，则可不传参 + +### 普通模式 + +```python +from feapder.db.redisdb import RedisDB + +db = RedisDB(ip_ports="localhost:6379", db=0, user_pass=None) +``` + +使用地址连接 + +```python +from feapder.db.redisdb import RedisDB + +db = RedisDB.from_url("redis://[[username]:[password]]@[host]:[port]/[db]") +``` + +### 哨兵模式 + +```python +from feapder.db.redisdb import RedisDB + +db = RedisDB(ip_ports="172.25.21.4:26379,172.25.21.5:26379,172.25.21.6:26379", db=0, user_pass=None, service_name="my_master") +``` + +注意：多个地址用逗号分隔，需传递`service_name` + +对应setting配置文件，配置方式为： + +```python +REDISDB_IP_PORTS = "172.25.21.4:26379,172.25.21.5:26379,172.25.21.6:26379" +REDISDB_USER_PASS = "" +REDISDB_DB = 0 +REDISDB_SERVICE_NAME = "my_master" +``` + +### 集群模式 + +```python +from feapder.db.redisdb import RedisDB + +db = RedisDB(ip_ports="172.25.21.4:26379,172.25.21.5:26379,172.25.21.6:26379", db=0, user_pass=None) +``` + +注意：多个地址用逗号分隔，不用传递`service_name` + +对应setting配置文件，配置方式为： + +```python +REDISDB_IP_PORTS = "172.25.21.4:26379,172.25.21.5:26379,172.25.21.6:26379" +REDISDB_USER_PASS = "" +REDISDB_DB = 0 +``` + +## 方法： + +详见源码，此处不一一列举， 源码：`feapder.db.redisdb` \ No newline at end of file diff --git a/docs/source_code/Request.md b/docs/source_code/Request.md index e30ba5d2..7e3df49f 100644 --- a/docs/source_code/Request.md +++ b/docs/source_code/Request.md @@ -36,6 +36,8 @@ Request除了支持requests的所有参数外，更需要关心的是框架中 @param random_user_agent: 是否随机User-Agent (True/False) 当setting中的RANDOM_HEADERS设置为True时该参数生效 默认True @param download_midware: 下载中间件。默认为parser中的download_midware @param is_abandoned: 当发生异常时是否放弃重试 True/False. 默认False +@param render: 是否用浏览器渲染 +@param render_time: 渲染时长，即打开网页等待指定时间后再获取源码 -- 以下参数于requests参数使用方式一致 @param method: 请求方式，如POST或GET，默认根据data值是否为空来判断 @@ -53,7 +55,7 @@ Request除了支持requests的所有参数外，更需要关心的是框架中 @param stream: 如果为 False，将会立即下载响应内容 @param cert: -- -@param **kwargs: 其他值: 如 Request(item=item) 则item可直接用 reqeust.item 取出 +@param **kwargs: 其他值: 如 Request(item=item) 则item可直接用 request.item 取出 --------- ``` diff --git a/docs/source_code/Response.md b/docs/source_code/Response.md index 1be3d353..0fa80e60 100644 --- a/docs/source_code/Response.md +++ b/docs/source_code/Response.md @@ -89,15 +89,31 @@ def re_first(self, regex, default=None, replace_entities=False): response.re(">>str(content, errors='strict') Traceback (most recent call last): - File "/Users/liubo/workspace/feapder/venv2/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 3343, in run_code + File "/Users/Boris/workspace/feapder/venv2/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 3343, in run_code exec(code_obj, self.user_global_ns, self.user_ns) File "", line 1, in str(content, errors='strict') diff --git "a/docs/source_code/Spider\350\277\233\351\230\266.md" "b/docs/source_code/Spider\350\277\233\351\230\266.md" index 03fc450c..215898a8 100644 --- "a/docs/source_code/Spider\350\277\233\351\230\266.md" +++ "b/docs/source_code/Spider\350\277\233\351\230\266.md" @@ -1,20 +1,38 @@ -# Spider +# Spider进阶 ## Spider参数： ```python -@param redis_key: 任务等数据存放在redis中的key前缀 -@param min_task_count: 任务队列中最少任务数, 少于这个数量才会添加任务，默认1。start_monitor_task 模式下生效 -@param check_task_interval: 检查是否还有任务的时间间隔；默认5秒 -@param thread_count: 线程数，默认为配置文件中的线程数 -@param begin_callback: 爬虫开始回调函数 -@param end_callback: 爬虫结束回调函数 -@param delete_tabs: 爬虫启动时删除的表（redis里的key），元组类型。 支持正则; 常用于清空任务队列，否则重启时会断点续爬 -@param auto_stop_when_spider_done: 爬虫抓取完毕后是否自动结束或等待任务，默认自动结束 -@param auto_start_requests: 爬虫是否自动添加任务 -@param send_run_time: 发送运行时间 -@param batch_interval: 抓取时间间隔 默认为0 天为单位 多次启动时，只有当前时间与第一次抓取结束的时间间隔大于指定的时间间隔时，爬虫才启动 -@param wait_lock: 下发任务时否等待锁，若不等待锁，可能会存在多进程同时在下发一样的任务，因此分布式环境下请将该值设置True +def __init__( + self, + redis_key=None, + min_task_count=1, + check_task_interval=5, + thread_count=None, + begin_callback=None, + end_callback=None, + delete_keys=(), + keep_alive=None, + auto_start_requests=None, + send_run_time=False, + batch_interval=0, + wait_lock=True +): + """ + @param redis_key: 任务等数据存放在redis中的key前缀 + @param min_task_count: 任务队列中最少任务数, 少于这个数量才会添加任务，默认1。start_monitor_task 模式下生效 + @param check_task_interval: 检查是否还有任务的时间间隔；默认5秒 + @param thread_count: 线程数，默认为配置文件中的线程数 + @param begin_callback: 爬虫开始回调函数 + @param end_callback: 爬虫结束回调函数 + @param delete_keys: 爬虫启动时删除的key，类型: 元组/bool/string。 支持正则; 常用于清空任务队列，否则重启时会断点续爬 + @param keep_alive: 爬虫是否常驻 + @param auto_start_requests: 爬虫是否自动添加任务 + @param send_run_time: 发送运行时间 + @param batch_interval: 抓取时间间隔 默认为0 天为单位 多次启动时，只有当前时间与第一次抓取结束的时间间隔大于指定的时间间隔时，爬虫才启动 + @param wait_lock: 下发任务时否等待锁，若不等待锁，可能会存在多进程同时在下发一样的任务，因此分布式环境下请将该值设置True + """ + ``` 下面介绍下理解起来可能有疑惑的参数 @@ -23,14 +41,14 @@ redis_key为redis中存储任务等信息的key前缀，如redis_key="feapder:spider_test", 则redis中会生成如下 -![-w365](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/02/21/16139009217536.jpg?x-oss-process=style/markdown-media) +![-w365](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/02/21/16139009217536.jpg) key的命名方式为[配置文件](source_code/配置文件.md)中定义的 # 任务表模版 - TAB_REQUSETS = "{redis_key}:z_requsets" + TAB_REQUESTS = "{redis_key}:z_requsets" # 任务失败模板 - TAB_FAILED_REQUSETS = "{redis_key}:z_failed_requsets" + TAB_FAILED_REQUESTS = "{redis_key}:z_failed_requsets" # 爬虫状态表模版 TAB_SPIDER_STATUS = "{redis_key}:z_spider_status" # item 表模版 @@ -38,11 +56,174 @@ key的命名方式为[配置文件](source_code/配置文件.md)中定义的 # 爬虫时间记录表 TAB_SPIDER_TIME = "{redis_key}:h_spider_time" + ### 2. min_task_count -这个参数用于控制最小任务数的，少于这个数量再下发任务，防止redis中堆积任务太多，内存撑爆 +这个参数用于控制最小任务数的，少于这个数量再下发任务，防止redis中堆积任务太多，内存撑爆，通常用于从数据库中取任务，下发。 + +此参数需要使用`start_monitor_task`方式才会生效，示例如下： + +```python +import feapder +from feapder.db.mysqldb import MysqlDB + + +class SpiderTest(feapder.Spider): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.db = MysqlDB() + + def start_requests(self): + sql = "select url from feapder_test where state = 0 limit 1000" + result = self.db.find(sql) + for url, in result: + yield feapder.Request(url) + + def parser(self, request, response): + print(response) + + +if __name__ == "__main__": + spider = SpiderTest( + redis_key="feapder:spider_test", min_task_count=100 + ) + # 监控任务，若任务数小于min_task_count，则调用start_requests下发一批，注start_requests产生的任务会一次下发完，比如本例，会一次下发1000个任务，然后任务队列中少于100条任务时，再下发1000条 + spider.start_monitor_task() + # 采集 + # spider.start() +``` + +`spider.start_monitor_task()` 与 `spider.start()` 分开运行，属于master、worker两种进程 + +### 3. delete_keys + +通常在开发阶段使用，如想清空任务队列重新抓取，或防止由于任务防丢策略导致爬虫需等待10分钟才能取到任务的情况。使用场景见[运行问题](question/运行问题?id=_1-二次运行时卡住，不继续抓取) + +delete_keys 接收类型为tuple/bool/string，支持正则，拿以下的key举例 + +![-w365](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/02/21/16139009217536.jpg) + +删除`feapder:spider_test_z_requests` 可写为 `delete_keys="*z_requests"` + +删除全部可写为`delete_keys="*"` + +### 4. keep_alive + +用于`spider.start_monitor_task()` 与 `spider.start()` 这种master、worker模式。 + +`keep_alive=True`时，爬虫做完任务后不会退出，继续等待任务。 + +### 5. send_run_time + +是否将运行时间作为报警信息发送 + +### 6. batch_interval + +设置每次采集的时间间隔，如我们设置7天，当爬虫正常结束后，7天内我们二次运行爬虫时会自动退出，不执行采集逻辑 + +``` +def spider_test(): + spider = test_spider.TestSpider(redis_key="feapder:test_spider", batch_interval=7) + spider.start() + +``` +运行： + + 上次运行结束时间为 2021-03-15 14:42:31 与当前时间间隔 为 3秒, 小于规定的抓取时间间隔 7天。爬虫不执行，退出～ + +## Spider方法 + +Spider继承至BaseParser，并且BaseParser是对开发者暴露的常用方法接口，因此推荐先看[BaseParser](source_code/BaseParser)，Spider方法如下： + +### 1. start_monitor_task + +下发及监控任务，与`keep_alive`参数配合使用，用于常驻进程的爬虫 + +使用： + + spider = test_spider.TestSpider(redis_key="feapder:test_spider", keep_alive=True) + # 下发及监控任务 + spider.start_monitor_task() + # 采集（进程常驻） + # spider.start() + +`spider.start_monitor_task()` 与 `spider.start()` 分开运行，属于master、worker两种进程 + +## 其他细节 + +### 1. 任务防丢 + +Spider爬虫支持任务防丢，断点续爬，实现原理如下： + +Spider利用了redis有序集合来存储任务，有序集合有个分数，爬虫取任务时，只取小于当前时间戳分数的任务，同时将任务分数修改为当前时间戳+10分钟，（这个取任务与改分数是原子性的操作）。**当任务做完时，且数据已入库后，再主动将任务删除。** + +目的：将取到的任务分数修改成10分钟后，可防止其他爬虫节点取到同样的任务，同时当爬虫意外退出后，任务也不会丢失，10分钟后还可以取到。 + +10分钟是可配置的，为配置文件中的`REQUEST_LOST_TIMEOUT`参数 + +### 2. 任务重试 + +任务请求失败或解析函数抛出异常时，会自动重试，默认重试次数为100次，可通过配置文件`SPIDER_MAX_RETRY_TIMES`参数修改。当任务超过最大重试次数时，默认会将失败的任务存储到Redis的`{redis_key}:z_failed_requsets`里，供人工排查。 + +相关配置为： + +```python +# 每个请求最大重试次数 +SPIDER_MAX_RETRY_TIMES = 100 +# 重新尝试失败的requests 当requests重试次数超过允许的最大重试次数算失败 +RETRY_FAILED_REQUESTS = False +# 保存失败的request +SAVE_FAILED_REQUEST = True +# 任务失败数 超过WARNING_FAILED_COUNT则报警 +WARNING_FAILED_COUNT = 1000 +``` + +当`RETRY_FAILED_REQUESTS=True`时，爬虫再次启动时会将失败的任务重新下发到任务队列中，重新抓取 + +### 3. 去重 + +支持任务去重和数据去重，任务默认是临时去重，去重库保留1个月，即只去重1个月内的任务，数据是永久去重。默认去重是关闭的，相关配置为： + +``` +ITEM_FILTER_ENABLE = False # item 去重 +REQUEST_FILTER_ENABLE = False # request 去重 +``` + +修改默认去重库： + +``` +from feapder.buffer.request_buffer import RequestBuffer +from feapder.buffer.item_buffer import ItemBuffer +from feapder.dedup import Dedup + +RequestBuffer.dedup = Dedup(filter_type=Dedup.MemoryFilter) +ItemBuffer.dedup = Dedup(filter_type=Dedup.MemoryFilter) +``` + +RequestBuffer 为任务入库前缓冲的buffer，ItemBuffer为数据入库前缓冲的buffer + +关于去重库详情见：[海量数据去重](source_code/dedup) + +### 4. 加速采集 + +与爬虫采集速度的相关配置为： + +```python +# 爬虫相关 +# COLLECTOR +COLLECTOR_SLEEP_TIME = 1 # 从任务队列中获取任务到内存队列的间隔 +COLLECTOR_TASK_COUNT = 10 # 每次获取任务数量 + +# SPIDER +SPIDER_THREAD_COUNT = 1 # 爬虫并发数 +SPIDER_SLEEP_TIME = 0 # 下载时间间隔（解析完一个response后休眠时间） +SPIDER_MAX_RETRY_TIMES = 100 # 每个请求最大重试次数 +``` + +COLLECTOR 为从任务队列中取任务到内存队列的线程，SPIDER为实际采集的线程 +`COLLECTOR_TASK_COUNT` 建议 >= `SPIDER_THREAD_COUNT`, 这样每个线程的爬虫才有任务可做。但COLLECTOR_TASK_COUNT不建议过大，不然分布式时，一个池子里的任务都被节点A取走了，其他节点取不到任务了。 ----- +### 了解更多 -未完待续 +更多配置，详见[配置文件](source_code/配置文件) diff --git a/docs/source_code/UpdateItem.md b/docs/source_code/UpdateItem.md index 7465e5bc..3036628a 100644 --- a/docs/source_code/UpdateItem.md +++ b/docs/source_code/UpdateItem.md @@ -1,3 +1,73 @@ # UpdateItem -未完待续 \ No newline at end of file +UpdateItem用于更新数据，继承至Item，所以使用方式基本与Item一致，下面只说不同之处 + +## 更新逻辑 + +更新逻辑借助了数据库的唯一索引，即插入数据时发现数据已存在，则更新。因此要求数据表必须存在唯一索引，才能使用UpdateItem + +比如将title设置唯一，要求每条数据的title都不能重复 + +![-w781](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/03/16/16158245077159.jpg) + +或联合索引，要求title与url不能同时重复 + +![-w761](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/03/16/16158245648750.jpg) + + +## 指定更新的字段 + +方式1：指定`__update_key__` + +```python +from feapder import UpdateItem + + +class SpiderDataItem(UpdateItem): + + __update_key__ = ["title"] # 更新title字段 + + def __init__(self, *args, **kwargs): + # self.id = None + self.title = None + self.url = None +``` + +方式2：赋值`update_key` + +```python +from feapder import UpdateItem + + +class SpiderDataItem(UpdateItem): + + + def __init__(self, *args, **kwargs): + # self.id = None + self.title = None + self.url = None + +item = SpiderDataItem() +item.update_key = "title" # 支持列表、元组、字符串 +``` + +方式3：将普通的item转为UpdateItem，然后再指定更新的key + +```python +from feapder import Item + + +class SpiderDataItem(Item): + + + def __init__(self, *args, **kwargs): + # self.id = None + self.title = None + self.url = None + +item = SpiderDataItem() +item = item.to_UpdateItem() +item.update_key = "title" +``` + +**推荐方式1，直接改Item类，不用修改爬虫代码** diff --git a/docs/source_code/UserPool.md b/docs/source_code/UserPool.md new file mode 100644 index 00000000..dc850e94 --- /dev/null +++ b/docs/source_code/UserPool.md @@ -0,0 +1,225 @@ +# 用户池使用说明 + +用户池分为三种，使用场景如下 +1. `GuestUserPool`：游客用户池，用于从不需要登录的页面获取cookie +2. `NormalUserPool`：普通用户池，管理大量账号的信息，从需要登录的页面获取cookie +3. `GoldUserPool`：昂贵的用户池，用于账号单价较高，需要限制使用频率、使用时间的场景 + +## GuestUserPool使用方式 +> 环境：redis + +### 导包 +``` +from typing import Optional + +from feapder.network.user_pool import GuestUser +from feapder.network.user_pool import GuestUserPool +``` + +### 默认的用户池 +使用webdriver访问page_url生产cookie +``` +user_pool = GuestUserPool( + "test:user_pool", page_url="https://www.baidu.com" +) + +``` + +### 自定义登录方法 +``` +class CustomGuestUserPool(GuestUserPool): + def login(self) -> Optional[GuestUser]: + # 此处为假数据，正常需通过网站获取cookie + user = GuestUser( + user_agent="xxx", + proxies="yyy", + cookies={"some_key": "some_value{}".format(time.time())}, + ) + return user + +user_pool = CustomGuestUserPool( + "test:user_pool", min_users=10, keep_alive=True +) +``` + +### 获取用户 +无用户时会先登录生产用户 +``` +user = user_pool.get_user(block=True) +print("取到user：", user) +print("cookie：", user.cookies) +print("user_agent：", user.user_agent) +print("proxies：", user.proxies) + +``` +### 删除用户 +``` +user_pool.del_user(user.user_id) +``` + +### 维护一定数量的用户 +run方法需单独起一个进程调用，此进程会常驻，当用户数不足时会及时补充 +``` +user_pool.run() +``` + +## NormalUserPool使用方式 +> 环境：redis、mysql + +### 导包 +``` +from feapder.network.user_pool import NormalUser +from feapder.network.user_pool import NormalUserPool +``` + +### 自定义登录的方法 +``` +class CustomNormalUserPool(NormalUserPool): + def login(self, user: NormalUser) -> NormalUser: + # 此处为假数据，正常需通过登录网站获取cookie + username = user.username + password = user.password + + # 登录获取cookie + cookie = "xxx" + user.cookies = cookie + + return user + +user_pool = CustomNormalUserPool( + "test:user_pool", + table_userbase="test_userbase", + login_retry_times=0, + keep_alive=True, +) +``` +- table_userbase 为mysql里存储用户信息的表，此表会自动创建，需手动录入用户账密 + + 例如： + + ![](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/12/22/16401504359853.jpg) + + + +### 获取用户 +无用户时会先登录生产用户 +``` +user = user_pool.get_user() +print("取到user：", user) +print("cookie：", user.cookies) +print("user_agent：", user.user_agent) +print("proxies：", user.proxies) + +``` +### 删除用户 +``` +user_pool.del_user(user.user_id) +``` + +### 维护一定数量的用户 +run方法需单独起一个进程调用，此进程会常驻，当用户数不足时会及时补充 +``` +user_pool.run() +``` + +### 标记账号被封 +以后不再使用 +``` +user_pool.tag_user_locked(user.user_id) +``` + +## GoldUserPool使用方式 +> 环境：redis + +### 导包 +``` +from feapder.network.user_pool import GoldUser +from feapder.network.user_pool import GoldUserPool +``` + +### 定义用户信息 +``` +users = [ + GoldUser( + username="zhangsan", + password="1234", + max_use_times=10, + use_interval=5, + ), + GoldUser( + username="lisi", + password="1234", + max_use_times=10, + use_interval=5, + login_interval=50, + ), +] +``` + +### 自定义登录的方法 +``` +class CustomGoldUserPool(GoldUserPool): + def login(self, user: GoldUser) -> GoldUser: + # 此处为假数据，正常需通过登录网站获取cookie + username = user.username + password = user.password + + # 登录获取cookie + cookie = "zzzz" + user.cookies = cookie + + return user + +user_pool = CustomGoldUserPool( + "test:user_pool", + users=users, + keep_alive=True, +) +``` + +### 获取用户 +无用户时会先登录生产用户 +``` +user = user_pool.get_user() +print("取到user：", user) +print("cookie：", user.cookies) +print("user_agent：", user.user_agent) +print("proxies：", user.proxies) + +``` + +### 获取指定用户 +无用户时会先登录生产用户 +``` +user = user_pool.get_user(username="用户名") +print("取到user：", user) +print("cookie：", user.cookies) +print("user_agent：", user.user_agent) +print("proxies：", user.proxies) + +``` + +### 删除用户 +``` +user_pool.del_user(user.user_id) +``` + +### 维护一定数量的用户 +run方法需单独起一个进程调用，此进程会常驻，当用户数不足时会及时补充 +``` +user_pool.run() +``` + +### 用户延时使用 +``` +user_pool.delay_use(user.user_id, delay_seconds) +``` + +### 用户独占使用 +某个用户被指定的爬虫独占使用，独占时间内其他爬虫不可使用 +``` +user = user_pool.get_user( + username="用户名", + used_for_spider_name="爬虫名" +) +``` \ No newline at end of file diff --git a/docs/source_code/custom_downloader.md b/docs/source_code/custom_downloader.md new file mode 100644 index 00000000..eb7c8c05 --- /dev/null +++ b/docs/source_code/custom_downloader.md @@ -0,0 +1,300 @@ +# 自定义下载器 + +下载器一共分为三种：**普通下载器**、**支持保持session的下载器**以及**浏览器渲染下载器**。默认已经在框架中内置，setting中的配置如下 + +``` +DOWNLOADER = "feapder.network.downloader.RequestsDownloader" # 请求下载器 +SESSION_DOWNLOADER = "feapder.network.downloader.RequestsSessionDownloader" +RENDER_DOWNLOADER = "feapder.network.downloader.SeleniumDownloader" # 渲染下载器 +``` + +- session下载器当配置中`USE_SESSION = True`时会启用 +- 渲染下载器当使用浏览器下载功能时会启用 + +这些下载器均为插件的形式，我们可以自定义 + +## 自定义普通下载器 + +1. 编写下载器。如在 `xxx-spider/downloader/my_downloader.py `下自定义了如下下载器 + + ``` + import requests + + from feapder.network.downloader.base import Downloader + from feapder.network.response import Response + + class RequestsDownloader(Downloader): + def download(self, request) -> Response: + response = requests.request( + request.method, request.url, **request.requests_kwargs + ) + # 将requests的response转化为feapder的Response 对象，方便后续解析时使用xpath、re等方法 + response = Response(response) + return response + ``` + + 注：这里返回的response对象不强制要求为是feapder的Response。返回值会传到解析函数的response参数里，若返回的是文本，则接收到的也是文本。 + + 但为了代码可读性，建议将返回值转为feapder的Response后再返回。 + + 转feapder的Response的方式有如下几种 + + ``` + # 方式1 + # response参数为reqeusts的response + Response(response) + + # 方式2 + Response.from_text(text="html内容") + ``` + +2. 在settings中指定下载器 + + ``` + DOWNLOADER = "downloader.my_downloader.RequestsDownloader" + ``` + +## 自定义session下载器 + +1. 和普通下载器一样，都是继承`Downloader`，如何保持session，可自定义。代码示例 `xxx-spider/downloader/my_downloader.py ` + + ``` + class RequestsSessionDownloader(Downloader): + session = None + + @property + def _session(self): + if not self.__class__.session: + self.__class__.session = requests.Session() + # pool_connections – 缓存的 urllib3 连接池个数 pool_maxsize – 连接池中保存的最大连接数 + http_adapter = HTTPAdapter(pool_connections=1000, pool_maxsize=1000) + # 任何使用该session会话的 HTTP 请求，只要其 URL 是以给定的前缀开头，该传输适配器就会被使用到。 + self.__class__.session.mount("http", http_adapter) + + return self.__class__.session + + def download(self, request) -> Response: + response = self._session.request( + request.method, request.url, **request.requests_kwargs + ) + response = Response(response) + return response + ``` + +2. 在settings中指定下载器 + + ``` + SESSION_DOWNLOADER = "downloader.my_downloader.RequestsSessionDownloader" + ``` + +注意，这里要配置 `SESSION_DOWNLOADER` + +## 自定义浏览器渲染下载器 + +1. 编写下载器 `xxx-spider/downloader/my_downloader.py ` + +**若浏览器框架本身不支持多线程，但想在多线程中使用，如playwright使用，参考如下：** + +``` +import feapder.setting as setting +import feapder.utils.tools as tools +from feapder.network.downloader.base import RenderDownloader +from feapder.network.response import Response +from feapder.utils.webdriver import WebDriverPool, PlaywrightDriver + + +class MyDownloader(RenderDownloader): + webdriver_pool: WebDriverPool = None + + @property + def _webdriver_pool(self): + if not self.__class__.webdriver_pool: + self.__class__.webdriver_pool = WebDriverPool( + **setting.PLAYWRIGHT, driver_cls=PlaywrightDriver, thread_safe=True + ) + + return self.__class__.webdriver_pool + + def download(self, request) -> Response: + # 代理优先级 自定义 > 配置文件 > 随机 + if request.custom_proxies: + proxy = request.get_proxy() + elif setting.PLAYWRIGHT.get("proxy"): + proxy = setting.PLAYWRIGHT.get("proxy") + else: + proxy = request.get_proxy() + + # user_agent优先级 自定义 > 配置文件 > 随机 + if request.custom_ua: + user_agent = request.get_user_agent() + elif setting.PLAYWRIGHT.get("user_agent"): + user_agent = setting.PLAYWRIGHT.get("user_agent") + else: + user_agent = request.get_user_agent() + + cookies = request.get_cookies() + url = request.url + render_time = request.render_time or setting.PLAYWRIGHT.get("render_time") + wait_until = setting.PLAYWRIGHT.get("wait_until") or "domcontentloaded" + if request.get_params(): + url = tools.joint_url(url, request.get_params()) + + driver: PlaywrightDriver = self._webdriver_pool.get( + user_agent=user_agent, proxy=proxy + ) + try: + if cookies: + driver.url = url + driver.cookies = cookies + driver.page.goto(url, wait_until=wait_until) + + if render_time: + tools.delay_time(render_time) + + html = driver.page.content() + response = Response.from_dict( + { + "url": driver.page.url, + "cookies": driver.cookies, + "_content": html.encode(), + "status_code": 200, + "elapsed": 666, + "headers": { + "User-Agent": driver.user_agent, + "Cookie": tools.cookies2str(driver.cookies), + }, + } + ) + + response.driver = driver + response.browser = driver + return response + except Exception as e: + self._webdriver_pool.remove(driver) + raise e + + def close(self, driver): + if driver: + self._webdriver_pool.remove(driver) + + def put_back(self, driver): + """ + 释放浏览器对象 + """ + self._webdriver_pool.put(driver) + + def close_all(self): + """ + 关闭所有浏览器 + """ + # 不支持 + # self._webdriver_pool.close() + pass +``` + +这里使用了WebDriverPool，参数`thread_safe=True`，即要保证使用时的线程安全，确保同个浏览器对象只能被同一个线程调用 + +**若浏览器框架本身支持多线程，如selenium，则参考如下** + +``` +import feapder.setting as setting +import feapder.utils.tools as tools +from feapder.network.downloader.base import RenderDownloader +from feapder.network.response import Response +from feapder.utils.webdriver import WebDriverPool, SeleniumDriver + + +class MyDownloader(RenderDownloader): + webdriver_pool: WebDriverPool = None + + @property + def _webdriver_pool(self): + if not self.__class__.webdriver_pool: + self.__class__.webdriver_pool = WebDriverPool( + **setting.WEBDRIVER, driver=SeleniumDriver + ) + + return self.__class__.webdriver_pool + + def download(self, request) -> Response: + # 代理优先级 自定义 > 配置文件 > 随机 + if request.custom_proxies: + proxy = request.get_proxy() + elif setting.WEBDRIVER.get("proxy"): + proxy = setting.WEBDRIVER.get("proxy") + else: + proxy = request.get_proxy() + + # user_agent优先级 自定义 > 配置文件 > 随机 + if request.custom_ua: + user_agent = request.get_user_agent() + elif setting.WEBDRIVER.get("user_agent"): + user_agent = setting.WEBDRIVER.get("user_agent") + else: + user_agent = request.get_user_agent() + + cookies = request.get_cookies() + url = request.url + render_time = request.render_time or setting.WEBDRIVER.get("render_time") + if request.get_params(): + url = tools.joint_url(url, request.get_params()) + + browser: SeleniumDriver = self._webdriver_pool.get( + user_agent=user_agent, proxy=proxy + ) + try: + browser.get(url) + if cookies: + browser.cookies = cookies + # 刷新使cookie生效 + browser.get(url) + + if render_time: + tools.delay_time(render_time) + + html = browser.page_source + response = Response.from_dict( + { + "url": browser.current_url, + "cookies": browser.cookies, + "_content": html.encode(), + "status_code": 200, + "elapsed": 666, + "headers": { + "User-Agent": browser.user_agent, + "Cookie": tools.cookies2str(browser.cookies), + }, + } + ) + + response.driver = browser + response.browser = browser + return response + except Exception as e: + self._webdriver_pool.remove(browser) + raise e + + def close(self, driver): + if driver: + self._webdriver_pool.remove(driver) + + def put_back(self, driver): + """ + 释放浏览器对象 + """ + self._webdriver_pool.put(driver) + + def close_all(self): + """ + 关闭所有浏览器 + """ + self._webdriver_pool.close() +``` + +2. 在settings中指定下载器 + +``` +RENDER_DOWNLOADER = "downloader.my_downloader.MyDownloader" +``` + +注，这里要写`RENDER_DOWNLOADER` \ No newline at end of file diff --git a/docs/source_code/dedup.md b/docs/source_code/dedup.md index 53bf089f..3647a673 100644 --- a/docs/source_code/dedup.md +++ b/docs/source_code/dedup.md @@ -59,7 +59,7 @@ def test_MemoryFilter(): ```python from feapder.dedup import Dedup - def test_BloomFilter(): +def test_BloomFilter(): dedup = Dedup(Dedup.BloomFilter, redis_url="redis://@localhost:6379/0") # 逐条去重 @@ -70,7 +70,7 @@ from feapder.dedup import Dedup assert dedup.add(datas) == [1, 1] assert dedup.get(datas) == [1, 1] ``` - + ## 过滤数据 Dedup可以通过如下方法，过滤掉已存在的数据 @@ -97,12 +97,12 @@ def test_filter(): - **filter_type**：去重类型，支持BloomFilter、MemoryFilter、ExpireFilter三种 - **redis_url**不是必须传递的，若项目中存在setting.py文件，且已配置redis连接方式，则可以不传递redis_url - ![-w294](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/03/07/16151133801599.jpg?x-oss-process=style/markdown-media) - + ![-w294](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/03/07/16151133801599.jpg) + ``` import feapder from feapder.dedup import Dedup - + class TestSpider(feapder.Spider): def __init__(self, *args, **kwargs): self.dedup = Dedup() # 默认是永久去重 @@ -110,11 +110,27 @@ def test_filter(): - **name**: 过滤器名称 该名称会默认以dedup作为前缀 `dedup:expire_set:[name]`或`dedup:bloomfilter:[name]`。 默认ExpireFilter name=过期时间，BloomFilter name=`dedup:bloomfilter:bloomfilter` - ![-w499](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/03/07/16151136442498.jpg?x-oss-process=style/markdown-media) - + ![-w499](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/03/07/16151136442498.jpg) + 若对不同数据源去重，可通过name参数来指定不同去重库 - **absolute_name**：过滤器绝对名称 不会加dedup前缀 - **expire_time**：ExpireFilter的过期时间 单位为秒，其他两种过滤器不用指定 - **error_rate**：BloomFilter/MemoryFilter的误判率 默认为0.00001 - **to_md5**：去重前是否将数据转为MD5，默认是 + +## 爬虫中使用 + +框架支持对请求和入库的数据进行去重，仅需要在[配置文件](source_code/配置文件)中进行配置即可 + +```python +ITEM_FILTER_ENABLE = False # item 去重 +REQUEST_FILTER_ENABLE = False # request 去重 +``` + +或者可以直接导入此去重模块使用 + +```python +from feapder.dedup import Dedup +``` + diff --git a/docs/source_code/logger.md b/docs/source_code/logger.md new file mode 100644 index 00000000..edeaeb9b --- /dev/null +++ b/docs/source_code/logger.md @@ -0,0 +1,40 @@ +# 日志配置及使用 + +## 日志配置 + +见配置文件，相关配置如下： + +```python +LOG_NAME = os.path.basename(os.getcwd()) +LOG_PATH = "log/%s.log" % LOG_NAME # log存储路径 +LOG_LEVEL = "DEBUG" +LOG_COLOR = True # 是否带有颜色 +LOG_IS_WRITE_TO_CONSOLE = True # 是否打印到控制台 +LOG_IS_WRITE_TO_FILE = False # 是否写文件 +LOG_MODE = "w" # 写文件的模式 +LOG_MAX_BYTES = 10 * 1024 * 1024 # 每个日志文件的最大字节数 +LOG_BACKUP_COUNT = 20 # 日志文件保留数量 +LOG_ENCODING = "utf8" # 日志文件编码 +OTHERS_LOG_LEVAL = "ERROR" # 第三方库的log等级 +``` + +框架屏蔽了requests、selenium等一些第三方库的日志，OTHERS_LOG_LEVAL是用来控制这些第三库日志等级的。 + +## 使用日志工具 + + +```python +from feapder.utils.log import log + +log.debug("xxx") +log.info("xxx") +log.warning("xxx") +log.error("xxx") +log.critical("xxx") +``` + +默认是带有颜色的日志： + +![-w583](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/08/06/16282311862710.jpg) + +日志等级：CRITICAL > ERROR > WARNING > INFO > DEBUG diff --git a/docs/source_code/pipeline.md b/docs/source_code/pipeline.md new file mode 100644 index 00000000..6a04dbf1 --- /dev/null +++ b/docs/source_code/pipeline.md @@ -0,0 +1,91 @@ +# Pipeline + +Pipeline是数据入库时流经的管道，用户可自定义，以便对接其他数据库。 + +框架已内置mysql、mongo、csv管道，其他管道作为扩展方式提供，可从[feapder_pipelines](https://github.com/Boris-code/feapder_pipelines)项目中按需安装 + +项目地址：https://github.com/Boris-code/feapder_pipelines + +## 选择内置的pipeline + +在配置文件 `setting.py` 中的 `ITEM_PIPELINES` 中启用： + +```python +ITEM_PIPELINES = [ + "feapder.pipelines.mysql_pipeline.MysqlPipeline", + # "feapder.pipelines.mongo_pipeline.MongoPipeline", + # "feapder.pipelines.csv_pipeline.CsvPipeline", + # "feapder.pipelines.console_pipeline.ConsolePipeline", +] +``` + +然后 爬虫中`yield`的`item`会流经选择的pipeline自动存储 + +## 自定义pipeline + +注：item会被聚合成多条一起流经pipeline，方便批量入库 + +### 1. 编写pipeline + +```python +from feapder.pipelines import BasePipeline +from typing import Dict, List, Tuple + + +class Pipeline(BasePipeline): + """ + pipeline 是单线程的，批量保存数据的操作，不建议在这里写网络请求代码，如下载图片等 + """ + + def save_items(self, table, items: List[Dict]) -> bool: + """ + 保存数据 + Args: + table: 表名 + items: 数据，[{},{},...] + + Returns: 是否保存成功 True / False + 若False，不会将本批数据入到去重库，以便再次入库 + + """ + + print("自定义pipeline， 保存数据 >>>>", table, items) + + return True + + def update_items(self, table, items: List[Dict], update_keys=Tuple) -> bool: + """ + 更新数据, 与UpdateItem配合使用，若爬虫中没使用UpdateItem，则可不实现此接口 + Args: + table: 表名 + items: 数据，[{},{},...] + update_keys: 更新的字段, 如 ("title", "publish_time") + + Returns: 是否更新成功 True / False + 若False，不会将本批数据入到去重库，以便再次入库 + + """ + + print("自定义pipeline， 更新数据 >>>>", table, items, update_keys) + + return True +``` + +`Pipeline`需继承`BasePipeline`，类名和存放位置随意，需要实现`save_items`接口。一定要有返回值，返回`False`表示数据没保存成功，会触发重试逻辑 + +`update_items`接口与`UpdateItem`配合使用，更新数据时使用，若爬虫中没使用UpdateItem，则可不实现此接口 + +### 2. 编写配置文件 + +```python +# 数据入库的pipeline，支持多个 +ITEM_PIPELINES = [ + "pipeline.Pipeline" +] +``` + +将编写好的pipeline配置进来，值为类的模块路径，需要指定到具体的类名 + +## 示例 + +地址：https://github.com/Boris-code/feapder/tree/master/tests/test-pipeline diff --git a/docs/source_code/proxy.md b/docs/source_code/proxy.md new file mode 100644 index 00000000..de87845a --- /dev/null +++ b/docs/source_code/proxy.md @@ -0,0 +1,104 @@ +# 代理使用说明 + +代理使用有三种方式 +1. 使用框架内置代理池 +2. 自定义代理池 +3. 请求中直接指定 + +## 方式1. 使用框架内置代理池 + +### 配置代理 + +在配置文件中配置代理提取接口 + +```python +# 设置代理 +PROXY_EXTRACT_API = None # 代理提取API ，返回的代理分割符为\r\n +PROXY_ENABLE = True +PROXY_MAX_FAILED_TIMES = 5 # 代理最大失败次数，超过则不使用，自动删除 +``` + +要求API返回的代理格式为使用 /r/n 分隔： + +``` +ip:port +ip:port +ip:port +``` + +这样feapder在请求时会自动随机使用上面的代理请求了 + +## 管理代理 + +1. 删除代理（默认是请求异常连续5次，再删除代理） + + 例如在发生异常时删除代理 + + ```python + import feapder + class TestProxy(feapder.AirSpider): + def start_requests(self): + yield feapder.Request("https://www.baidu.com") + + def parse(self, request, response): + print(response) + + def exception_request(self, request, response): + request.del_proxy() + + ``` + +## 方式2. 自定义代理池 + +1. 编写代理池：例如在你的项目下创建个my_proxypool.py，实现下面的函数 + + ```python + from feapder.network.proxy_pool import BaseProxyPool + + class MyProxyPool(BaseProxyPool): + def get_proxy(self): + """ + 获取代理 + Returns: + {"http": "xxx", "https": "xxx"} + """ + pass + + def del_proxy(self, proxy): + """ + @summary: 删除代理 + --------- + @param proxy: xxx + """ + pass + ``` + +3. 修改setting的代理配置 + + ``` + PROXY_POOL = "my_proxypool.MyProxyPool" # 代理池 + ``` + + 将编写好的代理池配置进来，值为类的模块路径，需要指定到具体的类名 + + + +## 方式3. 不使用代理池，直接给请求指定代理 + +直接给request.proxies赋值即可，例如在下载中间件里使用 + +```python +import feapder + +class TestProxy(feapder.AirSpider): + def start_requests(self): + yield feapder.Request("https://www.baidu.com") + + def download_midware(self, request): + # 这里使用代理使用即可 + request.proxies = {"https": "https://ip:port", "http": "http://ip:port"} + return request + + def parse(self, request, response): + print(response) +``` \ No newline at end of file diff --git a/docs/source_code/tools.md b/docs/source_code/tools.md index dfaf3b67..ebd79173 100644 --- a/docs/source_code/tools.md +++ b/docs/source_code/tools.md @@ -1,4 +1,17 @@ # tools -未完待续 +`feapder.utils.tools`里封装了爬虫中常用的函数，目前共计**129**个，可通过阅读源码了解使用 + +## 举例 + +### 时间格式化 + +```python +from feapder.utils import tools + +time = "昨天" + +date = tools.format_time(time) +assert date == "2021-03-15 00:00:00" +``` diff --git "a/docs/source_code/\346\212\245\350\255\246\345\217\212\347\233\221\346\216\247.md" "b/docs/source_code/\346\212\245\350\255\246\345\217\212\347\233\221\346\216\247.md" new file mode 100644 index 00000000..87dbc695 --- /dev/null +++ "b/docs/source_code/\346\212\245\350\255\246\345\217\212\347\233\221\346\216\247.md" @@ -0,0 +1,121 @@ +# 报警及监控 + +支持钉钉、飞书、企业微信、邮件报警 + +## 钉钉报警 + +条件：需要有钉钉群，需要获取钉钉机器人的Webhook地址 + +获取方式参考官方文档：https://developers.dingtalk.com/document/app/custom-robot-access + +安全设置选择自定义关键词，填入**feapder** + +![-w547](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/03/27/16167753030324.jpg) + +或使用加签方式，然后在setting中设置密钥 + +相关配置： + +```python +# 钉钉报警 +DINGDING_WARNING_URL = "" # 钉钉机器人api +DINGDING_WARNING_PHONE = "" # 报警人 支持列表，可指定多个 +DINGDING_WARNING_ALL = False # 是否提示所有人， 默认为False +DINGDING_WARNING_SECRET = None # 加签密钥 +``` + +## 企业微信报警 + +条件：需要企业微信群，并获取企业微信机器人的Webhook地址 + +获取方式：https://weibanzhushou.com/blog/330 + +报警简介： + +- 仅支持文本模式 +- 当用户手机号码为空字符串或`WECHAT_WARNING_ALL`为`True`时将会`@全体成员` + + +相关设置： + +```python +# 企业微信报警 +WECHAT_WARNING_URL = "" # 企业微信机器人api +WECHAT_WARNING_PHONE = "" # 报警人 将会在群内@此人, 支持列表，可指定多人 +WECHAT_WARNING_ALL = False # 是否提示所有人， 默认为False +``` + +## 飞书报警 + +可参考文档设置机器人：https://open.feishu.cn/document/ukTMukTMukTM/ucTM5YjL3ETO24yNxkjN#e1cdee9f + +然后在feapder的setting文件中修改如下配置 + +``` +FEISHU_WARNING_URL = "" # 飞书机器人api +FEISHU_WARNING_USER = None # 报警人 {"open_id":"ou_xxxxx", "name":"xxxx"} 或 [{"open_id":"ou_xxxxx", "name":"xxxx"}] +FEISHU_WARNING_ALL = False # 是否提示所有人， 默认为False +``` + +## 邮件报警 + +相关配置： + +``` +# 邮件报警 +EMAIL_SENDER = "" # 发件人 +EMAIL_PASSWORD = "" # 授权码 +EMAIL_RECEIVER = "" # 收件人 支持列表，可指定多个 +``` + +邮件报警目前支持163邮箱作为发送者，`EMAIL_SENDER`为邮箱账号，如`feapder@163.com`, `EMAIL_PASSWORD`为授权码，不是登录密码，获取授权码的流程如下： + +1. 设置 -> POP3/SMTP/IMAP + + ![-w258](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/03/27/16167719328720.jpg) + +2. 开启SMTP服务 + + ![-w444](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/03/27/16167719490656.jpg) + + 开启后，会弹出授权码，该授权码即为EMAIL_PASSWORD + +3. 设置反垃圾规则为高级 + + ![-w1112](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/03/27/16167719655644.jpg) + +4. 将本邮箱账号添加到白名单中 + +## Qmsg酱报警 + +Qmsg酱是一个QQ消息推送机器人，用来通知自己消息的免费服务。 + +可以参考文档：https://qmsg.zendee.cn/docs/api/ + +```python +# QMSG报警 +QMSG_WARNING_URL = "" # qmsg机器人api +QMSG_WARNING_QQ = "" # 指定要接收消息的QQ号或者QQ群。多个以英文逗号分割，例如：12345,12346,支持列表，可指定多人 +QMSG_WARNING_BOT = "" # 机器人的QQ号 +``` + + +## 报警间隔及报警级别 + +框架会对相同的报警进行过滤，防止刷屏，默认的报警时间间隔为1小时，可通过以下配置修改： + +```python +WARNING_INTERVAL = 3600 # 相同报警的报警时间间隔，防止刷屏 +WARNING_LEVEL = "DEBUG" # 报警级别， DEBUG / ERROR +``` + +DEBUG级别的报警包含一些运行信息，ERROR级别的报警都是有问题的报警，需要及时处理 + + +## 可视化监控 + +支持对爬虫运行情况进行监控，除了数据监控和请求监控外，用户还可自定义监控内容，详情参考[自定义监控](source_code/监控打点?id=自定义监控) + +![](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/09/14/16316112326191.jpg) + +需 feapder>=1.6.6, 需配合feaplat爬虫管理平台 \ No newline at end of file diff --git "a/docs/source_code/\346\265\217\350\247\210\345\231\250\346\270\262\346\237\223-Playwright.md" "b/docs/source_code/\346\265\217\350\247\210\345\231\250\346\270\262\346\237\223-Playwright.md" new file mode 100644 index 00000000..8483b126 --- /dev/null +++ "b/docs/source_code/\346\265\217\350\247\210\345\231\250\346\270\262\346\237\223-Playwright.md" @@ -0,0 +1,258 @@ +# 浏览器渲染-Playwright + +采集动态页面时（Ajax渲染的页面），常用的有两种方案。一种是找接口拼参数，这种方式比较复杂但效率高，需要一定的爬虫功底；另外一种是采用浏览器渲染的方式，直接获取源码，简单方便 + +框架支持playwright渲染下载，每个线程持有一个playwright实例 + + +## 使用方式： + +1. 修改配置文件的渲染下载器： + + ``` + RENDER_DOWNLOADER="feapder.network.downloader.PlaywrightDownloader" + ``` +2. 使用 + + ```python + def start_requests(self): + yield feapder.Request("https://news.qq.com/", render=True) + ``` + +在返回的Request中传递`render=True`即可 + +框架支持`chromium`、`firefox`、`webkit` 三种浏览器渲染，可通过[配置文件](source_code/配置文件)进行配置。相关配置如下： + +```python +PLAYWRIGHT = dict( + user_agent=None, # 字符串 或 无参函数，返回值为user_agent + proxy=None, # xxx.xxx.xxx.xxx:xxxx 或 无参函数，返回值为代理地址 + headless=False, # 是否为无头浏览器 + driver_type="chromium", # chromium、firefox、webkit + timeout=30, # 请求超时时间 + window_size=(1024, 800), # 窗口大小 + executable_path=None, # 浏览器路径，默认为默认路径 + download_path=None, # 下载文件的路径 + render_time=0, # 渲染时长，即打开网页等待指定时间后再获取源码 + wait_until="networkidle", # 等待页面加载完成的事件,可选值："commit", "domcontentloaded", "load", "networkidle" + use_stealth_js=False, # 使用stealth.min.js隐藏浏览器特征 + page_on_event_callback=None, # page.on() 事件的回调 如 page_on_event_callback={"dialog": lambda dialog: dialog.accept()} + storage_state_path=None, # 保存浏览器状态的路径 + url_regexes=None, # 拦截接口，支持正则，数组类型 + save_all=False, # 是否保存所有拦截的接口, 配合url_regexes使用，为False时只保存最后一次拦截的接口 +) +``` + + - `feapder.Request` 也支持`render_time`参数， 优先级大于配置文件中的`render_time` + + - 代理使用优先级：`feapder.Request`指定的代理 > 配置文件中的`PROXY_EXTRACT_API` > webdriver配置文件中的`proxy` + + - user_agent使用优先级：`feapder.Request`指定的header里的`User-Agent` > 框架随机的`User-Agent` > webdriver配置文件中的`user_agent` + +## 设置User-Agent + +> 每次生成一个新的浏览器实例时生效 + +### 方式1： + +通过配置文件的 `user_agent` 参数设置 + +### 方式2： + +通过 `feapder.Request`携带，优先级大于配置文件, 如： + +```python +def download_midware(self, request): + request.headers = { + "User-Agent": "xxxxxxxx" + } + return request +``` + +## 设置代理 + +> 每次生成一个新的浏览器实例时生效 + +### 方式1： + +通过配置文件的 `proxy` 参数设置 + +### 方式2： + +通过 `feapder.Request`携带，优先级大于配置文件, 如： + +```python +def download_midware(self, request): + request.proxies = { + "https": "https://xxx.xxx.xxx.xxx:xxxx" + } + return request +``` + +## 设置Cookie + +通过 `feapder.Request`携带，如： + +```python +def download_midware(self, request): + request.headers = { + "Cookie": "key=value; key2=value2" + } + return request +``` + +或者 + +```python +def download_midware(self, request): + request.cookies = { + "key": "value", + "key2": "value2", + } + return request +``` + +或者 + +```python +def download_midware(self, request): + request.cookies = [ + { + "domain": "xxx", + "name": "xxx", + "value": "xxx", + "expirationDate": "xxx" + }, + ] + return request +``` + +## 拦截数据示例 + +> 注意：主函数使用run方法运行，不能使用start + +```python +from playwright.sync_api import Response +from feapder.utils.webdriver import ( + PlaywrightDriver, + InterceptResponse, + InterceptRequest, +) + +import feapder + + +def on_response(response: Response): + print(response.url) + + +class TestPlaywright(feapder.AirSpider): + __custom_setting__ = dict( + RENDER_DOWNLOADER="feapder.network.downloader.PlaywrightDownloader", + PLAYWRIGHT=dict( + user_agent=None, # 字符串 或 无参函数，返回值为user_agent + proxy=None, # xxx.xxx.xxx.xxx:xxxx 或 无参函数，返回值为代理地址 + headless=False, # 是否为无头浏览器 + driver_type="chromium", # chromium、firefox、webkit + timeout=30, # 请求超时时间 + window_size=(1024, 800), # 窗口大小 + executable_path=None, # 浏览器路径，默认为默认路径 + download_path=None, # 下载文件的路径 + render_time=0, # 渲染时长，即打开网页等待指定时间后再获取源码 + wait_until="networkidle", # 等待页面加载完成的事件,可选值："commit", "domcontentloaded", "load", "networkidle" + use_stealth_js=False, # 使用stealth.min.js隐藏浏览器特征 + # page_on_event_callback=dict(response=on_response), # 监听response事件 + # page.on() 事件的回调 如 page_on_event_callback={"dialog": lambda dialog: dialog.accept()} + storage_state_path=None, # 保存浏览器状态的路径 + url_regexes=["wallpaper/list"], # 拦截接口，支持正则，数组类型 + save_all=True, # 是否保存所有拦截的接口 + ), + ) + + def start_requests(self): + yield feapder.Request( + "http://www.soutushenqi.com/image/search/?searchWord=%E6%A0%91%E5%8F%B6", + render=True, + ) + + def parse(self, reqeust, response): + driver: PlaywrightDriver = response.driver + + intercept_response: InterceptResponse = driver.get_response("wallpaper/list") + intercept_request: InterceptRequest = intercept_response.request + + req_url = intercept_request.url + req_header = intercept_request.headers + req_data = intercept_request.data + print("请求url", req_url) + print("请求header", req_header) + print("请求data", req_data) + + data = driver.get_json("wallpaper/list") + print("接口返回的数据", data) + + print("------ 测试save_all=True ------- ") + + # 测试save_all=True + all_intercept_response: list = driver.get_all_response("wallpaper/list") + for intercept_response in all_intercept_response: + intercept_request: InterceptRequest = intercept_response.request + req_url = intercept_request.url + req_header = intercept_request.headers + req_data = intercept_request.data + print("请求url", req_url) + print("请求header", req_header) + print("请求data", req_data) + + all_intercept_json = driver.get_all_json("wallpaper/list") + for intercept_json in all_intercept_json: + print("接口返回的数据", intercept_json) + + # 千万别忘了 + driver.clear_cache() + + +if __name__ == "__main__": + TestPlaywright(thread_count=1).run() +``` +可通过配置的`page_on_event_callback`参数自定义事件的回调，如设置`on_response`的事件回调，亦可直接使用`url_regexes`设置拦截的接口 + +## 操作浏览器对象示例 + +> 注意：主函数使用run方法运行，不能使用start + +```python +import time + +from playwright.sync_api import Page + +import feapder +from feapder.utils.webdriver import PlaywrightDriver + + +class TestPlaywright(feapder.AirSpider): + __custom_setting__ = dict( + RENDER_DOWNLOADER="feapder.network.downloader.PlaywrightDownloader", + ) + + def start_requests(self): + yield feapder.Request("https://www.baidu.com", render=True) + + def parse(self, reqeust, response): + driver: PlaywrightDriver = response.driver + page: Page = driver.page + + page.type("#kw", "feapder") + page.click("#su") + page.wait_for_load_state("networkidle") + time.sleep(1) + + html = page.content() + response.text = html # 使response加载最新的页面 + for data_container in response.xpath("//div[@class='c-container']"): + print(data_container.xpath("string(.//h3)").extract_first()) + + +if __name__ == "__main__": + TestPlaywright(thread_count=1).run() +``` \ No newline at end of file diff --git "a/docs/source_code/\346\265\217\350\247\210\345\231\250\346\270\262\346\237\223-Selenium.md" "b/docs/source_code/\346\265\217\350\247\210\345\231\250\346\270\262\346\237\223-Selenium.md" new file mode 100644 index 00000000..089f9537 --- /dev/null +++ "b/docs/source_code/\346\265\217\350\247\210\345\231\250\346\270\262\346\237\223-Selenium.md" @@ -0,0 +1,281 @@ +# 浏览器渲染-Selenium + +采集动态页面时（Ajax渲染的页面），常用的有两种方案。一种是找接口拼参数，这种方式比较复杂但效率高，需要一定的爬虫功底；另外一种是采用浏览器渲染的方式，直接获取源码，简单方便 + +框架内置一个浏览器渲染池，默认的池子大小为1，请求时重复利用浏览器实例，只有当代理失效请求异常时，才会销毁、创建一个新的浏览器实例 + +内置浏览器渲染支持 **CHROME**、**EDGE**、**PHANTOMJS**、**FIREFOX** + +## 使用方式： + +```python +def start_requests(self): + yield feapder.Request("https://news.qq.com/", render=True) +``` +在返回的Request中传递`render=True`即可 + +框架支持`CHROME`、`EDGE`、`PHANTOMJS`、`FIREFOX` 三种浏览器渲染，可通过[配置文件](source_code/配置文件)进行配置。相关配置如下： + +```python +# 浏览器渲染 +WEBDRIVER = dict( + pool_size=1, # 浏览器的数量 + load_images=True, # 是否加载图片 + user_agent=None, # 字符串 或 无参函数，返回值为user_agent + proxy=None, # xxx.xxx.xxx.xxx:xxxx 或 无参函数，返回值为代理地址 + headless=False, # 是否为无头浏览器 + driver_type="CHROME", # CHROME、EDGE、PHANTOMJS、FIREFOX + timeout=30, # 请求超时时间 + window_size=(1024, 800), # 窗口大小 + executable_path=None, # 浏览器路径，默认为默认路径 + render_time=0, # 渲染时长，即打开网页等待指定时间后再获取源码 + custom_argument=["--ignore-certificate-errors"], # 自定义浏览器渲染参数 + xhr_url_regexes=None, # 拦截xhr接口，支持正则，数组类型 + auto_install_driver=False, # 自动下载浏览器驱动 支持chrome 和 firefox +) +``` + + - `feapder.Request` 也支持`render_time`参数， 优先级大于配置文件中的`render_time` + + - 代理使用优先级：`feapder.Request`指定的代理 > 配置文件中的`PROXY_EXTRACT_API` > webdriver配置文件中的`proxy` + + - user_agent使用优先级：`feapder.Request`指定的header里的`User-Agent` > 框架随机的`User-Agent` > webdriver配置文件中的`user_agent` + +## 设置User-Agent + +> 每次生成一个新的浏览器实例时生效 + +### 方式1： + +通过配置文件的 `user_agent` 参数设置 + +### 方式2： + +通过 `feapder.Request`携带，优先级大于配置文件, 如： + +```python +def download_midware(self, request): + request.headers = { + "User-Agent": "xxxxxxxx" + } + return request +``` + +## 设置代理 + +> 每次生成一个新的浏览器实例时生效 + +### 方式1： + +通过配置文件的 `proxy` 参数设置 + +### 方式2： + +通过 `feapder.Request`携带，优先级大于配置文件, 如： + +```python +def download_midware(self, request): + request.proxies = { + "https": "https://xxx.xxx.xxx.xxx:xxxx" + } + return request +``` + +## 设置Cookie + +通过 `feapder.Request`携带，如： + +```python +def download_midware(self, request): + request.headers = { + "Cookie": "key=value; key2=value2" + } + return request +``` + +或者 + +```python +def download_midware(self, request): + request.cookies = { + "key": "value", + "key2": "value2", + } + return request +``` + +或者 + +```python +def download_midware(self, request): + request.cookies = [ + { + "domain": "xxx", + "name": "xxx", + "value": "xxx", + "expirationDate": "xxx" + }, + ] + return request +``` + +## 操作浏览器对象 + +通过 `response.browser` 获取浏览器对象 + +代码示例：请求百度，搜索feapder + +```python +import time + +import feapder +from feapder.utils.webdriver import WebDriver + + +class TestRender(feapder.AirSpider): + def start_requests(self): + yield feapder.Request("http://www.baidu.com", render=True) + + def parse(self, request, response): + browser: WebDriver = response.browser + browser.find_element_by_id("kw").send_keys("feapder") + browser.find_element_by_id("su").click() + time.sleep(5) + print(browser.page_source) + + # response也是可以正常使用的 + # response.xpath("//title") + + # 若有滚动，可通过如下方式更新response，使其加载滚动后的内容 + # response.text = browser.page_source + + +if __name__ == "__main__": + TestRender().start() + +``` + +## 拦截xhr数据 + +### 设置拦截规则 + +```python +WEBDRIVER = dict( + ... + xhr_url_regexes=[ + "接口1正则", + "接口2正则", + ] +) +``` + +### 获取数据 + +```python +browser: WebDriver = response.browser +text = browser.xhr_text("接口1正则") +``` + +### 获取json格式数据 + +```python +browser: WebDriver = response.browser +data = browser.xhr_json("接口1正则") +``` + +### 获取response + +```python +browser: WebDriver = response.browser +xhr_response = browser.xhr_response("接口1正则") +print("请求接口", xhr_response.request.url) +print("请求头", xhr_response.request.headers) +print("请求体", xhr_response.request.data) +print("返回头", xhr_response.headers) +print("返回地址", xhr_response.url) +print("返回内容", xhr_response.content) +``` + + +### 示例： + +需求： +![](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/12/30/16408610725756.jpg) + +代码： + +```python +import time + +import feapder +from feapder.utils.webdriver import WebDriver + + +class TestRender(feapder.AirSpider): + __custom_setting__ = dict( + WEBDRIVER=dict( + pool_size=1, # 浏览器的数量 + load_images=True, # 是否加载图片 + user_agent=None, # 字符串 或 无参函数，返回值为user_agent + proxy=None, # xxx.xxx.xxx.xxx:xxxx 或 无参函数，返回值为代理地址 + headless=False, # 是否为无头浏览器 + driver_type="CHROME", # CHROME、EDGE、PHANTOMJS、FIREFOX + timeout=30, # 请求超时时间 + window_size=(1024, 800), # 窗口大小 + executable_path=None, # 浏览器路径，默认为默认路径 + render_time=0, # 渲染时长，即打开网页等待指定时间后再获取源码 + custom_argument=["--ignore-certificate-errors"], # 自定义浏览器渲染参数 + xhr_url_regexes=[ + "/ad", + ], # 拦截 http://www.spidertools.cn/spidertools/ad 接口 + ) + ) + + def start_requests(self): + yield feapder.Request("http://www.spidertools.cn", render=True) + + def parse(self, request, response): + browser: WebDriver = response.browser + time.sleep(3) + + # 获取接口数据 文本类型 + ad = browser.xhr_text("/ad") + print(ad) + + # 获取接口数据 转成json，本例因为返回的接口是文本，所以不转了 + # browser.xhr_json("/ad") + + xhr_response = browser.xhr_response("/ad") + print("请求接口", xhr_response.request.url) + # 请求头目前获取的不完整 + print("请求头", xhr_response.request.headers) + print("请求体", xhr_response.request.data) + print("返回头", xhr_response.headers) + print("返回地址", xhr_response.url) + print("返回内容", xhr_response.content) + + +if __name__ == "__main__": + TestRender().start() + +``` + +## 驱动版本自动适配 + +```python +WEBDRIVER = dict( + ... + auto_install_driver=True +) +``` + +即浏览器渲染相关配置 `auto_install_driver` 设置True，让其自动对比驱动版本，版本不符或驱动不存在时自动下载 + +## 关闭当前浏览器 + +```python +def parse(self, request, response): + response.close_browser(request) +``` + +关闭会自动重开一个新的浏览器实例 diff --git "a/docs/source_code/\347\233\221\346\216\247\346\211\223\347\202\271.md" "b/docs/source_code/\347\233\221\346\216\247\346\211\223\347\202\271.md" new file mode 100644 index 00000000..dd76890a --- /dev/null +++ "b/docs/source_code/\347\233\221\346\216\247\346\211\223\347\202\271.md" @@ -0,0 +1,94 @@ +# 监控打点 + +需配合爬虫管理系统 **feaplat** + +监控数据默认保留180天，滚动删除 + +## 爬虫中使用 + +> 需feapder>=1.6.6 + +feapder内置了监控打点，只需要部署到feaplat爬虫管理系统即可实现对请求和数据监控 + +![](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/09/14/16316112326191.jpg) + +- 注意使用 `yield item` 的方式入库的数据，才能看到数据监控的指标，图表的title是表名，折线图展示了每个字段是否有值的情况以及数据总量（total count） + +- document为下载情况 + +若想监控些其他的指标，参考自定义监控： + + +## 自定义监控 + +举例：编写`test_metrics.py`代码如下： + +```python +from feapder.utils import metrics + +# 初始化打点系统 +metrics.init() + +metrics.emit_counter("key", count=1, classify="test") + +metrics.close() +``` + +部署到feaplat： + +![](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/09/13/16315065474223.jpg) + +查看监控： + +![](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/09/13/16315067391666.jpg) + +再来解释下 +``` +metrics.emit_counter("key", count=1, classify="test") +``` +- key 对应上图中的折线 +- count 对应上图中的点数 +- classify 对应上图中的图表标题 + +若代码如下： +```python +from feapder.utils import metrics + +# 初始化打点系统 +metrics.init() + +metrics.emit_counter("key", count=1, classify="test") +metrics.emit_counter("key2", count=1, classify="test") +metrics.emit_counter("key3", count=1, classify="test") + +metrics.emit_counter("哈哈", count=1, classify="test2") + +metrics.close() +``` + +应该生成两张图表，第一个图表3条折线，实际生成如下： + +![](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/09/13/16315071385604.jpg) + + +如在feapder爬虫中使用，示例如下： + +```python +import feapder +from feapder.utils import metrics + + +class TestSpider(feapder.AirSpider): + def start_requests(self): + yield feapder.Request("https://www.baidu.com") + + def parse(self, request, response): + # 自定义监控 + metrics.emit_counter("success", count=1, classify="自定义的监控指标") + + +if __name__ == "__main__": + TestSpider().start() +``` + +我们只需要导包，然后`metrics.emit_counter`即可，不需要关心 `metrics.init`和`metrics.close`， 若在scrapy或其他python脚本中使用，必须调用`metrics.init`和`metrics.close` diff --git "a/docs/source_code/\351\205\215\347\275\256\346\226\207\344\273\266.md" "b/docs/source_code/\351\205\215\347\275\256\346\226\207\344\273\266.md" index 9e4bf4f2..e22be333 100644 --- "a/docs/source_code/\351\205\215\347\275\256\346\226\207\344\273\266.md" +++ "b/docs/source_code/\351\205\215\347\275\256\346\226\207\344\273\266.md" @@ -5,114 +5,207 @@ ## 全局配置 在项目的根目录下创建 setting.py, 全部的参数如下，按需选择 -![-w378](http://markdown-media.oss-cn-beijing.aliyuncs.com/2020/12/30/16093189206589.jpg?x-oss-process=style/markdown-media) - +![-w378](http://markdown-media.oss-cn-beijing.aliyuncs.com/2020/12/30/16093189206589.jpg) + +```python +# -*- coding: utf-8 -*- +"""爬虫配置文件""" +# import os +# import sys +# +# # MYSQL +# MYSQL_IP = "localhost" +# MYSQL_PORT = 3306 +# MYSQL_DB = "" +# MYSQL_USER_NAME = "" +# MYSQL_USER_PASS = "" +# +# # MONGODB +# MONGO_IP = "localhost" +# MONGO_PORT = 27017 +# MONGO_DB = "" +# MONGO_USER_NAME = "" +# MONGO_USER_PASS = "" +# +# # REDIS +# # ip:port 多个可写为列表或者逗号隔开 如 ip1:port1,ip2:port2 或 ["ip1:port1", "ip2:port2"] +# REDISDB_IP_PORTS = "localhost:6379" +# REDISDB_USER_PASS = "" +# REDISDB_DB = 0 +# # 适用于redis哨兵模式 +# REDISDB_SERVICE_NAME = "" +# +# # 数据入库的pipeline，可自定义，默认MysqlPipeline +# ITEM_PIPELINES = [ +# "feapder.pipelines.mysql_pipeline.MysqlPipeline", +# # "feapder.pipelines.mongo_pipeline.MongoPipeline", +# # "feapder.pipelines.console_pipeline.ConsolePipeline", +# ] +# EXPORT_DATA_MAX_FAILED_TIMES = 10 # 导出数据时最大的失败次数，包括保存和更新，超过这个次数报警 +# EXPORT_DATA_MAX_RETRY_TIMES = 10 # 导出数据时最大的重试次数，包括保存和更新，超过这个次数则放弃重试 +# +# # 爬虫相关 +# # COLLECTOR +# COLLECTOR_TASK_COUNT = 32 # 每次获取任务数量，追求速度推荐32 +# +# # SPIDER +# SPIDER_THREAD_COUNT = 1 # 爬虫并发数，追求速度推荐32 +# # 下载时间间隔 单位秒。 支持随机 如 SPIDER_SLEEP_TIME = [2, 5] 则间隔为 2~5秒之间的随机数，包含2和5 +# SPIDER_SLEEP_TIME = 0 +# SPIDER_MAX_RETRY_TIMES = 10 # 每个请求最大重试次数 +# KEEP_ALIVE = False # 爬虫是否常驻 + +# 下载 +# DOWNLOADER = "feapder.network.downloader.RequestsDownloader" +# SESSION_DOWNLOADER = "feapder.network.downloader.RequestsSessionDownloader" +# RENDER_DOWNLOADER = "feapder.network.downloader.SeleniumDownloader" +# # RENDER_DOWNLOADER="feapder.network.downloader.PlaywrightDownloader", +# MAKE_ABSOLUTE_LINKS = True # 自动转成绝对连接 + +# # 浏览器渲染 +# WEBDRIVER = dict( +# pool_size=1, # 浏览器的数量 +# load_images=True, # 是否加载图片 +# user_agent=None, # 字符串 或 无参函数，返回值为user_agent +# proxy=None, # xxx.xxx.xxx.xxx:xxxx 或 无参函数，返回值为代理地址 +# headless=False, # 是否为无头浏览器 +# driver_type="CHROME", # CHROME、EDGE、PHANTOMJS、FIREFOX +# timeout=30, # 请求超时时间 +# window_size=(1024, 800), # 窗口大小 +# executable_path=None, # 浏览器路径，默认为默认路径 +# render_time=0, # 渲染时长，即打开网页等待指定时间后再获取源码 +# custom_argument=[ +# "--ignore-certificate-errors", +# "--disable-blink-features=AutomationControlled", +# ], # 自定义浏览器渲染参数 +# xhr_url_regexes=None, # 拦截xhr接口，支持正则，数组类型 +# auto_install_driver=True, # 自动下载浏览器驱动 支持chrome 和 firefox +# download_path=None, # 下载文件的路径 +# use_stealth_js=False, # 使用stealth.min.js隐藏浏览器特征 +# ) +# +# PLAYWRIGHT = dict( +# user_agent=None, # 字符串 或 无参函数，返回值为user_agent +# proxy=None, # xxx.xxx.xxx.xxx:xxxx 或 无参函数，返回值为代理地址 +# headless=False, # 是否为无头浏览器 +# driver_type="chromium", # chromium、firefox、webkit +# timeout=30, # 请求超时时间 +# window_size=(1024, 800), # 窗口大小 +# executable_path=None, # 浏览器路径，默认为默认路径 +# download_path=None, # 下载文件的路径 +# render_time=0, # 渲染时长，即打开网页等待指定时间后再获取源码 +# wait_until="networkidle", # 等待页面加载完成的事件,可选值："commit", "domcontentloaded", "load", "networkidle" +# use_stealth_js=False, # 使用stealth.min.js隐藏浏览器特征 +# page_on_event_callback=None, # page.on() 事件的回调 如 page_on_event_callback={"dialog": lambda dialog: dialog.accept()} +# storage_state_path=None, # 保存浏览器状态的路径 +# url_regexes=None, # 拦截接口，支持正则，数组类型 +# save_all=False, # 是否保存所有拦截的接口, 配合url_regexes使用，为False时只保存最后一次拦截的接口 +# ) +# +# # 爬虫启动时，重新抓取失败的requests +# RETRY_FAILED_REQUESTS = False +# # 保存失败的request +# SAVE_FAILED_REQUEST = True +# # request防丢机制。（指定的REQUEST_LOST_TIMEOUT时间内request还没做完，会重新下发 重做） +# REQUEST_LOST_TIMEOUT = 600 # 10分钟 +# # request网络请求超时时间 +# REQUEST_TIMEOUT = 22 # 等待服务器响应的超时时间，浮点数，或(connect timeout, read timeout)元组 +# # item在内存队列中最大缓存数量 +# ITEM_MAX_CACHED_COUNT = 5000 +# # item每批入库的最大数量 +# ITEM_UPLOAD_BATCH_MAX_SIZE = 1000 +# # item入库时间间隔 +# ITEM_UPLOAD_INTERVAL = 1 +# # 内存任务队列最大缓存的任务数，默认不限制；仅对AirSpider有效。 +# TASK_MAX_CACHED_SIZE = 0 +# +# # 下载缓存 利用redis缓存，但由于内存大小限制，所以建议仅供开发调试代码时使用，防止每次debug都需要网络请求 +# RESPONSE_CACHED_ENABLE = False # 是否启用下载缓存 成本高的数据或容易变需求的数据，建议设置为True +# RESPONSE_CACHED_EXPIRE_TIME = 3600 # 缓存时间 秒 +# RESPONSE_CACHED_USED = False # 是否使用缓存 补采数据时可设置为True +# +# # 设置代理 +# PROXY_EXTRACT_API = None # 代理提取API ，返回的代理分割符为\r\n +# PROXY_ENABLE = True +# +# # 随机headers +# RANDOM_HEADERS = True +# # UserAgent类型 支持 'chrome', 'opera', 'firefox', 'internetexplorer', 'safari'，'mobile' 若不指定则随机类型 +# USER_AGENT_TYPE = "chrome" +# # 默认使用的浏览器头 +# DEFAULT_USERAGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36" +# # requests 使用session +# USE_SESSION = False +# +# # 去重 +# ITEM_FILTER_ENABLE = False # item 去重 +# REQUEST_FILTER_ENABLE = False # request 去重 +# ITEM_FILTER_SETTING = dict( +# filter_type=1 # 永久去重（BloomFilter） = 1 、内存去重（MemoryFilter） = 2、 临时去重（ExpireFilter）= 3、轻量去重（LiteFilter）= 4 +# ) +# REQUEST_FILTER_SETTING = dict( +# filter_type=3, # 永久去重（BloomFilter） = 1 、内存去重（MemoryFilter） = 2、 临时去重（ExpireFilter）= 3、 轻量去重（LiteFilter）= 4 +# expire_time=2592000, # 过期时间1个月 +# ) +# +# # 报警 支持钉钉、飞书、企业微信、邮件 +# # 钉钉报警 +# DINGDING_WARNING_URL = "" # 钉钉机器人api +# DINGDING_WARNING_PHONE = "" # 报警人 支持列表，可指定多个 +# DINGDING_WARNING_ALL = False # 是否提示所有人， 默认为False +# # 飞书报警 +# # https://open.feishu.cn/document/ukTMukTMukTM/ucTM5YjL3ETO24yNxkjN#e1cdee9f +# FEISHU_WARNING_URL = "" # 飞书机器人api +# FEISHU_WARNING_USER = None # 报警人 {"open_id":"ou_xxxxx", "name":"xxxx"} 或 [{"open_id":"ou_xxxxx", "name":"xxxx"}] +# FEISHU_WARNING_ALL = False # 是否提示所有人， 默认为False +# # 邮件报警 +# EMAIL_SENDER = "" # 发件人 +# EMAIL_PASSWORD = "" # 授权码 +# EMAIL_RECEIVER = "" # 收件人 支持列表，可指定多个 +# EMAIL_SMTPSERVER = "smtp.163.com" # 邮件服务器 默认为163邮箱 +# # 企业微信报警 +# WECHAT_WARNING_URL = "" # 企业微信机器人api +# WECHAT_WARNING_PHONE = "" # 报警人 将会在群内@此人, 支持列表，可指定多人 +# WECHAT_WARNING_ALL = False # 是否提示所有人， 默认为False +# # 时间间隔 +# WARNING_INTERVAL = 3600 # 相同报警的报警时间间隔，防止刷屏; 0表示不去重 +# WARNING_LEVEL = "DEBUG" # 报警级别， DEBUG / INFO / ERROR +# WARNING_FAILED_COUNT = 1000 # 任务失败数 超过WARNING_FAILED_COUNT则报警 +# +# LOG_NAME = os.path.basename(os.getcwd()) +# LOG_PATH = "log/%s.log" % LOG_NAME # log存储路径 +# LOG_LEVEL = "DEBUG" +# LOG_COLOR = True # 是否带有颜色 +# LOG_IS_WRITE_TO_CONSOLE = True # 是否打印到控制台 +# LOG_IS_WRITE_TO_FILE = False # 是否写文件 +# LOG_MODE = "w" # 写文件的模式 +# LOG_MAX_BYTES = 10 * 1024 * 1024 # 每个日志文件的最大字节数 +# LOG_BACKUP_COUNT = 20 # 日志文件保留数量 +# LOG_ENCODING = "utf8" # 日志文件编码 +# OTHERS_LOG_LEVAL = "ERROR" # 第三方库的log等级 +# +# # 切换工作路径为当前项目路径 +# project_path = os.path.abspath(os.path.dirname(__file__)) +# os.chdir(project_path) # 切换工作路经 +# sys.path.insert(0, project_path) +# print("当前工作路径为 " + os.getcwd()) ``` -# redis key -# 任务表模版 -TAB_REQUSETS = "{redis_key}:z_requsets" -# 任务失败模板 -TAB_FAILED_REQUSETS = "{redis_key}:z_failed_requsets" -# 爬虫状态表模版 -TAB_SPIDER_STATUS = "{redis_key}:z_spider_status" -# item 表模版 -TAB_ITEM = "{redis_key}:s_{item_name}" -# 爬虫时间记录表 -TAB_SPIDER_TIME = "{redis_key}:h_spider_time" - -# MYSQL -MYSQL_IP = os.getenv("MYSQL_IP") -MYSQL_PORT = int(os.getenv("MYSQL_PORT", 3306)) -MYSQL_DB = os.getenv("MYSQL_DB") -MYSQL_USER_NAME = os.getenv("MYSQL_USER_NAME") -MYSQL_USER_PASS = os.getenv("MYSQL_USER_PASS") - -# REDIS -# ip:port 多个可写为列表或者逗号隔开 如 ip1:port1,ip2:port2 或 ["ip1:port1", "ip2:port2"] -REDISDB_IP_PORTS = os.getenv("REDISDB_IP_PORTS") -REDISDB_USER_PASS = os.getenv("REDISDB_USER_PASS") -# 默认 0 到 15 共16个数据库 -REDISDB_DB = int(os.getenv("REDISDB_DB", 0)) -# 适用于redis哨兵模式 -REDISDB_SERVICE_NAME = os.getenv("REDISDB_SERVICE_NAME") - -# 爬虫相关 -# COLLECTOR -COLLECTOR_SLEEP_TIME = 1 # 从任务队列中获取任务到内存队列的间隔 -COLLECTOR_TASK_COUNT = 10 # 每次获取任务数量 - -# SPIDER -SPIDER_THREAD_COUNT = 1 # 爬虫并发数 -SPIDER_SLEEP_TIME = 0 # 下载时间间隔（解析完一个response后休眠时间） -SPIDER_TASK_COUNT = 1 # 每个parser从内存队列中获取任务的数量 -SPIDER_MAX_RETRY_TIMES = 100 # 每个请求最大重试次数 -# 是否主动执行添加 设置为False 需要手动调用start_monitor_task，适用于多进程情况下 -SPIDER_AUTO_START_REQUESTS = True - -# 重新尝试失败的requests 当requests重试次数超过允许的最大重试次数算失败 -RETRY_FAILED_REQUESTS = False -# request 超时时间，超过这个时间重新做（不是网络请求的超时时间）单位秒 -REQUEST_TIME_OUT = 600 # 10分钟 -# 保存失败的request -SAVE_FAILED_REQUEST = True - -# 下载缓存 利用redis缓存，由于内存小，所以仅供测试时使用 -RESPONSE_CACHED_ENABLE = False # 是否启用下载缓存 成本高的数据或容易变需求的数据，建议设置为True -RESPONSE_CACHED_EXPIRE_TIME = 3600 # 缓存时间 秒 -RESPONSE_CACHED_USED = False # 是否使用缓存 补才数据时可设置为True - -WARNING_FAILED_COUNT = 1000 # 任务失败数 超过WARNING_FAILED_COUNT则报警 -# 爬虫初始化工作 -# redis 存放item与request的根目录 -REDIS_KEY = "" -# 每次启动时需要删除的表 -DELETE_TABS = [] -# 爬虫做完request后是否自动结束或者等待任务 -AUTO_STOP_WHEN_SPIDER_DONE = True -# 是否将item添加到 mysql 支持列表 指定添加的item 可模糊指定 -ADD_ITEM_TO_MYSQL = True -# 是否将item添加到 redis 支持列表 指定添加的item 可模糊指定 -ADD_ITEM_TO_REDIS = False +- 数据库连接信息默认读取的环境变量，因此若不想将自己的账号暴露给其他同事，建议写在环境变量里，环境变量的`key`与配置文件的`key`相同 -# PROCESS 进程数 未用 -PROCESS_COUNT = 1 +- `LOG_LEVEL`为`DEBUG`时，日志输出比较详细，开发阶段使用。线上部署时，可将`LOG_LEVEL`改为`INFO` -# 设置代理 -PROXY_EXTRACT_API = None # 代理提取API ，返回的代理分割符为\r\n -PROXY_ENABLE = True +## 自定义配置 -# 随机headers -RANDOM_HEADERS = True -# requests 使用session -USE_SESSION = False +常用于一个项目下有多个爬虫，不同的爬虫使用不同的配置。我们可以使用类变量`__custom_setting__`自定义配置，示例如下： -# 去重 -ITEM_FILTER_ENABLE = False # item 去重 -REQUEST_FILTER_ENABLE = False # request 去重 +```python +import feapder -# 报警 -DINGDING_WARNING_URL = "" # 钉钉机器人api -DINGDING_WARNING_PHONE = "" # 报警人 -LINGXI_TOKEN = "" # 灵犀报警token -LOG_NAME = os.path.basename(os.getcwd()) -LOG_PATH = "log/%s.log" % LOG_NAME # log存储路径 -LOG_LEVEL = "DEBUG" -LOG_IS_WRITE_TO_FILE = False -OTHERS_LOG_LEVAL = "ERROR" # 第三方库的log等级 +class SpiderTest(feapder.AirSpider): + __custom_setting__ = dict( + SPIDER_MAX_RETRY_TIMES=20, + ) ``` - -## 自定义配置 - -常用于一个项目下有多个爬虫，不同的爬虫使用不同的配置。我们可以使用类变量`__custom_setting__`自定义配置，示例如下： - - import spider - from items import * - - - class SpiderTest(feapder.AirSpider): - __custom_setting__ = dict( - PROXY_EXTRACT_API="代理提取地址", - ) \ No newline at end of file diff --git a/docs/usage/AirSpider.md b/docs/usage/AirSpider.md index aee35e78..71ac053c 100644 --- a/docs/usage/AirSpider.md +++ b/docs/usage/AirSpider.md @@ -5,29 +5,37 @@ AirSpider是一款轻量爬虫，学习成本低。面对一些数据量较少 ## 1. 创建爬虫 命令参考：[命令行工具](command/cmdline.md?id=_2-创建爬虫) - + 示例 - feapder create -s air_spider_test - +```python +feapder create -s air_spider_test + +请选择爬虫模板 +> AirSpider + Spider + TaskSpider + BatchSpider +``` + 生成如下 - + import feapder - - + + class AirSpiderTest(feapder.AirSpider): def start_requests(self): yield feapder.Request("https://www.baidu.com") - + def parse(self, request, response): print(response) - - + + if __name__ == "__main__": AirSpiderTest().start() - - + + 可直接运行 ## 2. 代码讲解 @@ -64,7 +72,7 @@ AirSpider是一款轻量爬虫，学习成本低。面对一些数据量较少 def parse(self, request, response): xxx = request.xxx print(xxx) - + 直接在feapder.Request中携带即可，xxx为携带数据的key，可以随意写，只要不和feapder.Request默认参数冲突即可。默认参数参考[Request](source_code/Request.md)。可以携带任意类型的值，如字典、类等 取值：如何携带就如何取值，如上我们携带xxx， 那么`request.xxx` 可将xxx值取出，取出的值和携带的值类型一致。 @@ -100,34 +108,64 @@ request.参数， 这里的参数支持requests所有参数，同时也可携带 def xxx(self, request): """ 我是自定义的下载中间件 - :param request: - :return: + :param request: + :return: """ request.headers = {'User-Agent':"lalala"} return request - + 自定义的下载中间件只有指定的请求才会经过。其他未指定下载中间件的请求，还是会经过默认的下载中间件 -## 8. 失败重试 +## 8. 校验 + +```python +def validate(self, request, response): + """ + @summary: 校验函数, 可用于校验response是否正确 + 若函数内抛出异常，则重试请求 + 若返回True 或 None，则进入解析函数 + 若返回False，则抛弃当前请求 + 可通过request.callback_name 区分不同的回调函数，编写不同的校验逻辑 + --------- + @param request: + @param response: + --------- + @result: True / None / False + """ + pass +``` + +例如： + +```python +def validate(self, request, response): + if response.status_code != 200: + raise Exception("response code not 200") # 重试 + + # if "哈哈" not in response.text: + # return False # 抛弃当前请求 +``` + +## 9. 失败重试 框架支持重试机制，下载失败或解析函数抛出异常会自动重试请求。 例如下面代码，校验了返回的code是否为200，非200抛出异常，触发重试 def parse(self, request, response): - if response.code != 200: + if response.status_code != 200: raise Exception("非法页面") - + 默认最大重试次数为100次，我们可以引入配置文件或自定义配置来修改重试次数，详情参考[配置文件](source_code/配置文件.md) -## 9. 爬虫配置 +## 10. 爬虫配置 爬虫配置支持自定义配置或引入配置文件`setting.py`的方式。 配置文件：在工作区间的根目录下引入`setting.py`，具体参考[配置文件](source_code/配置文件.md) -![-w261](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/02/21/16138971894815.jpg?x-oss-process=style/markdown-media) +![-w261](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/02/21/16138971894815.jpg) 自定义配置： @@ -137,22 +175,22 @@ request.参数， 这里的参数支持requests所有参数，同时也可携带 __custom_setting__ = dict( PROXY_EXTRACT_API="代理提取地址", ) - + 上例是配置代理提取地址，以便爬虫使用代理，自定义配置支持配置文件中的所有参数。 配置优先级： 自定义配置 > 配置文件，即自定义配置会覆盖配置文件里的配置信息，不过自定义配置只对自己有效，配置文件可以是多个爬虫公用的 AirSpider不支持去重，因此配置文件中的去重配置无效 -## 10. 加快采集速度 +## 11. 加快采集速度 默认爬虫为1线程，我们可通过修改线程数来加快采集速度。除了在配置文件中修改或使用自定义配置外，可以在启动函数中传递线程数 if __name__ == "__main__": AirSpiderTest(thread_count=10).start() - -## 11. 数据入库 + +## 12. 数据入库 框架内封装了`MysqlDB`、`RedisDB`，与pymysql不同的是，MysqlDB 使用了线程池，且对方法进行了封装，使用起来更方便。RedisDB 支持 哨兵模式、集群模式。使用方法如下： @@ -160,7 +198,7 @@ AirSpider不支持去重，因此配置文件中的去重配置无效 from feapder.db.mysqldb import MysqlDB from feapder.db.redisdb import RedisDB - + 以mysql为例，获取mysql对象 方式1：若`setting.py` 或 `__custom_setting__`中指定了数据库连接信息，则可以直接以`db = MysqlDB()`方式获取 @@ -172,7 +210,7 @@ AirSpider不支持去重，因此配置文件中的去重配置无效 MYSQL_DB = "feapder", MYSQL_USER_NAME = "feapder", MYSQL_USER_PASS = "feapder123" - + ) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -187,11 +225,115 @@ AirSpider不支持去重，因此配置文件中的去重配置无效 user_pass="feapder123", db="feapder" ) - + MysqlDB 的具体使用方法见 [MysqlDB](source_code/MysqlDB.md) RedisDB 的具体使用方法见 [RedisDB](source_code/RedisDB.md) -## 12. 完整的代码示例 +框架也支持数据自动入库，详见[数据自动入库](usage/Spider?id=_5-数据自动入库) + +## 13. 浏览器渲染下载 + +采集动态页面时（Ajax渲染的页面），常用的有两种方案。一种是找接口拼参数，这种方式比较复杂但效率高，需要一定的爬虫功底；另外一种是采用浏览器渲染的方式，直接获取源码，简单方便 + +使用方式： +```python +def start_requests(self): + yield feapder.Request("https://news.qq.com/", render=True) +``` +在返回的Request中传递`render=True`即可 + +框架支持`CHROME`、`EDGE`和`PHANTOMJS`浏览器渲染，可通过[配置文件](source_code/配置文件)进行配置。相关配置如下： + +```python +# 浏览器渲染 +WEBDRIVER = dict( + pool_size=1, # 浏览器的数量 + load_images=True, # 是否加载图片 + user_agent=None, # 字符串 或 无参函数，返回值为user_agent + proxy=None, # xxx.xxx.xxx.xxx:xxxx 或 无参函数，返回值为代理地址 + headless=False, # 是否为无头浏览器 + driver_type="CHROME", # CHROME、EDGE或PHANTOMJS, + timeout=30, # 请求超时时间 + window_size=(1024, 800), # 窗口大小 + executable_path=None, # 浏览器路径，默认为默认路径 + render_time=0, # 渲染时长，即打开网页等待指定时间后再获取源码 +) +``` + +## 14. 自定义下载器 + +自定义下载器即在下载中间件里下载，然后返回response即可，如使用httpx库下载以便支持http2 + + +```python +import feapder +import httpx + + +class AirSpeedTest(feapder.AirSpider): + def start_requests(self): + yield feapder.Request("http://www.baidu.com") + + def download_midware(self, request): + with httpx.Client(http2=True) as client: + response = client.get(request.url) + + return request, response + + def parse(self, request, response): + print(response) + + +if __name__ == "__main__": + AirSpeedTest(thread_count=1).start() +``` + +注意，解析函数里的response已经变成了下载中间件返回的response，而非默认的。若想用xpath、css等解析功能，写法如下 + +```python + +import feapder +import httpx +from feapder.network.selector import Selector + + +class AirSpeedTest(feapder.AirSpider): + def start_requests(self): + yield feapder.Request("http://www.baidu.com") + + def download_midware(self, request): + with httpx.Client(http2=True) as client: + response = client.get(request.url) + + return request, response + + def parse(self, request, response): + selector = Selector(response.text) + title = selector.xpath("//title/text()").extract_first() + print(title) +``` + +## 15. 主动停止爬虫 + +``` +import feapder + + +class AirTest(feapder.AirSpider): + def start_requests(self): + yield feapder.Request("http://www.baidu.com") + + def parse(self, request, response): + self.stop_spider() # 停止爬虫，可以在任意地方调用该方法 + + +if __name__ == "__main__": + AirTest().start() +``` + +## 16. 完整的代码示例 + +AirSpider：https://github.com/Boris-code/feapder/blob/master/tests/air-spider/test_air_spider.py -[https://github.com/Boris-code/feapder/blob/master/tests/air-spider/test_air_spider.py](https://github.com/Boris-code/feapder/blob/master/tests/air-spider/test_air_spider.py) \ No newline at end of file +浏览器渲染：https://github.com/Boris-code/feapder/blob/master/tests/test_rander.py diff --git a/docs/usage/BatchSpider.md b/docs/usage/BatchSpider.md index f2a5027e..d85bbce9 100644 --- a/docs/usage/BatchSpider.md +++ b/docs/usage/BatchSpider.md @@ -6,14 +6,21 @@ BatchSpider是一款分布式批次爬虫，对于需要周期性采集的数据 参考 [Spider](usage/Spider?id=_1-创建项目) - ## 2. 创建爬虫 命令参考：[命令行工具](command/cmdline.md?id=_2-创建爬虫) 示例: - feapder create -s batch_spider_test 3 +```python +feapder create -s batch_spider_test + +请选择爬虫模板 + AirSpider + Spider + TaskSpider +> BatchSpider +``` 生成如下 @@ -43,7 +50,7 @@ class BatchSpiderTest(feapder.BatchSpider): if __name__ == "__main__": spider = BatchSpiderTest( - redis_key="xxx:xxxx", # redis中存放任务等信息的根key + redis_key="xxx:xxxx", # 分布式爬虫调度信息存储位置 task_table="", # mysql中的任务表 task_keys=["id", "xxx"], # 需要获取任务表里的字段名，可添加多个 task_state="state", # mysql中任务状态字段 @@ -70,7 +77,7 @@ BatchSpider参数： 1. redis_key：redis中存储任务等信息的key前缀，如redis_key="feapder:spider_test", 则redis中会生成如下 - ![-w365](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/02/21/16139009217536.jpg?x-oss-process=style/markdown-media) + ![-w365](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/02/21/16139009217536.jpg) 1. task_table：mysql中的任务表，为抓取的任务种子，需要运行前手动创建好 2. task_keys：任务表里需要获取的字段，框架会将这些字段的数据查询出来，传递给爬虫，然后拼接请求 @@ -100,7 +107,7 @@ BatchSpider参数： 任务表为存储任务种子的，表结构需要包含`id`、`任务状态`两个字段，如我们需要对某些地址进行采集，设计如下 -![-w752](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/02/22/16139762922842.jpg?x-oss-process=style/markdown-media) +![-w752](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/02/22/16139762922842.jpg) 建表语句： @@ -128,17 +135,17 @@ def start_requests(self, task): pass ``` -任务拼接在`start_requests`里处理。这里的task参数为元组，值为BatchSpider启动参数中指定的`task_keys`对应的值 +任务拼接在`start_requests`里处理。这里的task参数为BatchSpider启动参数中指定的`task_keys`对应的值 如表`batch_spider_task`，现有任务信息如下： -![-w398](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/02/22/16139773315622.jpg?x-oss-process=style/markdown-media) +![-w398](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/02/22/16139773315622.jpg) 启动参数配置如下，注意`task_keys=["id", "url"]`： ``` def crawl_test(args): spider = test_spider.TestSpider( - redis_key="feapder:test_batch_spider", # redis中存放任务等信息的根key + redis_key="feapder:test_batch_spider", # 分布式爬虫调度信息存储位置 task_table="batch_spider_task", # mysql中的任务表 task_keys=["id", "url"], # 需要获取任务表里的字段名，可添加多个 task_state="state", # mysql中任务状态字段 @@ -162,6 +169,19 @@ def crawl_test(args): yield feapder.Request(url, task_id=id) ``` +task值的获取方式，支持以下几种： + +```python +# 列表方式 +id, url = task +id = task[0] +url = task[1] +# 字典方式 +id, url = task.id, task.url +id, url = task.get("id"), task.get("url") +id, url = task["id"], task["url"] +``` + ## 8. 更新任务状态 @@ -224,7 +244,7 @@ def failed_request(self, request, response): ## 10.增量采集 -每个批次开始时，框架默认会重置状态非-1的任务为0，然后重新抓取。但是有些需求是增强采集的，做过的任务无需再次处理。重置任务是`init_task`方法实现的，我们可以将`init_task`方法置空来实现增量采集 +每个批次开始时，框架默认会重置状态非-1的任务为0，然后重新抓取。但是有些需求是增量采集的，做过的任务无需再次处理。重置任务是`init_task`方法实现的，我们可以将`init_task`方法置空来实现增量采集 ``` def init_task(self): @@ -239,7 +259,7 @@ def failed_request(self, request, response): def test_debug(): spider = test_spider.TestSpider.to_DebugBatchSpider( task_id=1, - redis_key="feapder:test_batch_spider", # redis中存放任务等信息的根key + redis_key="feapder:test_batch_spider", # 分布式爬虫调度信息存储位置 task_table="batch_spider_task", # mysql中的任务表 task_keys=["id", "url"], # 需要获取任务表里的字段名，可添加多个 task_state="state", # mysql中任务状态字段 @@ -261,7 +281,7 @@ DebugBatchSpider爬虫支持传递`task_id`或直接传递`task`来指定任务 ## 12. 运行BatchSpider -与[Spider](usage/Spider?id=_7-运行多个spider)运行方式类型。但因每个爬虫都有maser和work两个入口，因此框架提供一种更方便的方式，写法如下 +与[Spider](usage/Spider?id=_7-运行多个spider)运行方式类似。但因每个爬虫都有maser和work两个入口，因此框架提供一种更方便的方式，写法如下 ``` from spiders import * @@ -270,7 +290,7 @@ from feapder import ArgumentParser def crawl_test(args): spider = test_spider.TestSpider( - redis_key="feapder:test_batch_spider", # redis中存放任务等信息的根key + redis_key="feapder:test_batch_spider", # 分布式爬虫调度信息存储位置 task_table="batch_spider_task", # mysql中的任务表 task_keys=["id", "url"], # 需要获取任务表里的字段名，可添加多个 task_state="state", # mysql中任务状态字段 diff --git a/docs/usage/Spider.md b/docs/usage/Spider.md index 26c8e64d..47736c21 100644 --- a/docs/usage/Spider.md +++ b/docs/usage/Spider.md @@ -14,7 +14,7 @@ Spider是一款基于redis的分布式爬虫，适用于海量数据采集，支 创建好项目后，开发时我们需要将项目设置为工作区间，否则引入非同级目录下的文件时，编译器会报错。不过因为main.py在项目的根目录下，因此不影响正常运行。 -![-w925](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/02/21/16139218044066.jpg?x-oss-process=style/markdown-media) +![-w925](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/02/21/16139218044066.jpg) 设置工作区间方式（以pycharm为例）：项目->右键->Mark Directory as -> Sources Root @@ -25,7 +25,15 @@ Spider是一款基于redis的分布式爬虫，适用于海量数据采集，支 示例: - feapder create -s spider_test 2 +```python +feapder create -s spider_test + +请选择爬虫模板 + AirSpider +> Spider + TaskSpider + BatchSpider +``` 生成如下 @@ -64,7 +72,7 @@ Spider参数： redis_key为redis中存储任务等信息的key前缀，如redis_key="feapder:spider_test", 则redis中会生成如下 -![-w365](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/02/21/16139009217536.jpg?x-oss-process=style/markdown-media) +![-w365](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/02/21/16139009217536.jpg) 更详细的说明可查看 [Spider进阶](source_code/Spider进阶.md) @@ -98,13 +106,13 @@ class SpiderDataItem(Item): 代码示例： -![-w682](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/02/21/16139031333228.jpg?x-oss-process=style/markdown-media) +![-w682](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/02/21/16139031333228.jpg) 返回item后，item会流经到框架的ItemBuffer, ItemBuffer每.05秒或当item数量积攒到5000个，便会批量将这些item批量入库。表名为类名去掉Item的小写，如SpiderDataItem数据会落入到spider_data表。 Item详细介绍参考[Item](source_code/Item.md) -ItemBuffer详细介绍请参考[ItemBuffer](source_code/ItemBuffer.md) +[comment]: <> (ItemBuffer详细介绍请参考[ItemBuffer](source_code/ItemBuffer.md)) ## 6. 调试 @@ -125,7 +133,7 @@ ItemBuffer详细介绍请参考[ItemBuffer](source_code/ItemBuffer.md) 可以看到，代码中 `to_DebugSpider`方法可以将原爬虫直接转为debug爬虫，然后通过传递request参数抓取指定的任务。 -通常结合断点来进行调试，bebug模式下，运行产生的数据默认不入库 +通常结合断点来进行调试，debug模式下，运行产生的数据默认不入库 除了指定request参数外，还可以指定`request_dict`参数，request_dict接收字典类型，如`request_dict={"url":"http://www.baidu.com"}`, 其作用于传递request一致。request 与 request_dict 二者选一传递即可 @@ -135,7 +143,7 @@ ItemBuffer详细介绍请参考[ItemBuffer](source_code/ItemBuffer.md) 例如如下项目： -![-w300](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/02/21/16139224711465.jpg?x-oss-process=style/markdown-media) +![-w300](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/02/21/16139224711465.jpg) 项目中包含了两个spider，main.py写法如下： @@ -186,6 +194,10 @@ if __name__ == "__main__": python3 main.py --test_spider -## 8. 完整的代码示例 +## 8. 分布式 + +分布式说白了就是启动多个进程，处理同一批任务。`Spider`支持启动多份，且不会重复发下任务，我们可以在多个服务器上部署启动，也可以在同一个机器上启动多次。 + +## 9. 完整的代码示例 -[https://github.com/Boris-code/feapder/tree/master/tests/spider](https://github.com/Boris-code/feapder/tree/master/tests/spider) \ No newline at end of file +[https://github.com/Boris-code/feapder/tree/master/tests/spider](https://github.com/Boris-code/feapder/tree/master/tests/spider) diff --git a/docs/usage/TaskSpider.md b/docs/usage/TaskSpider.md new file mode 100644 index 00000000..5978dff9 --- /dev/null +++ b/docs/usage/TaskSpider.md @@ -0,0 +1,133 @@ +# TaskSpider + +TaskSpider是一款分布式爬虫，内部封装了取种子任务的逻辑，内置支持从redis或者mysql获取任务，也可通过自定义实现从其他来源获取任务 + +## 1. 创建项目 + +参考 [Spider](usage/Spider?id=_1-创建项目) + +## 2. 创建爬虫 + +命令参考：[命令行工具](command/cmdline.md?id=_2-创建爬虫) + +示例: + +```python +feapder create -s task_spider_test + +请选择爬虫模板 + AirSpider + Spider +> TaskSpider + BatchSpider +``` + +示例代码： + +```python +import feapder +from feapder import ArgumentParser + + +class TaskSpiderTest(feapder.TaskSpider): + # 自定义数据库，若项目中有setting.py文件，此自定义可删除 + # redis 必须，mysql可选 + __custom_setting__ = dict( + REDISDB_IP_PORTS="localhost:6379", + REDISDB_USER_PASS="", + REDISDB_DB=0, + MYSQL_IP="localhost", + MYSQL_PORT=3306, + MYSQL_DB="feapder", + MYSQL_USER_NAME="feapder", + MYSQL_USER_PASS="feapder123", + ) + + def add_task(self): + # 加种子任务 框架会调用这个函数，方便往redis里塞任务，但不能写成死循环。实际业务中可以自己写个脚本往redis里塞任务 + self._redisdb.zadd(self._task_table, {"id": 1, "url": "https://www.baidu.com"}) + + def start_requests(self, task): + task_id, url = task + yield feapder.Request(url, task_id=task_id) + + def parse(self, request, response): + # 提取网站title + print(response.xpath("//title/text()").extract_first()) + # 提取网站描述 + print(response.xpath("//meta[@name='description']/@content").extract_first()) + print("网站地址: ", response.url) + + # mysql 需要更新任务状态为做完 即 state=1 + # yield self.update_task_batch(request.task_id) + +def start(args): + """ + 用mysql做种子表 + """ + spider = TaskSpiderTest( + task_table="spider_task", # 任务表名 + task_keys=["id", "url"], # 表里查询的字段 + redis_key="test:task_spider", # redis里做任务队列的key + keep_alive=True, # 是否常驻 + ) + if args == 1: + spider.start_monitor_task() + else: + spider.start() + + +def start2(args): + """ + 用redis做种子表 + """ + spider = TaskSpiderTest( + task_table="spider_task2", # 任务表名 + task_table_type="redis", # 任务表类型为redis + redis_key="test:task_spider", # redis里做任务队列的key + keep_alive=True, # 是否常驻 + use_mysql=False, # 若用不到mysql，可以不使用 + ) + if args == 1: + spider.start_monitor_task() + else: + spider.start() + + +if __name__ == "__main__": + parser = ArgumentParser(description="测试TaskSpider") + + parser.add_argument("--start", type=int, nargs=1, help="用mysql做种子表 (1|2）", function=start) + parser.add_argument("--start2", type=int, nargs=1, help="用redis做种子表 (1|2）", function=start2) + + parser.start() + + # 下发任务 python3 task_spider_test.py --start 1 + # 采集 python3 task_spider_test.py --start 2 +``` + +## 3. 代码讲解 + +#### 3.1 main + +main函数为命令行参数解析，分别定义了两种获取任务的方式。start函数为从mysql里获取任务，前提是需要有任务表。start2函数为从redis里获取任务，指定了根任务的key为`spider_task2`，key的类型为zset + +启动：TaskSpider分为master及work两种程序 + +1. master负责下发任务，监控批次进度，创建批次等功能，启动方式： + + spider.start_monitor_task() + +2. worker负责消费任务，抓取数据，启动方式： + + spider.start() + +#### 3.1 add_task: + +框架内置的函数，在调用start_monitor_task时会自动调度此函数，用于初始化任务种子，若不需要，可直接删除此函数 + +本代码示例为向redis的`spider_task2`的key加了个值为`{"id": 1, "url": "https://www.baidu.com"}`的种子 + + + + diff --git "a/docs/usage/\344\275\277\347\224\250\345\211\215\345\277\205\350\257\273.md" "b/docs/usage/\344\275\277\347\224\250\345\211\215\345\277\205\350\257\273.md" index 94ca32dc..88794fa8 100644 --- "a/docs/usage/\344\275\277\347\224\250\345\211\215\345\277\205\350\257\273.md" +++ "b/docs/usage/\344\275\277\347\224\250\345\211\215\345\277\205\350\257\273.md" @@ -22,7 +22,7 @@ feapder爬虫框架内置三种爬虫 分布式批次爬虫，对于需要周期性采集的数据，优先考虑使用本爬虫。 本爬虫会自动维护个批次信息表，详细的记录了每个批次时间、任务完成情况、批次周期等信息，示例数据如下 -![-w899](http://markdown-media.oss-cn-beijing.aliyuncs.com/2020/12/20/16084680404224.jpg?x-oss-process=style/markdown-media) +![-w899](http://markdown-media.oss-cn-beijing.aliyuncs.com/2020/12/20/16084680404224.jpg) 另外本爬虫与其他爬虫最大的区别是，会维护个批次时间信息，本批次未完成下一批次不会开始。 diff --git "a/docs/usage/\347\210\254\350\231\253\351\233\206\346\210\220.md" "b/docs/usage/\347\210\254\350\231\253\351\233\206\346\210\220.md" index 8c17204f..03d7af90 100644 --- "a/docs/usage/\347\210\254\350\231\253\351\233\206\346\210\220.md" +++ "b/docs/usage/\347\210\254\350\231\253\351\233\206\346\210\220.md" @@ -10,14 +10,14 @@ ### 常规做法 每个新闻源写一个或多个爬虫，如下： -![-w526](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/03/03/16146986664270.jpg?x-oss-process=style/markdown-media) +![-w526](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/03/03/16146986664270.jpg) 这样每个爬虫之间比较独立，如果有上百个数据源，需要启动上百个爬虫脚本，不便于管理 ### 本框架做法 本框架支持上述的常规做法同时，支持了更友好的管理方式，可将这些爬虫集成为一个爬虫，我们只需维护这一个爬虫即可，当然也支持分布式。 -![-w528](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/03/03/16146992324366.jpg?x-oss-process=style/markdown-media) +![-w528](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/03/03/16146992324366.jpg) 注： Spider爬虫与BatchSpider爬虫支持集成，AirSpider不支持 @@ -177,7 +177,7 @@ def batch_spider_integration_test(args): 任务表： -![-w444](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/03/03/16147423559139.jpg?x-oss-process=style/markdown-media) +![-w444](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/03/03/16147423559139.jpg) 任务表里需要有一个字段存储解析器的类名，与对应的任务关联，在我们取任务时携带这个类名，这样框架才知道这条任务归属于哪个解析器 diff --git a/feapder/VERSION b/feapder/VERSION index 512a1faa..7b0231f5 100644 --- a/feapder/VERSION +++ b/feapder/VERSION @@ -1 +1 @@ -1.1.9 +1.9.3 \ No newline at end of file diff --git a/feapder/__init__.py b/feapder/__init__.py index 1eb18d7f..565be4b9 100644 --- a/feapder/__init__.py +++ b/feapder/__init__.py @@ -5,18 +5,21 @@ @summary: --------- @author: Boris -@email: boris@bzkj.tech +@email: boris_liu@foxmail.com """ -import os, sys +import os import re +import sys -sys.path.insert(0, re.sub(r"([\\/]items)|([\\/]spiders)", "", os.getcwd())) +sys.path.insert(0, re.sub(r"([\\/]items$)|([\\/]spiders$)", "", os.getcwd())) __all__ = [ "AirSpider", "Spider", + "TaskSpider", "BatchSpider", "BaseParser", + "TaskParser", "BatchParser", "Request", "Response", @@ -25,8 +28,8 @@ "ArgumentParser", ] -from feapder.core.spiders import Spider, BatchSpider, AirSpider -from feapder.core.base_parser import BaseParser, BatchParser +from feapder.core.spiders import AirSpider, Spider, TaskSpider, BatchSpider +from feapder.core.base_parser import BaseParser, TaskParser, BatchParser from feapder.network.request import Request from feapder.network.response import Response from feapder.network.item import Item, UpdateItem diff --git a/feapder/buffer/__init__.py b/feapder/buffer/__init__.py index 6c03df55..ff422dde 100644 --- a/feapder/buffer/__init__.py +++ b/feapder/buffer/__init__.py @@ -5,5 +5,5 @@ @summary: --------- @author: Boris -@email: boris@bzkj.tech +@email: boris_liu@foxmail.com ''' \ No newline at end of file diff --git a/feapder/buffer/item_buffer.py b/feapder/buffer/item_buffer.py index ced7bef8..35f9bb01 100644 --- a/feapder/buffer/item_buffer.py +++ b/feapder/buffer/item_buffer.py @@ -5,86 +5,122 @@ @summary: item 管理器， 负责缓冲添加到数据库中的item， 由该manager统一添加。防止多线程同时访问数据库 --------- @author: Boris -@email: boris@bzkj.tech +@email: boris_liu@foxmail.com """ import threading from queue import Queue -import feapder.setting as setting import feapder.utils.tools as tools +from feapder import setting from feapder.db.redisdb import RedisDB from feapder.dedup import Dedup from feapder.network.item import Item, UpdateItem -from feapder.utils.export_data import ExportData +from feapder.pipelines import BasePipeline +from feapder.pipelines.mysql_pipeline import MysqlPipeline +from feapder.utils import metrics from feapder.utils.log import log -MAX_ITEM_COUNT = 5000 # 缓存中最大item数 -UPLOAD_BATCH_MAX_SIZE = 1000 +MYSQL_PIPELINE_PATH = "feapder.pipelines.mysql_pipeline.MysqlPipeline" -class Singleton(object): - def __new__(cls, *args, **kwargs): - if not hasattr(cls, "_inst"): - cls._inst = super(Singleton, cls).__new__(cls) - - return cls._inst - - -class ItemBuffer(threading.Thread, Singleton): +class ItemBuffer(threading.Thread): dedup = None + __redis_db = None - def __init__(self, redis_key): + def __init__(self, redis_key, task_table=None): if not hasattr(self, "_table_item"): super(ItemBuffer, self).__init__() self._thread_stop = False self._is_adding_to_db = False self._redis_key = redis_key + self._task_table = task_table - self._items_queue = Queue(maxsize=MAX_ITEM_COUNT) - self._db = RedisDB() + self._items_queue = Queue(maxsize=setting.ITEM_MAX_CACHED_COUNT) - self._table_item = setting.TAB_ITEM - self._table_request = setting.TAB_REQUSETS.format(redis_key=redis_key) + self._table_request = setting.TAB_REQUESTS.format(redis_key=redis_key) + self._table_failed_items = setting.TAB_FAILED_ITEMS.format( + redis_key=redis_key + ) self._item_tables = { - # 'xxx_item': {'tab_item': 'xxx:xxx_item'} # 记录item名与redis中item名对应关系 + # 'item_name': 'table_name' # 缓存item名与表名对应关系 } self._item_update_keys = { - # 'xxx:xxx_item': ['id', 'name'...] # 记录redis中item名与需要更新的key对应关系 + # 'table_name': ['id', 'name'...] # 缓存table_name与__update_key__的关系 + } + + self._item_pipelines = { + # 'table_name': ['pipeline1', 'pipeline2'] # 缓存table_name与pipelines的关系 } - self._export_data = ExportData() if setting.ADD_ITEM_TO_MYSQL else None + self._pipelines = self.load_pipelines() - self.db_tip() + self._have_mysql_pipeline = MYSQL_PIPELINE_PATH in setting.ITEM_PIPELINES + self._mysql_pipeline = None if setting.ITEM_FILTER_ENABLE and not self.__class__.dedup: - self.__class__.dedup = Dedup(to_md5=False) - - def db_tip(self): - msg = "" - if setting.ADD_ITEM_TO_MYSQL: - msg += "item 自动入mysql " - if setting.ADD_ITEM_TO_REDIS: - msg += "item 自动入redis " - if not msg: - log.warning("*** 请注意检查item是否入库 !!!") - else: - log.info(msg) + if setting.ITEM_FILTER_SETTING.get( + "filter_type" + ) == Dedup.BloomFilter or setting.ITEM_FILTER_SETTING.get("name"): + self.__class__.dedup = Dedup( + to_md5=False, **setting.ITEM_FILTER_SETTING + ) + else: + self.__class__.dedup = Dedup( + to_md5=False, + name=self._redis_key, + **setting.ITEM_FILTER_SETTING, + ) + + # 导出重试的次数 + self.export_retry_times = 0 + # 导出失败的次数 TODO 非air爬虫使用redis统计 + self.export_falied_times = 0 + + @property + def redis_db(self): + if self.__class__.__redis_db is None: + self.__class__.__redis_db = RedisDB() + + return self.__class__.__redis_db + + def load_pipelines(self): + pipelines = [] + for pipeline_path in setting.ITEM_PIPELINES: + pipeline = tools.import_cls(pipeline_path)() + if not isinstance(pipeline, BasePipeline): + raise ValueError(f"{pipeline_path} 需继承 feapder.pipelines.BasePipeline") + pipelines.append(pipeline) + + return pipelines + + @property + def mysql_pipeline(self): + if not self._mysql_pipeline: + self._mysql_pipeline = tools.import_cls(MYSQL_PIPELINE_PATH)() + + return self._mysql_pipeline def run(self): + self._thread_stop = False while not self._thread_stop: self.flush() - tools.delay_time(0.5) + tools.delay_time(setting.ITEM_UPLOAD_INTERVAL) self.close() def stop(self): self._thread_stop = True + self._started.clear() def put_item(self, item): + if isinstance(item, Item): + # 入库前的回调 + item.pre_to_db() + self._items_queue.put(item) def flush(self): @@ -115,7 +151,7 @@ def flush(self): else: # request-redis requests.append(data) - if data_count >= UPLOAD_BATCH_MAX_SIZE: + if data_count >= setting.ITEM_UPLOAD_BATCH_MAX_SIZE: self.__add_item_to_db( items, update_items, requests, callbacks, items_fingerprints ) @@ -185,10 +221,10 @@ def __pick_items(self, items, is_update_item=False): 将每个表之间的数据分开 拆分后 原items为空 @param items: @param is_update_item: - @return: + @return: 表名与数据的字典 """ datas_dict = { - # 'xxx:xxx_item': [{}, {}] redis 中的item名与对应的数据 + # 'table_name': [{}, {}] } while items: @@ -196,95 +232,76 @@ def __pick_items(self, items, is_update_item=False): # 取item下划线格式的名 # 下划线类的名先从dict中取，没有则现取，然后存入dict。加快下次取的速度 item_name = item.item_name - item_table = self._item_tables.get(item_name) - if not item_table: - item_name_underline = item.name_underline - tab_item = self._table_item.format( - redis_key=self._redis_key, item_name=item_name_underline - ) - - item_table = {} - item_table["tab_item"] = tab_item - - self._item_tables[item_name] = item_table + table_name = self._item_tables.get(item_name) + if not table_name: + table_name = item.table_name + self._item_tables[item_name] = table_name + self._item_pipelines[table_name] = item.pipelines - else: - tab_item = item_table.get("tab_item") - - # 入库前的回调 - item.per_to_db() + if is_update_item and table_name not in self._item_update_keys: + self._item_update_keys[table_name] = item.update_key - if tab_item not in datas_dict: - datas_dict[tab_item] = [] + if table_name not in datas_dict: + datas_dict[table_name] = [] - datas_dict[tab_item].append(item.to_dict) - - if is_update_item and tab_item not in self._item_update_keys: - self._item_update_keys[tab_item] = item.update_key + datas_dict[table_name].append(item.to_dict) return datas_dict - def __export_to_db(self, tab_item, datas, is_update=False, update_keys=()): - export_success = False - # 打点 校验 - to_table = tools.get_info(tab_item, ":s_(.*?)_item$", fetch_one=True) - item_name = to_table + "_item" - self.check_datas(table=to_table, datas=datas) - - if setting.ADD_ITEM_TO_MYSQL: # 任务表需要入mysql - if isinstance(setting.ADD_ITEM_TO_MYSQL, (list, tuple)): - for item in setting.ADD_ITEM_TO_MYSQL: - if item in item_name: - export_success = ( - self._export_data.export_items(tab_item, datas) - if not is_update - else self._export_data.update_items( - tab_item, datas, update_keys=update_keys - ) - ) + def __export_to_db(self, table, datas, is_update=False, update_keys=(), used_pipelines=None): + pipelines = used_pipelines or self._pipelines # 优先采用指定的pipelines + for pipeline in pipelines: + if is_update: + if table == self._task_table and not isinstance( + pipeline, MysqlPipeline + ): + continue + + if not pipeline.update_items(table, datas, update_keys=update_keys): + log.error( + f"{pipeline.__class__.__name__} 更新数据失败. table: {table} items: {datas}" + ) + return False else: - export_success = ( - self._export_data.export_items(tab_item, datas) - if not is_update - else self._export_data.update_items( - tab_item, datas, update_keys=update_keys + if not pipeline.save_items(table, datas): + log.error( + f"{pipeline.__class__.__name__} 保存数据失败. table: {table} items: {datas}" ) + return False + + # 若是任务表, 且上面的pipeline里没mysql，则需调用mysql更新任务 + if not self._have_mysql_pipeline and is_update and table == self._task_table: + if not self.mysql_pipeline.update_items( + table, datas, update_keys=update_keys + ): + log.error( + f"{self.mysql_pipeline.__class__.__name__} 更新数据失败. table: {table} items: {datas}" ) + return False - if setting.ADD_ITEM_TO_REDIS: - if isinstance(setting.ADD_ITEM_TO_REDIS, (list, tuple)): - for item in setting.ADD_ITEM_TO_REDIS: - if item in item_name: - self._db.sadd(tab_item, datas) - export_success = True - log.info("共导出 %s 条数据 到redis %s" % (len(datas), tab_item)) - break - - else: - self._db.sadd(tab_item, datas) - export_success = True - log.info("共导出 %s 条数据 到redis %s" % (len(datas), tab_item)) - - return export_success + self.metric_datas(table=table, datas=datas) + return True def __add_item_to_db( - self, items, update_items, requests, callbacks, items_fingerprints + self, items, update_items, requests, callbacks, items_fingerprints ): - export_success = False + export_success = True self._is_adding_to_db = True # 去重 if setting.ITEM_FILTER_ENABLE: items, items_fingerprints = self.__dedup_items(items, items_fingerprints) - # 分捡 + # 分捡（返回值包含 pipelines_dict） items_dict = self.__pick_items(items) update_items_dict = self.__pick_items(update_items, is_update_item=True) # item批量入库 + failed_items = {"add": [], "update": [], "requests": []} while items_dict: - tab_item, datas = items_dict.popitem() + table, datas = items_dict.popitem() + used_pipelines = self._item_pipelines.get(table) log.debug( """ @@ -292,55 +309,129 @@ def __add_item_to_db( 表名: %s datas: %s """ - % (tab_item, tools.dumps_json(datas, indent=16)) + % (table, tools.dumps_json(datas, indent=16)) ) - export_success = self.__export_to_db(tab_item, datas) + if not self.__export_to_db(table, datas, used_pipelines=used_pipelines): + export_success = False + failed_items["add"].append({"table": table, "datas": datas}) # 执行批量update while update_items_dict: - tab_item, datas = update_items_dict.popitem() + table, datas = update_items_dict.popitem() + used_pipelines = self._item_pipelines.get(table) + log.debug( """ -------------- item 批量更新 -------------- 表名: %s datas: %s """ - % (tab_item, tools.dumps_json(datas, indent=16)) + % (table, tools.dumps_json(datas, indent=16)) ) - update_keys = self._item_update_keys.get(tab_item) - export_success = self.__export_to_db( - tab_item, datas, is_update=True, update_keys=update_keys - ) + update_keys = self._item_update_keys.get(table) + if not self.__export_to_db( + table, datas, is_update=True, update_keys=update_keys, used_pipelines=used_pipelines + ): + export_success = False + failed_items["update"].append( + {"table": table, "datas": datas, "update_keys": update_keys} + ) - # 执行回调 - while callbacks: - try: - callback = callbacks.pop(0) - callback() - except Exception as e: - log.exception(e) + if export_success: + # 执行回调 + while callbacks: + try: + callback = callbacks.pop(0) + callback() + except Exception as e: + log.exception(e) + + # 删除做过的request + if requests: + self.redis_db.zrem(self._table_request, requests) + + # 去重入库 + if setting.ITEM_FILTER_ENABLE: + if items_fingerprints: + self.__class__.dedup.add(items_fingerprints, skip_check=True) + else: + failed_items["requests"] = requests - # 删除做过的request - if requests: - self._db.zrem(self._table_request, requests) + if self.export_retry_times > setting.EXPORT_DATA_MAX_RETRY_TIMES: + if self._redis_key != "air_spider": + # 失败的item记录到redis + self.redis_db.sadd(self._table_failed_items, failed_items) - # 去重入库 - if export_success and setting.ITEM_FILTER_ENABLE: - if items_fingerprints: - self.__class__.dedup.add(items_fingerprints, skip_check=True) + # 删除做过的request + if requests: + self.redis_db.zrem(self._table_request, requests) + + log.error( + "入库超过最大重试次数，不再重试，数据记录到redis，items:\n {}".format( + tools.dumps_json(failed_items) + ) + ) + self.export_retry_times = 0 + + else: + tip = ["入库不成功"] + if callbacks: + tip.append("不执行回调") + if requests: + tip.append("不删除任务") + exists = self.redis_db.zexists(self._table_request, requests) + for exist, request in zip(exists, requests): + if exist: + self.redis_db.zadd(self._table_request, requests, 300) + + if setting.ITEM_FILTER_ENABLE: + tip.append("数据不入去重库") + + if self._redis_key != "air_spider": + tip.append("将自动重试") + + tip.append("失败items:\n {}".format(tools.dumps_json(failed_items))) + log.error("，".join(tip)) + + self.export_falied_times += 1 + + if self._redis_key != "air_spider": + self.export_retry_times += 1 + + if self.export_falied_times > setting.EXPORT_DATA_MAX_FAILED_TIMES: + # 报警 + msg = "《{}》爬虫导出数据失败，失败次数：{}，请检查爬虫是否正常".format( + self._redis_key, self.export_falied_times + ) + log.error(msg) + tools.send_msg( + msg=msg, + level="error", + message_prefix="《%s》爬虫导出数据失败" % (self._redis_key), + ) self._is_adding_to_db = False - def check_datas(self, table, datas): + def metric_datas(self, table, datas): """ 打点 记录总条数及每个key情况 @param table: 表名 @param datas: 数据 列表 @return: """ - pass + total_count = 0 + for data in datas: + total_count += 1 + for k, v in data.items(): + metrics.emit_counter(k, int(bool(v)), classify=table) + metrics.emit_counter("total count", total_count, classify=table) def close(self): - pass + # 调用pipeline的close方法 + for pipeline in self._pipelines: + try: + pipeline.close() + except: + pass diff --git a/feapder/buffer/request_buffer.py b/feapder/buffer/request_buffer.py index 8b3c86d5..70677a94 100644 --- a/feapder/buffer/request_buffer.py +++ b/feapder/buffer/request_buffer.py @@ -5,7 +5,7 @@ @summary: request 管理器， 负责缓冲添加到数据库中的request --------- @author: Boris -@email: boris@bzkj.tech +@email: boris_liu@foxmail.com """ import collections @@ -13,49 +13,67 @@ import feapder.setting as setting import feapder.utils.tools as tools +from feapder.db.memorydb import MemoryDB from feapder.db.redisdb import RedisDB -from feapder.utils.log import log from feapder.dedup import Dedup +from feapder.utils.log import log MAX_URL_COUNT = 1000 # 缓存中最大request数 -class Singleton(object): - def __new__(cls, *args, **kwargs): - if not hasattr(cls, "_inst"): - cls._inst = super(Singleton, cls).__new__(cls) - - return cls._inst - - -class RequestBuffer(threading.Thread, Singleton): +class AirSpiderRequestBuffer: dedup = None - def __init__(self, redis_key): - if not hasattr(self, "_requests_deque"): - super(RequestBuffer, self).__init__() + def __init__(self, db=None, dedup_name: str = None): + self._db = db or MemoryDB() - self._thread_stop = False - self._is_adding_to_db = False + if not self.__class__.dedup and setting.REQUEST_FILTER_ENABLE: + if setting.REQUEST_FILTER_SETTING.get( + "filter_type" + ) == Dedup.BloomFilter or setting.REQUEST_FILTER_SETTING.get("name"): + self.__class__.dedup = Dedup( + to_md5=False, **setting.REQUEST_FILTER_SETTING + ) + else: + self.__class__.dedup = Dedup( + to_md5=False, name=dedup_name, **setting.REQUEST_FILTER_SETTING + ) + + def is_exist_request(self, request): + if ( + request.filter_repeat + and setting.REQUEST_FILTER_ENABLE + and not self.__class__.dedup.add(request.fingerprint) + ): + log.debug("request已存在 url = %s" % request.url) + return True + return False + + def put_request(self, request, ignore_max_size=True): + if self.is_exist_request(request): + return + else: + self._db.add(request, ignore_max_size=ignore_max_size) + + +class RequestBuffer(AirSpiderRequestBuffer, threading.Thread): + def __init__(self, redis_key): + AirSpiderRequestBuffer.__init__(self, db=RedisDB(), dedup_name=redis_key) + threading.Thread.__init__(self) - self._requests_deque = collections.deque() - self._del_requests_deque = collections.deque() - self._db = RedisDB() + self._thread_stop = False + self._is_adding_to_db = False - self._table_request = setting.TAB_REQUSETS.format(redis_key=redis_key) - self._table_failed_request = setting.TAB_FAILED_REQUSETS.format( - redis_key=redis_key - ) + self._requests_deque = collections.deque() + self._del_requests_deque = collections.deque() - if not self.__class__.dedup and setting.REQUEST_FILTER_ENABLE: - self.__class__.dedup = Dedup( - filter_type=Dedup.ExpireFilter, - name=redis_key, - expire_time=2592000, - to_md5=False, - ) # 过期时间为一个月 + self._table_request = setting.TAB_REQUESTS.format(redis_key=redis_key) + self._table_failed_request = setting.TAB_FAILED_REQUESTS.format( + redis_key=redis_key + ) def run(self): + self._thread_stop = False while not self._thread_stop: try: self.__add_request_to_db() @@ -66,6 +84,7 @@ def run(self): def stop(self): self._thread_stop = True + self._started.clear() def put_request(self, request): self._requests_deque.append(request) @@ -118,12 +137,7 @@ def __add_request_to_db(self): priority = request.priority # 如果需要去重并且库中已重复 则continue - if ( - request.filter_repeat - and setting.REQUEST_FILTER_ENABLE - and not self.__class__.dedup.add(request.fingerprint) - ): - log.debug("request已存在 url = %s" % request.url) + if self.is_exist_request(request): continue else: request_list.append(str(request.to_dict)) diff --git a/feapder/commands/cmdline.py b/feapder/commands/cmdline.py index 52970dd3..91d0531e 100644 --- a/feapder/commands/cmdline.py +++ b/feapder/commands/cmdline.py @@ -5,41 +5,111 @@ @summary: --------- @author: Boris -@email: boris@bzkj.tech +@email: boris_liu@foxmail.com """ +import re import sys from os.path import dirname, join +import os + +import requests from feapder.commands import create_builder +from feapder.commands import retry from feapder.commands import shell +from feapder.commands import zip + +HELP = """ +███████╗███████╗ █████╗ ██████╗ ██████╗ ███████╗██████╗ +██╔════╝██╔════╝██╔══██╗██╔══██╗██╔══██╗██╔════╝██╔══██╗ +█████╗ █████╗ ███████║██████╔╝██║ ██║█████╗ ██████╔╝ +██╔══╝ ██╔══╝ ██╔══██║██╔═══╝ ██║ ██║██╔══╝ ██╔══██╗ +██║ ███████╗██║ ██║██║ ██████╔╝███████╗██║ ██║ +╚═╝ ╚══════╝╚═╝ ╚═╝╚═╝ ╚═════╝ ╚══════╝╚═╝ ╚═╝ + +Version: {version} +Document: https://feapder.com + +Usage: + feapder [options] [args] + +Available commands: +""" + +NEW_VERSION_TIP = """ +────────────────────────────────────────────────────── +New version available \033[31m{version}\033[0m → \033[32m{new_version}\033[0m +Run \033[33mpip install --upgrade feapder\033[0m to update! +""" + +with open(join(dirname(dirname(__file__)), "VERSION"), "rb") as f: + VERSION = f.read().decode("ascii").strip() def _print_commands(): - with open(join(dirname(dirname(__file__)), "VERSION"), "rb") as f: - version = f.read().decode("ascii").strip() - - print("feapder {}".format(version)) - print("\nUsage:") - print(" feapder [options] [args]\n") - print("Available commands:") - cmds = {"create": "create project、spider、item and so on", "shell": "debug response"} + print(HELP.rstrip().format(version=VERSION)) + cmds = { + "create": "create project、spider、item and so on", + "shell": "debug response", + "zip": "zip project", + "retry": "retry failed request or item", + } for cmdname, cmdclass in sorted(cmds.items()): print(" %-13s %s" % (cmdname, cmdclass)) print('\nUse "feapder -h" to see more info about a command') +def check_new_version(): + try: + url = "https://pypi.org/simple/feapder/" + resp = requests.get(url, timeout=3, verify=False) + html = resp.text + + last_stable_version = re.findall(r"feapder-([\d.]*?).tar.gz", html)[-1] + now_version = VERSION + now_stable_version = re.sub("-beta.*", "", VERSION) + + if now_stable_version < last_stable_version or ( + now_stable_version == last_stable_version and "beta" in now_version + ): + new_version = f"feapder=={last_stable_version}" + if new_version: + version = f"feapder=={VERSION.replace('-beta', 'b')}" + tip = NEW_VERSION_TIP.format(version=version, new_version=new_version) + # 修复window下print不能带颜色输出的问题 + if os.name == "nt": + os.system("") + print(tip) + except Exception as e: + pass + + def execute(): - args = sys.argv - if len(args) < 2: - _print_commands() - return - - command = args.pop(1) - if command == "create": - create_builder.main() - elif command == "shell": - shell.main() - else: - _print_commands() + try: + args = sys.argv + if len(args) < 2: + _print_commands() + check_new_version() + return + + command = args.pop(1) + if command == "create": + create_builder.main() + elif command == "shell": + shell.main() + elif command == "zip": + zip.main() + elif command == "retry": + retry.main() + else: + _print_commands() + except KeyboardInterrupt: + pass + + check_new_version() + + +if __name__ == "__main__": + execute() diff --git a/feapder/commands/create/__init__.py b/feapder/commands/create/__init__.py index 6dfaae03..490035fa 100644 --- a/feapder/commands/create/__init__.py +++ b/feapder/commands/create/__init__.py @@ -5,6 +5,9 @@ "CreateInit", "CreateJson", "CreateTable", + "CreateCookies", + "CreateSetting", + "CreateParams", ] from .create_table import CreateTable @@ -13,3 +16,6 @@ from .create_init import CreateInit from .create_item import CreateItem from .create_project import CreateProject +from .create_cookies import CreateCookies +from .create_setting import CreateSetting +from .create_params import CreateParams diff --git a/feapder/commands/create/create_cookies.py b/feapder/commands/create/create_cookies.py new file mode 100644 index 00000000..68fc8fa2 --- /dev/null +++ b/feapder/commands/create/create_cookies.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- +""" +Created on 2021/4/25 10:22 上午 +--------- +@summary: 将浏览器的cookie转为request的cookie +--------- +@author: Boris +@email: boris_liu@foxmail.com +""" + +import json + +import pyperclip + +from feapder.utils.tools import get_cookies_from_str, print_pretty + + +class CreateCookies: + def get_data(self): + """ + @summary: 从剪切板中读取内容 + --------- + --------- + @result: + """ + input("请复制浏览器cookie (列表或字符串格式), 复制后按任意键读取剪切板内容\n") + + text = pyperclip.paste() + print(text + "\n") + + return text + + def create(self): + data = self.get_data() + cookies = {} + try: + data_json = json.loads(data) + + for data in data_json: + cookies[data.get("name")] = data.get("value") + + except: + cookies = get_cookies_from_str(data) + + print_pretty(cookies) diff --git a/feapder/commands/create/create_item.py b/feapder/commands/create/create_item.py index 3f0b58e0..d8726381 100644 --- a/feapder/commands/create/create_item.py +++ b/feapder/commands/create/create_item.py @@ -12,13 +12,14 @@ import os import feapder.utils.tools as tools +from feapder import setting from feapder.db.mysqldb import MysqlDB from .create_init import CreateInit def deal_file_info(file): file = file.replace("{DATE}", tools.get_current_date()) - file = file.replace("{USER}", getpass.getuser()) + file = file.replace("{USER}", os.getenv("FEAPDER_USER") or getpass.getuser()) return file @@ -30,9 +31,7 @@ def __init__(self): def select_columns(self, table_name): # sql = 'SHOW COLUMNS FROM ' + table_name - sql = "SELECT COLUMN_NAME, COLUMN_TYPE, IS_NULLABLE, COLUMN_DEFAULT, EXTRA, COLUMN_KEY, COLUMN_COMMENT FROM INFORMATION_SCHEMA.Columns WHERE table_name = '{}'".format( - table_name - ) + sql = f"SELECT COLUMN_NAME, COLUMN_TYPE, IS_NULLABLE, COLUMN_DEFAULT, EXTRA, COLUMN_KEY, COLUMN_COMMENT FROM INFORMATION_SCHEMA.Columns WHERE table_name = '{table_name}' and table_schema = '{setting.MYSQL_DB}'" columns = self._db.find(sql) return columns @@ -45,10 +44,7 @@ def select_tables_name(self, tables_name): --------- @result: """ - sql = ( - "select table_name from information_schema.tables where table_name like '%s'" - % tables_name - ) + sql = f"select table_name from information_schema.tables where table_name like '{tables_name}' and table_schema = '{setting.MYSQL_DB}'" tables_name = self._db.find(sql) return tables_name @@ -69,10 +65,15 @@ def convert_table_name_to_hump(self, table_name): return table_hump_format - def get_item_template(self): - template_path = os.path.abspath( - os.path.join(__file__, "../../../templates/item_template.tmpl") - ) + def get_item_template(self, item_type): + if item_type == "Item": + template_path = os.path.abspath( + os.path.join(__file__, "../../../templates/item_template.tmpl") + ) + else: + template_path = os.path.abspath( + os.path.join(__file__, "../../../templates/update_item_template.tmpl") + ) with open(template_path, "r", encoding="utf-8") as file: item_template = file.read() @@ -83,9 +84,10 @@ def create_item(self, item_template, columns, table_name, support_dict): # 组装 类名 item_template = item_template.replace("${item_name}", table_name_hump_format) if support_dict: - item_template = item_template.replace("${table_name}", table_name + " 1") + item_template = item_template.replace("${command}", table_name + " 1") else: - item_template = item_template.replace("${table_name}", table_name) + item_template = item_template.replace("${command}", table_name) + item_template = item_template.replace("${table_name}", table_name) # 组装 属性 propertys = "" @@ -99,6 +101,7 @@ def create_item(self, item_template, columns, table_name, support_dict): column_comment = column[6] try: + column_default = None if column_default == "NULL" else column_default value = ( "kwargs.get('{column_name}')".format(column_name=column_name) if support_dict @@ -118,52 +121,17 @@ def create_item(self, item_template, columns, table_name, support_dict): ) if column_extra == "auto_increment" or column_default is not None: - propertys += ( - "# self.{column_name} = {value} # type : {column_type} | allow_null : {is_nullable} | key : {column_key} | default_value : {column_default} | extra : {column_extra} | column_comment : {column_comment}".format( - column_name=column_name, - value=value, - column_type=column_type, - is_nullable=is_nullable, - column_key=column_key, - column_default=column_default, - column_extra=column_extra, - column_comment=column_comment, - ) - + "\n" - + " " * 8 - ) + propertys += f"# self.{column_name} = {value}" else: if value is None or isinstance(value, (float, int)) or support_dict: - propertys += ( - "self.{column_name} = {value} # type : {column_type} | allow_null : {is_nullable} | key : {column_key} | default_value : {column_default} | extra : {column_extra}| column_comment : {column_comment}".format( - column_name=column_name, - value=value, - column_type=column_type, - is_nullable=is_nullable, - column_key=column_key, - column_default=column_default, - column_extra=column_extra, - column_comment=column_comment, - ) - + "\n" - + " " * 8 - ) + propertys += f"self.{column_name} = {value}" else: - propertys += ( - "self.{column_name} = '{value}' # type : {column_type} | allow_null : {is_nullable} | key : {column_key} | default_value : {column_default} | extra : {column_extra}| column_comment : {column_comment}".format( - column_name=column_name, - value=value, - column_type=column_type, - is_nullable=is_nullable, - column_key=column_key, - column_default=column_default, - column_extra=column_extra, - column_comment=column_comment, - ) - + "\n" - + " " * 8 - ) + propertys += f"self.{column_name} = '{value}'" + + if column_comment: + propertys += f" # {column_comment}" + propertys += "\n" + " " * 8 item_template = item_template.replace("${propertys}", propertys.strip()) item_template = deal_file_info(item_template) @@ -182,9 +150,10 @@ def save_template_to_file(self, item_template, table_name): file.write(item_template) print("\n%s 生成成功" % item_file) - self._create_init.create() + if os.path.basename(os.path.dirname(os.path.abspath(item_file))) == "items": + self._create_init.create() - def create(self, tables_name, support_dict): + def create(self, tables_name, item_type, support_dict): input_tables_name = tables_name tables_name = self.select_tables_name(tables_name) @@ -197,7 +166,7 @@ def create(self, tables_name, support_dict): table_name = table_name[0] columns = self.select_columns(table_name) - item_template = self.get_item_template() + item_template = self.get_item_template(item_type) item_template = self.create_item( item_template, columns, table_name, support_dict ) diff --git a/feapder/commands/create/create_json.py b/feapder/commands/create/create_json.py index 0232e1a3..204f4423 100644 --- a/feapder/commands/create/create_json.py +++ b/feapder/commands/create/create_json.py @@ -8,7 +8,7 @@ @email: boris_liu@foxmail.com """ -import sys +import pyperclip import feapder.utils.tools as tools @@ -21,10 +21,14 @@ def get_data(self): --------- @result: """ - print("请输入需要转换的内容： （xxx:xxx格式，支持多行）") + input("请复制需要转换的内容（xxx:xxx格式，支持多行），复制后按任意键读取剪切板内容\n") + + text = pyperclip.paste() + print(text + "\n") + data = [] - while True: - line = sys.stdin.readline().strip().replace("\t", " " * 4) + for line in text.split("\n"): + line = line.strip().replace("\t", " " * 4) if not line: break @@ -49,4 +53,4 @@ def create(self, sort_keys=False): else: json[result[0]] = result[1].strip() - print(tools.dumps_json(json, sort_keys=sort_keys)) \ No newline at end of file + print(tools.dumps_json(json, sort_keys=sort_keys)) diff --git a/feapder/commands/create/create_params.py b/feapder/commands/create/create_params.py new file mode 100644 index 00000000..05ddce1e --- /dev/null +++ b/feapder/commands/create/create_params.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +""" +Created on 2021/4/25 10:22 上午 +--------- +@summary: 将浏览器的cookie转为request的cookie +--------- +@author: Boris +@email: boris_liu@foxmail.com +""" + +import sys + +from feapder.utils.tools import dumps_json + + +class CreateParams: + def get_data(self): + """ + @summary: 从控制台读取多行 + --------- + --------- + @result: + """ + print("请输入请求地址") + data = [] + while True: + line = sys.stdin.readline().strip() + if not line: + break + + data.append(line) + + return "".join(data) + + def get_params(self, url): + params_json = {} + params = url.split("?")[-1].split("&") + for param in params: + key_value = param.split("=", 1) + params_json[key_value[0]] = key_value[1] + + return params_json + + def create(self): + data = self.get_data() + + params = self.get_params(data) + url = data.split("?")[0] + + print(f'url = "{url}"') + print(f"params = {dumps_json(params)}") diff --git a/feapder/commands/create/create_project.py b/feapder/commands/create/create_project.py index 83d9576a..c500f6af 100644 --- a/feapder/commands/create/create_project.py +++ b/feapder/commands/create/create_project.py @@ -17,7 +17,7 @@ def deal_file_info(file): file = file.replace("{DATE}", tools.get_current_date()) - file = file.replace("{USER}", getpass.getuser()) + file = file.replace("{USER}", os.getenv("FEAPDER_USER") or getpass.getuser()) return file diff --git a/feapder/commands/create/create_setting.py b/feapder/commands/create/create_setting.py new file mode 100644 index 00000000..f80c6d0a --- /dev/null +++ b/feapder/commands/create/create_setting.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +""" +Created on 2021/4/23 13:20 +--------- +@summary: 生成配置文件 +--------- +@author: mkdir700 +@email: mkdir700@gmail.com +""" + +import os +import shutil + + +class CreateSetting: + def create(self): + if os.path.exists("setting.py"): + confirm = input("配置文件已存在 是否覆盖 (y/n). ") + if confirm != "y": + print("取消覆盖 退出") + return + + template_file_path = os.path.abspath( + os.path.join(__file__, "../../../templates/project_template/setting.py") + ) + shutil.copy(template_file_path, "./", follow_symlinks=False) + print("配置文件生成成功") diff --git a/feapder/commands/create/create_spider.py b/feapder/commands/create/create_spider.py index 8c923dd2..f464e059 100644 --- a/feapder/commands/create/create_spider.py +++ b/feapder/commands/create/create_spider.py @@ -18,7 +18,7 @@ def deal_file_info(file): file = file.replace("{DATE}", tools.get_current_date()) - file = file.replace("{USER}", getpass.getuser()) + file = file.replace("{USER}", os.getenv("FEAPDER_USER") or getpass.getuser()) return file @@ -49,14 +49,16 @@ def cover_to_underline(self, key): return key def get_spider_template(self, spider_type): - if spider_type == 1: + if spider_type == "AirSpider": template_path = "air_spider_template.tmpl" - elif spider_type == 2: + elif spider_type == "Spider": template_path = "spider_template.tmpl" - elif spider_type == 3: + elif spider_type == "TaskSpider": + template_path = "task_spider_template.tmpl" + elif spider_type == "BatchSpider": template_path = "batch_spider_template.tmpl" else: - raise ValueError("spider type error, support 1 2 3") + raise ValueError("spider type error, only support AirSpider、 Spider、TaskSpider、BatchSpider") template_path = os.path.abspath( os.path.join(__file__, "../../../templates", template_path) @@ -66,30 +68,38 @@ def get_spider_template(self, spider_type): return spider_template - def create_spider(self, spider_template, spider_name): + def create_spider(self, spider_template, spider_name, file_name): spider_template = spider_template.replace("${spider_name}", spider_name) + spider_template = spider_template.replace("${file_name}", file_name) spider_template = deal_file_info(spider_template) return spider_template - def save_spider_to_file(self, spider, spider_name): - spider_underline = self.cover_to_underline(spider_name) - spider_file = spider_underline + ".py" - - if os.path.exists(spider_file): - confirm = input("%s 文件已存在 是否覆盖 (y/n). " % spider_file) + def save_spider_to_file(self, spider, spider_name, file_name): + if os.path.exists(file_name): + confirm = input("%s 文件已存在 是否覆盖 (y/n). " % file_name) if confirm != "y": print("取消覆盖 退出") return - with open(spider_file, "w", encoding="utf-8") as file: + with open(file_name, "w", encoding="utf-8") as file: file.write(spider) print("\n%s 生成成功" % spider_name) - self._create_init.create() + if os.path.basename(os.path.dirname(os.path.abspath(file_name))) == "spiders": + self._create_init.create() def create(self, spider_name, spider_type): - if spider_name.islower(): - spider_name = tools.key2hump(spider_name) + # 检查spider_name + if not re.search("^[a-zA-Z][a-zA-Z0-9_]*$", spider_name): + print("爬虫命名不符合规范，请用蛇形或驼峰命名方式") + return + + underline_format = self.cover_to_underline(spider_name) + spider_name = tools.key2hump(underline_format) + file_name = underline_format + ".py" + + print(spider_name, file_name) + spider_template = self.get_spider_template(spider_type) - spider = self.create_spider(spider_template, spider_name) - self.save_spider_to_file(spider, spider_name) + spider = self.create_spider(spider_template, spider_name, file_name) + self.save_spider_to_file(spider, spider_name, file_name) diff --git a/feapder/commands/create/create_table.py b/feapder/commands/create/create_table.py index e4ebddda..15162782 100644 --- a/feapder/commands/create/create_table.py +++ b/feapder/commands/create/create_table.py @@ -8,9 +8,10 @@ @email: boris_liu@foxmail.com """ -import sys import time +import pyperclip + import feapder.setting as setting import feapder.utils.tools as tools from feapder.db.mysqldb import MysqlDB @@ -21,7 +22,7 @@ class CreateTable: def __init__(self): self._db = MysqlDB() - def is_vaild_date(self, date): + def is_valid_date(self, date): try: if ":" in date: time.strptime(date, "%Y-%m-%d %H:%M:%S") @@ -32,26 +33,24 @@ def is_vaild_date(self, date): return False def get_key_type(self, value): - try: - value = eval(value) - except: - value = value - - key_type = "varchar(255)" if isinstance(value, int): key_type = "int" elif isinstance(value, float): key_type = "double" elif isinstance(value, str): - if self.is_vaild_date(value): + if self.is_valid_date(value): if ":" in value: key_type = "datetime" else: key_type = "date" - elif len(value) > 255: + elif len(value) > 50: key_type = "text" else: key_type = "varchar(255)" + elif isinstance(value, (dict, list)): + key_type = "longtext" + else: + key_type = "varchar(255)" return key_type @@ -62,18 +61,15 @@ def get_data(self): --------- @result: """ - data = "" - while True: - line = sys.stdin.readline().strip() - if not line: - break - data += line + input("请复制json格式数据, 复制后按任意键读取剪切板内容\n") + + text = pyperclip.paste() + print(text + "\n") - return tools.get_json(data) + return tools.get_json(text) def create(self, table_name): # 输入表字段 - print('请输入表数据 json格式 如 {"name":"张三"}\n等待输入：\n') data = self.get_data() if not isinstance(data, dict): @@ -82,54 +78,73 @@ def create(self, table_name): # 拼接表结构 sql = """ CREATE TABLE `{db}`.`{table_name}` ( - `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'id 自动递增', + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'id主键', {other_key} - `gtime` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '抓取时间', - PRIMARY KEY (`id`), + `crawl_time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '采集时间', {unique} + PRIMARY KEY (`id`) ) COMMENT=''; """ - print("请设置注释 回车跳过") + # print("请设置注释 回车跳过") other_key = "" for key, value in data.items(): key = key2underline(key) + comment = "" + if key == "id": + key = "data_id" + comment = "原始数据id" + key_type = self.get_key_type(value) - comment = input("%s : %s -> comment：" % (key, key_type)) + # comment = input("%s : %s -> comment：" % (key, key_type)) - other_key += "`{key}` {key_type} COMMENT '{comment}',\n ".format( - key=key, key_type=key_type, comment=comment + other_key += ( + "`{key}` {key_type} COMMENT '{comment}',\n ".format( + key=key, key_type=key_type, comment=comment + ) ) print("\n") while True: - is_need_batch_date = input("是否添加batch_date 字段 （y/n）:") - if is_need_batch_date == "y": - other_key += "`{key}` {key_type} COMMENT '{comment}',\n ".format( - key="batch_date", key_type="date", comment="批次时间" + yes = input("是否添加批次字段 batch_date（y/n）:") + if yes == "y": + other_key += ( + "`{key}` {key_type} COMMENT '{comment}',\n ".format( + key="batch_date", key_type="date", comment="批次时间" + ) ) break - elif is_need_batch_date == "n": + elif yes == "n": break print("\n") while True: - unique = input("请设置唯一索引, 多个逗号间隔\n等待输入：\n").replace("，", ",") - if unique: + yes = input("是否设置唯一索引（y/n）:") + if yes == "y": + unique = input("请设置唯一索引, 多个逗号间隔\n等待输入：\n").replace("，", ",") + if unique: + unique = "UNIQUE `idx` USING BTREE (`%s`) comment ''," % "`,`".join( + unique.split(",") + ) + break + elif yes == "n": + unique = "" break - unique = "UNIQUE `idx` USING BTREE (`%s`) comment ''" % "`,`".join( - unique.split(",") - ) sql = sql.format( db=setting.MYSQL_DB, table_name=table_name, - other_key=other_key, + other_key=other_key.strip(), unique=unique, ) print(sql) - self._db.execute(sql) - print("\n%s 创建成功" % table_name) \ No newline at end of file + result=self._db.execute(sql) + # 建立表成功。受影响的行数为 0，因此返回0 + if result==0: + print("\n%s 创建成功" % table_name) + print("注意手动检查下字段类型，确保无误！！！") + else: + print("\n%s 创建失败" % table_name) diff --git a/feapder/commands/create_builder.py b/feapder/commands/create_builder.py index a4274c77..dec0ba05 100644 --- a/feapder/commands/create_builder.py +++ b/feapder/commands/create_builder.py @@ -8,6 +8,10 @@ @email: boris_liu@foxmail.com """ import argparse + +from terminal_layout import Fore +from terminal_layout.extensions.choice import Choice, StringStyle + import feapder.setting as setting from feapder.commands.create import * @@ -16,26 +20,18 @@ def main(): spider = argparse.ArgumentParser(description="生成器") spider.add_argument( - "-p", "--project", help="创建项目 如 feapder create -s ", metavar="" + "-p", "--project", help="创建项目 如 feapder create -p ", metavar="" ) spider.add_argument( "-s", "--spider", - nargs="+", - help="创建爬虫\n" - "如 feapder create -s " - "spider_type=1 AirSpider; " - "spider_type=2 Spider; " - "spider_type=3 BatchSpider;", + help="创建爬虫 如 feapder create -s ", metavar="", ) spider.add_argument( "-i", "--item", - nargs="+", - help="创建item 如 feapder create -i test 则生成test表对应的item。 " - "支持like语法模糊匹配所要生产的表。 " - "若想生成支持字典方式赋值的item，则create -item test 1", + help="创建item 如 feapder create -i 支持模糊匹配 如 feapder create -i %%table_name%%", metavar="", ) spider.add_argument( @@ -46,6 +42,11 @@ def main(): ) spider.add_argument("-j", "--json", help="创建json", action="store_true") spider.add_argument("-sj", "--sort_json", help="创建有序json", action="store_true") + spider.add_argument("-c", "--cookies", help="创建cookie", action="store_true") + spider.add_argument("--params", help="解析地址中的参数", action="store_true") + spider.add_argument( + "--setting", help="创建全局配置文件" "feapder create --setting", action="store_true" + ) # 指定数据库 spider.add_argument("--host", type=str, help="mysql 连接地址", metavar="") @@ -53,7 +54,6 @@ def main(): spider.add_argument("--username", type=str, help="mysql 用户名", metavar="") spider.add_argument("--password", type=str, help="mysql 密码", metavar="") spider.add_argument("--db", type=str, help="mysql 数据库名", metavar="") - args = spider.parse_args() if args.host: @@ -68,21 +68,35 @@ def main(): setting.MYSQL_DB = args.db if args.item: - item_name, *support_dict = args.item - support_dict = bool(support_dict) - CreateItem().create(item_name, support_dict) + c = Choice( + "请选择Item类型", + ["Item", "Item 支持字典赋值", "UpdateItem", "UpdateItem 支持字典赋值"], + icon_style=StringStyle(fore=Fore.green), + selected_style=StringStyle(fore=Fore.green), + ) + + choice = c.get_choice() + if choice: + index, value = choice + item_name = args.item + item_type = "Item" if index <= 1 else "UpdateItem" + support_dict = index in (1, 3) + + CreateItem().create(item_name, item_type, support_dict) elif args.spider: - spider_name, *spider_type = args.spider - if not spider_type: - spider_type = 1 - else: - spider_type = spider_type[0] - try: - spider_type = int(spider_type) - except: - raise ValueError("spider_type error, support 1, 2, 3") - CreateSpider().create(spider_name, spider_type) + c = Choice( + "请选择爬虫模板", + ["AirSpider", "Spider", "TaskSpider", "BatchSpider"], + icon_style=StringStyle(fore=Fore.green), + selected_style=StringStyle(fore=Fore.green), + ) + + choice = c.get_choice() + if choice: + index, spider_type = choice + spider_name = args.spider + CreateSpider().create(spider_name, spider_type) elif args.project: CreateProject().create(args.project) @@ -99,6 +113,18 @@ def main(): elif args.sort_json: CreateJson().create(sort_keys=True) + elif args.cookies: + CreateCookies().create() + + elif args.setting: + CreateSetting().create() + + elif args.params: + CreateParams().create() + + else: + spider.print_help() + if __name__ == "__main__": main() diff --git a/feapder/commands/retry.py b/feapder/commands/retry.py new file mode 100644 index 00000000..19a86f32 --- /dev/null +++ b/feapder/commands/retry.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- +""" +Created on 2022/11/18 12:33 PM +--------- +@summary: +--------- +@author: Boris +@email: boris_liu@foxmail.com +""" +import argparse + +from feapder.core.handle_failed_items import HandleFailedItems +from feapder.core.handle_failed_requests import HandleFailedRequests + + +def retry_failed_requests(redis_key): + handle_failed_requests = HandleFailedRequests(redis_key) + handle_failed_requests.reput_failed_requests_to_requests() + + +def retry_failed_items(redis_key): + handle_failed_items = HandleFailedItems(redis_key) + handle_failed_items.reput_failed_items_to_db() + handle_failed_items.close() + + +def parse_args(): + parser = argparse.ArgumentParser( + description="重试失败的请求或入库失败的item", + usage="usage: feapder retry [options] [args]", + ) + parser.add_argument( + "-r", + "--request", + help="重试失败的request 如 feapder retry --request ", + metavar="", + ) + parser.add_argument( + "-i", "--item", help="重试失败的item 如 feapder retry --item ", metavar="" + ) + args = parser.parse_args() + return args + + +def main(): + args = parse_args() + if args.request: + retry_failed_requests(args.request) + if args.item: + retry_failed_items(args.item) + + +if __name__ == "__main__": + main() diff --git a/feapder/commands/shell.py b/feapder/commands/shell.py index b2298daa..37483799 100644 --- a/feapder/commands/shell.py +++ b/feapder/commands/shell.py @@ -5,16 +5,145 @@ @summary: --------- @author: Boris -@email: boris@bzkj.tech +@email: boris_liu@foxmail.com """ -import json +import argparse import re +import shlex import sys import IPython +import pyperclip from feapder import Request +from feapder.utils import tools + + +def parse_curl(curl_str): + parser = argparse.ArgumentParser(description="") + parser.add_argument("target_url", type=str, nargs="?") + parser.add_argument("-X", "--request", type=str, nargs=1, default="") + parser.add_argument("-H", "--header", nargs=1, action="append", default=[]) + parser.add_argument("-d", "--data", nargs=1, action="append", default=[]) + parser.add_argument("--data-ascii", nargs=1, action="append", default=[]) + parser.add_argument("--data-binary", nargs=1, action="append", default=[]) + parser.add_argument("--data-urlencode", nargs=1, action="append", default=[]) + parser.add_argument("--data-raw", nargs=1, action="append", default=[]) + parser.add_argument("-F", "--form", nargs=1, action="append", default=[]) + parser.add_argument("--digest", action="store_true") + parser.add_argument("--ntlm", action="store_true") + parser.add_argument("--anyauth", action="store_true") + parser.add_argument("-e", "--referer", type=str) + parser.add_argument("-G", "--get", action="store_true", default=False) + parser.add_argument("-I", "--head", action="store_true") + parser.add_argument("-k", "--insecure", action="store_true") + parser.add_argument("-o", "--output", type=str) + parser.add_argument("-O", "--remote_name", action="store_true") + parser.add_argument("-r", "--range", type=str) + parser.add_argument("-u", "--user", type=str) + parser.add_argument("--url", type=str) + parser.add_argument("-A", "--user-agent", type=str) + parser.add_argument("--compressed", action="store_true", default=False) + + curl_split = shlex.split(curl_str) + try: + args = parser.parse_known_args(curl_split[1:])[0] + except: + raise ValueError("Could not parse arguments.") + + # 请求地址 + url = args.target_url + + # # 请求方法 + # try: + # method = args.request.lower() + # except AttributeError: + # method = args.request[0].lower() + + # 请求头 + headers = { + h[0].split(":", 1)[0]: ("".join(h[0].split(":", 1)[1]).strip()) + for h in args.header + } + if args.user_agent: + headers["User-Agent"] = args.user_agent + if args.referer: + headers["Referer"] = args.referer + if args.range: + headers["Range"] = args.range + + # Cookie + cookie_str = headers.pop("Cookie", "") or headers.pop("cookie", "") + cookies = tools.get_cookies_from_str(cookie_str) if cookie_str else {} + + # params + url, params = tools.parse_url_params(url) + + # data + data = "".join( + [ + "".join(d) + for d in args.data + + args.data_ascii + + args.data_binary + + args.data_raw + + args.form + ] + ) + if data: + data = re.sub(r"^\$", "", data) + + # method + if args.head: + method = "head" + elif args.get: + method = "get" + params.update(data) + elif args.request: + method = ( + args.request[0].lower() + if isinstance(args.request, list) + else args.request.lower() + ) + elif data: + method = "post" + else: + method = "get" + params.update(data) + + username = None + password = None + if args.user: + u = args.user + if ":" in u: + username, password = u.split(":") + else: + username = u + password = input(f"请输入用户{username}的密码") + + auth = None + if args.digest: + auth = "digest" + elif args.ntlm: + auth = "ntlm" + elif username: + auth = "basic" + + insecure = args.insecure + + return dict( + url=url, + method=method, + cookies=cookies, + headers=headers, + params=params, + data=data, + insecure=insecure, + username=username, + password=password, + auth=auth, + ) def request(**kwargs): @@ -29,64 +158,54 @@ def fetch_url(url): request(url=url) -def fetch_curl(curl_args): - """ - 解析及抓取curl请求 - :param curl_args: - [url, '-H', 'xxx', '-H', 'xxx', '--data-binary', '{"xxx":"xxx"}', '--compressed'] - :return: - """ - url = curl_args[0] - curl_args.pop(0) - - headers = {} - data = {} - for i in range(0, len(curl_args), 2): - if curl_args[i] == "-H": - regex = "([^:\s]*)[:|\s]*(.*)" - result = re.search(regex, curl_args[i + 1], re.S).groups() - if result[0] in headers: - headers[result[0]] = headers[result[0]] + "&" + result[1] - else: - headers[result[0]] = result[1].strip() - - elif curl_args[i] == "--data-binary": - data = json.loads(curl_args[i + 1]) - - request(url=url, data=data, headers=headers) +def fetch_curl(): + input("请复制请求为cURL (bash)，复制后按任意键读取剪切板内容\n") + curl = pyperclip.paste() + if curl: + kwargs = parse_curl(curl) + request(**kwargs) def usage(): """ -下载调试器 + 下载调试器 -usage: feapder shell [options] [args] + usage: feapder shell [options] [args] -optional arguments: - -u, --url 抓取指定url - -c, --curl 抓取curl格式的请求 + optional arguments: + -u, --url 抓取指定url + -c, --curl 抓取curl格式的请求 """ print(usage.__doc__) sys.exit() -def main(): - args = sys.argv - if len(args) < 3: - usage() - - elif args[1] in ("-h", "--help"): - usage() +def parse_args(): + parser = argparse.ArgumentParser( + description="测试请求", + usage="usage: feapder shell [options] [args]", + ) + parser.add_argument( + "-u", + "--url", + help="请求指定地址, 如 feapder shell --url http://www.spidertools.cn/", + metavar="", + ) + parser.add_argument("-c", "--curl", help="执行curl，调试响应", action="store_true") - elif args[1] in ("-u", "--url"): - fetch_url(args[2]) + args = parser.parse_args() + return parser, args - elif args[1] in ("-c", "--curl"): - fetch_curl(args[2:]) +def main(): + parser, args = parse_args() + if args.url: + fetch_url(args.url) + elif args.curl: + fetch_curl() else: - usage() + parser.print_help() if __name__ == "__main__": diff --git a/feapder/commands/zip.py b/feapder/commands/zip.py new file mode 100644 index 00000000..bb604f2e --- /dev/null +++ b/feapder/commands/zip.py @@ -0,0 +1,75 @@ +# -*- coding: utf-8 -*- +""" +Created on 2022/2/13 12:59 上午 +--------- +@summary: +--------- +@author: Boris +@email: boris_liu@foxmail.com +""" + +import argparse +import os +import re +import zipfile + + +def is_ignore_file(ignore_files: list, filename): + for ignore_file in ignore_files: + if re.search(ignore_file, filename): + return True + return False + + +def zip(dir_path, zip_name, ignore_dirs: list = None, ignore_files: list = None): + print(f"正在压缩 {dir_path} >> {zip_name}") + ignore_files.append(os.path.basename(zip_name)) + with zipfile.ZipFile(zip_name, "w") as file: + dir_name = os.path.basename(dir_path) + parent_dir = os.path.dirname(dir_path) + if parent_dir: + os.chdir(parent_dir) + for path, dirs, filenames in os.walk(dir_name): + # 修改原dirs，方式遍历忽略文件夹里的文件 + if ignore_dirs: + dirs[:] = [d for d in dirs if d not in ignore_dirs] + for filename in filenames: + if ignore_files and is_ignore_file(ignore_files, filename): + continue + + filepath = os.path.join(path, filename) + print(f" adding {filepath}") + file.write(filepath) + + print(f"压缩成功 {dir_path} >> {zip_name}") + + +def parse_args(): + parser = argparse.ArgumentParser( + description="压缩文件夹, 默认排除以下文件夹及文件 .git,__pycache__,.idea,venv,.DS_Store", + usage="feapder zip dir_path [zip_name]", + ) + parser.add_argument("dir_path", type=str, help="文件夹路径") + parser.add_argument("zip_name", type=str, nargs="?", help="压缩后的文件名，默认为文件夹名.zip") + parser.add_argument("-i", help="忽略文件，逗号分隔，支持正则", metavar="") + parser.add_argument("-I", help="忽略文件夹，逗号分隔，支持正则 ", metavar="") + parser.add_argument("-o", help="输出路径，默认为当前目录", metavar="") + + args = parser.parse_args() + return args + + +def main(): + ignore_dirs = [".git", "__pycache__", ".idea", "venv", "env"] + ignore_files = [".DS_Store"] + args = parse_args() + if args.i: + ignore_files.extend(args.i.split(",")) + if args.I: + ignore_dirs.extend(args.I.split(",")) + dir_path = args.dir_path + zip_name = args.zip_name or os.path.basename(dir_path) + ".zip" + if args.o: + zip_name = os.path.join(args.o, os.path.basename(zip_name)) + + zip(dir_path, zip_name, ignore_dirs=ignore_dirs, ignore_files=ignore_files) diff --git a/feapder/core/__init__.py b/feapder/core/__init__.py index 6c03df55..ff422dde 100644 --- a/feapder/core/__init__.py +++ b/feapder/core/__init__.py @@ -5,5 +5,5 @@ @summary: --------- @author: Boris -@email: boris@bzkj.tech +@email: boris_liu@foxmail.com ''' \ No newline at end of file diff --git a/feapder/core/base_parser.py b/feapder/core/base_parser.py index cf9906be..a06f9c44 100644 --- a/feapder/core/base_parser.py +++ b/feapder/core/base_parser.py @@ -13,6 +13,9 @@ from feapder.db.mysqldb import MysqlDB from feapder.network.item import UpdateItem from feapder.utils.log import log +from feapder.network.request import Request +from feapder.network.response import Response +from feapder.utils.perfect_dict import PerfectDict class BaseParser(object): @@ -26,47 +29,66 @@ def start_requests(self): pass - def parse(self, request, response): + def download_midware(self, request: Request): """ - @summary: 默认的回调函数 + @summary: 下载中间件 可修改请求的一些参数, 或可自定义下载，然后返回 request, response + --------- + @param request: + --------- + @result: return request / request, response + """ + + pass + + def validate(self, request: Request, response: Response): + """ + @summary: 校验函数, 可用于校验response是否正确 + 若函数内抛出异常，则重试请求 + 若返回True 或 None，则进入解析函数 + 若返回False，则抛弃当前请求 + 可通过request.callback_name 区分不同的回调函数，编写不同的校验逻辑 --------- @param request: @param response: --------- - @result: + @result: True / None / False """ pass - def download_midware(self, request): + def parse(self, request: Request, response: Response): """ - @summary: 下载中间件 可修改请求的一些参数 + @summary: 默认的解析函数 --------- @param request: + @param response: --------- - @result: return request / None (不会修改原来的request) + @result: """ pass - def exception_request(self, request, response): + def exception_request(self, request: Request, response: Response, e: Exception): """ @summary: 请求或者parser里解析出异常的request --------- @param request: @param response: + @param e: 异常 --------- @result: request / callback / None (返回值必须可迭代) """ pass - def failed_request(self, request, response): + def failed_request(self, request: Request, response: Response, e: Exception): """ @summary: 超过最大重试次数的request 可返回修改后的request 若不返回request，则将传进来的request直接人redis的failed表。否则将修改后的request入failed表 --------- @param request: + @param response: + @param e: 异常 --------- @result: request / item / callback / None (返回值必须可迭代) """ @@ -101,26 +123,12 @@ def close(self): pass -class BatchParser(BaseParser): - """ - @summary: 批次爬虫模版 - --------- - """ - - def __init__( - self, - task_table, - batch_record_table, - task_state, - date_format, - mysqldb=None, - ): +class TaskParser(BaseParser): + def __init__(self, task_table, task_state, mysqldb=None): self._mysqldb = mysqldb or MysqlDB() # mysqldb - self._task_table = task_table # mysql中的任务表 - self._batch_record_table = batch_record_table # mysql 中的批次记录表 self._task_state = task_state # mysql中任务表的state字段名 - self._date_format = date_format # 批次日期格式 + self._task_table = task_table # mysql中的任务表 def add_task(self): """ @@ -130,7 +138,7 @@ def add_task(self): @result: """ - def start_requests(self, task): + def start_requests(self, task: PerfectDict): """ @summary: --------- @@ -162,6 +170,8 @@ def update_task_state(self, task_id, state=1, **kwargs): else: log.error("置任务%s状态失败 sql=%s" % (task_id, sql)) + update_task = update_task_state + def update_task_batch(self, task_id, state=1, **kwargs): """ 批量更新任务 多处调用，更新的字段必须一致 @@ -180,6 +190,22 @@ def update_task_batch(self, task_id, state=1, **kwargs): return update_item + +class BatchParser(TaskParser): + """ + @summary: 批次爬虫模版 + --------- + """ + + def __init__( + self, task_table, batch_record_table, task_state, date_format, mysqldb=None + ): + super(BatchParser, self).__init__( + task_table=task_table, task_state=task_state, mysqldb=mysqldb + ) + self._batch_record_table = batch_record_table # mysql 中的批次记录表 + self._date_format = date_format # 批次日期格式 + @property def batch_date(self): """ diff --git a/feapder/core/collector.py b/feapder/core/collector.py index 271bc4b4..5b8ff652 100644 --- a/feapder/core/collector.py +++ b/feapder/core/collector.py @@ -5,12 +5,12 @@ @summary: request 管理 --------- @author: Boris -@email: boris@bzkj.tech +@email: boris_liu@foxmail.com """ -import collections import threading import time +from queue import Queue, Empty import feapder.setting as setting import feapder.utils.tools as tools @@ -18,16 +18,13 @@ from feapder.network.request import Request from feapder.utils.log import log -LOCAL_HOST_IP = tools.get_localhost_ip() - class Collector(threading.Thread): - def __init__(self, redis_key, process_num=None): + def __init__(self, redis_key): """ @summary: --------- @param redis_key: - @param process_num: 进程编号 --------- @result: """ @@ -37,91 +34,50 @@ def __init__(self, redis_key, process_num=None): self._thread_stop = False - self._todo_requests = collections.deque() - - self._tab_requests = setting.TAB_REQUSETS.format(redis_key=redis_key) - self._tab_spider_status = setting.TAB_SPIDER_STATUS.format( - redis_key=redis_key - ) - - self._spider_mark = LOCAL_HOST_IP + ( - "_%s" % process_num if process_num else "_0" - ) - - self._interval = setting.COLLECTOR_SLEEP_TIME - self._request_count = setting.COLLECTOR_TASK_COUNT + self._todo_requests = Queue(maxsize=setting.COLLECTOR_TASK_COUNT) + self._tab_requests = setting.TAB_REQUESTS.format(redis_key=redis_key) self._is_collector_task = False - self._db.clear(self._tab_spider_status) - def run(self): + self._thread_stop = False while not self._thread_stop: - try: self.__input_data() except Exception as e: log.exception(e) + time.sleep(0.1) self._is_collector_task = False - time.sleep(self._interval) - def stop(self): self._thread_stop = True + self._started.clear() def __input_data(self): - if len(self._todo_requests) >= self._request_count: - return - - # 汇报节点信息 - self._db.zadd(self._tab_spider_status, self._spider_mark, 0) # 未做 - - request_count = self._request_count # 先赋值 - # 根据等待节点数量，动态分配request - spider_wait_count = self._db.zget_count( - self._tab_spider_status, priority_min=0, priority_max=0 - ) - if spider_wait_count: - # 任务数量 - task_count = self._db.zget_count(self._tab_requests) - # 动态分配的数量 = 任务数量 / 休息的节点数量 + 1 - request_count = task_count // spider_wait_count + 1 - - request_count = ( - request_count - if request_count <= self._request_count - else self._request_count - ) - - if not request_count: + if setting.COLLECTOR_TASK_COUNT / setting.SPIDER_THREAD_COUNT > 1 and ( + self._todo_requests.qsize() > setting.SPIDER_THREAD_COUNT + or self._todo_requests.qsize() >= self._todo_requests.maxsize + ): + time.sleep(0.1) return - # requests_list = self._db.zget(self._tab_requests, count = request_count) - - # 取任务 current_timestamp = tools.get_current_timestamp() - priority_max = current_timestamp - setting.REQUEST_TIME_OUT # 普通的任务 与 已经超时的任务 + + # 取任务，只取当前时间戳以内的任务，同时将任务分数修改为 current_timestamp + setting.REQUEST_LOST_TIMEOUT requests_list = self._db.zrangebyscore_set_score( self._tab_requests, priority_min="-inf", - priority_max=priority_max, - score=current_timestamp, - count=request_count, + priority_max=current_timestamp, + score=current_timestamp + setting.REQUEST_LOST_TIMEOUT, + count=setting.COLLECTOR_TASK_COUNT, ) - # print('取任务', len(requests_list)) - if not requests_list: - pass - else: + if requests_list: self._is_collector_task = True - # 将取到的任务放回到redis， 以当前时间戳标记，表示正在做的任务。任务做完在request_buffer中删除，没做完则到超时时间后重新做 - # self._db.zadd(self._tab_requests, requests_list, prioritys=current_timestamp) - - # 汇报节点信息 - self._db.zadd(self._tab_spider_status, self._spider_mark, 1) # 正在做 - # 存request self.__put_requests(requests_list) + else: + time.sleep(0.1) def __put_requests(self, requests_list): for request in requests_list: @@ -142,19 +98,19 @@ def __put_requests(self, requests_list): request_dict = None if request_dict: - self._todo_requests.append(request_dict) - - def get_requests(self, count): - requests = [] - count = count if count <= len(self._todo_requests) else len(self._todo_requests) - while count: - requests.append(self._todo_requests.popleft()) - count -= 1 + self._todo_requests.put(request_dict) - return requests + def get_request(self): + try: + request = self._todo_requests.get(timeout=1) + return request + except Empty as e: + return None def get_requests_count(self): - return len(self._todo_requests) or self._db.zget_count(self._tab_requests) + return ( + self._todo_requests.qsize() or self._db.zget_count(self._tab_requests) or 0 + ) def is_collector_task(self): return self._is_collector_task diff --git a/feapder/core/handle_failed_items.py b/feapder/core/handle_failed_items.py new file mode 100644 index 00000000..655330f5 --- /dev/null +++ b/feapder/core/handle_failed_items.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- +""" +Created on 2022/11/18 11:33 AM +--------- +@summary: +--------- +@author: Boris +@email: boris_liu@foxmail.com +""" +import feapder.setting as setting +from feapder.buffer.item_buffer import ItemBuffer +from feapder.db.redisdb import RedisDB +from feapder.network.item import Item, UpdateItem +from feapder.utils.log import log + + +class HandleFailedItems: + def __init__(self, redis_key, task_table=None, item_buffer=None): + if redis_key.endswith(":s_failed_items"): + redis_key = redis_key.replace(":s_failed_items", "") + + self._redisdb = RedisDB() + self._item_buffer = item_buffer or ItemBuffer(redis_key, task_table=task_table) + + self._table_failed_items = setting.TAB_FAILED_ITEMS.format(redis_key=redis_key) + + def get_failed_items(self, count=1): + failed_items = self._redisdb.sget( + self._table_failed_items, count=count, is_pop=False + ) + return failed_items + + def reput_failed_items_to_db(self): + log.debug("正在重新写入失败的items...") + total_count = 0 + while True: + try: + failed_items = self.get_failed_items() + if not failed_items: + break + + for data_str in failed_items: + data = eval(data_str) + + for add in data.get("add"): + table = add.get("table") + datas = add.get("datas") + for _data in datas: + item = Item(**_data) + item.table_name = table + self._item_buffer.put_item(item) + total_count += 1 + + for update in data.get("update"): + table = update.get("table") + datas = update.get("datas") + update_keys = update.get("update_keys") + for _data in datas: + item = UpdateItem(**_data) + item.table_name = table + item.update_key = update_keys + self._item_buffer.put_item(item) + total_count += 1 + + # 入库成功后删除 + def delete_item(): + self._redisdb.srem(self._table_failed_items, data_str) + + self._item_buffer.put_item(delete_item) + self._item_buffer.flush() + + except Exception as e: + log.exception(e) + + if total_count: + log.debug("导入%s条失败item到数库" % total_count) + else: + log.debug("没有失败的item") + + def close(self): + self._item_buffer.close() diff --git a/feapder/core/handle_failed_requests.py b/feapder/core/handle_failed_requests.py index 4a5ae667..3c1cc880 100644 --- a/feapder/core/handle_failed_requests.py +++ b/feapder/core/handle_failed_requests.py @@ -14,22 +14,20 @@ from feapder.utils.log import log -class HandleFailedRequests(object): - """docstring for HandleFailedRequests""" - +class HandleFailedRequests: def __init__(self, redis_key): - super(HandleFailedRequests, self).__init__() - self._redis_key = redis_key + if redis_key.endswith(":z_failed_requests"): + redis_key = redis_key.replace(":z_failed_requests", "") self._redisdb = RedisDB() - self._request_buffer = RequestBuffer(self._redis_key) + self._request_buffer = RequestBuffer(redis_key) - self._table_failed_request = setting.TAB_FAILED_REQUSETS.format( + self._table_failed_request = setting.TAB_FAILED_REQUESTS.format( redis_key=redis_key ) - def get_failed_requests(self, count=100): - failed_requests = self._redisdb.zget(self._table_failed_request, count=10000) + def get_failed_requests(self, count=10000): + failed_requests = self._redisdb.zget(self._table_failed_request, count=count) failed_requests = [eval(failed_request) for failed_request in failed_requests] return failed_requests @@ -37,18 +35,20 @@ def reput_failed_requests_to_requests(self): log.debug("正在重置失败的requests...") total_count = 0 while True: - failed_requests = self.get_failed_requests() - if not failed_requests: - break + try: + failed_requests = self.get_failed_requests() + if not failed_requests: + break - for request in failed_requests: - request["retry_times"] = 0 - request_obj = Request.from_dict(request) - self._request_buffer.put_request(request_obj) + for request in failed_requests: + request["retry_times"] = 0 + request_obj = Request.from_dict(request) + self._request_buffer.put_request(request_obj) - total_count += 1 + total_count += 1 + except Exception as e: + log.exception(e) self._request_buffer.flush() log.debug("重置%s条失败requests为待抓取requests" % total_count) - diff --git a/feapder/core/parser_control.py b/feapder/core/parser_control.py index cca9decf..021d2956 100644 --- a/feapder/core/parser_control.py +++ b/feapder/core/parser_control.py @@ -5,34 +5,43 @@ @summary: parser 控制类 --------- @author: Boris -@email: boris@bzkj.tech +@email: boris_liu@foxmail.com """ +import inspect +import random import threading import time -from collections import Iterable +from collections.abc import Iterable import feapder.setting as setting import feapder.utils.tools as tools -from feapder.db.memory_db import MemoryDB +from feapder.buffer.item_buffer import ItemBuffer +from feapder.buffer.request_buffer import AirSpiderRequestBuffer +from feapder.core.base_parser import BaseParser +from feapder.db.memorydb import MemoryDB from feapder.network.item import Item from feapder.network.request import Request +from feapder.utils import metrics from feapder.utils.log import log -class PaserControl(threading.Thread): +class ParserControl(threading.Thread): DOWNLOAD_EXCEPTION = "download_exception" DOWNLOAD_SUCCESS = "download_success" DOWNLOAD_TOTAL = "download_total" - PAESERS_EXCEPTION = "parsers_exception" + PAESERS_EXCEPTION = "parser_exception" is_show_tip = False # 实时统计已做任务数及失败任务数，若失败任务数/已做任务数>0.5 则报警 _success_task_count = 0 _failed_task_count = 0 + _total_task_count = 0 + + _hook_parsers = set() def __init__(self, collector, redis_key, request_buffer, item_buffer): - super(PaserControl, self).__init__() + super(ParserControl, self).__init__() self._parsers = [] self._collector = collector self._redis_key = redis_key @@ -41,25 +50,19 @@ def __init__(self, collector, redis_key, request_buffer, item_buffer): self._thread_stop = False - self._wait_task_time = 0 - def run(self): + self._thread_stop = False while not self._thread_stop: try: - requests = self._collector.get_requests(setting.SPIDER_TASK_COUNT) - if not requests: + request = self._collector.get_request() + if not request: if not self.is_show_tip: - log.info("parser 等待任务 ...") + log.debug("等待任务...") self.is_show_tip = True - - # log.info('parser 等待任务 {}...'.format(tools.format_seconds(self._wait_task_time))) - - time.sleep(1) - self._wait_task_time += 1 continue self.is_show_tip = False - self.deal_requests(requests) + self.deal_request(request) except Exception as e: log.exception(e) @@ -69,31 +72,43 @@ def is_not_task(self): @classmethod def get_task_status_count(cls): - return cls._failed_task_count, cls._success_task_count - - def deal_requests(self, requests): - for request in requests: - - response = None - request_redis = request["request_redis"] - request = request["request_obj"] - - del_request_redis_after_item_to_db = False - del_request_redis_after_request_to_db = False - - for parser in self._parsers: - if parser.name == request.parser_name: - used_download_midware_enable = False - try: - # 记录需下载的文档 - self.record_download_status( - PaserControl.DOWNLOAD_TOTAL, parser.name - ) - - # 解析request - if request.auto_request: - request_temp = None - if request.download_midware: + return cls._failed_task_count, cls._success_task_count, cls._total_task_count + + def deal_request(self, request): + response = None + request_redis = request["request_redis"] + request = request["request_obj"] + + del_request_redis_after_item_to_db = False + del_request_redis_after_request_to_db = False + + for parser in self._parsers: + if parser.name == request.parser_name: + used_download_midware_enable = False + try: + self.__class__._total_task_count += 1 + # 记录需下载的文档 + self.record_download_status( + ParserControl.DOWNLOAD_TOTAL, parser.name + ) + + # 解析request + if request.auto_request: + request_temp = None + response = None + + # 下载中间件 + if request.download_midware: + if isinstance(request.download_midware, (list, tuple)): + request_temp = request + for download_midware in request.download_midware: + download_midware = ( + download_midware + if callable(download_midware) + else tools.get_method(parser, download_midware) + ) + request_temp = download_midware(request_temp) + else: download_midware = ( request.download_midware if callable(request.download_midware) @@ -102,17 +117,25 @@ def deal_requests(self, requests): ) ) request_temp = download_midware(request) - elif request.download_midware != False: - request_temp = parser.download_midware(request) - - if request_temp: - if not isinstance(request_temp, Request): - raise Exception( - "download_midware need return a request, but received type: {}".format( - type(request_temp) - ) + elif request.download_midware != False: + request_temp = parser.download_midware(request) + + # 请求 + if request_temp: + if ( + isinstance(request_temp, (tuple, list)) + and len(request_temp) == 2 + ): + request_temp, response = request_temp + + if not isinstance(request_temp, Request): + raise Exception( + "download_midware need return a request, but received type: {}".format( + type(request_temp) ) - used_download_midware_enable = True + ) + used_download_midware_enable = True + if response is None: response = ( request_temp.get_response() if not setting.RESPONSE_CACHED_USED @@ -120,108 +143,79 @@ def deal_requests(self, requests): save_cached=False ) ) - else: - response = ( - request.get_response() - if not setting.RESPONSE_CACHED_USED - else request.get_response_from_cached( - save_cached=False - ) - ) - - if response == None: - raise Exception( - "连接超时 url: %s" % (request.url or request_temp.url) - ) - else: - response = None - - if request.callback: # 如果有parser的回调函数，则用回调处理 - callback_parser = ( - request.callback - if callable(request.callback) - else tools.get_method(parser, request.callback) + response = ( + request.get_response() + if not setting.RESPONSE_CACHED_USED + else request.get_response_from_cached(save_cached=False) ) - results = callback_parser(request, response) - else: # 否则默认用parser处理 - results = parser.parse(request, response) - if results and not isinstance(results, Iterable): + if response == None: raise Exception( - "%s.%s返回值必须可迭代" - % (parser.name, request.callback or "parser") + "连接超时 url: %s" % (request.url or request_temp.url) ) - # 标识上一个result是什么 - result_type = 0 # 0\1\2 (初始值\request\item) - # 此处判断是request 还是 item - for result in results or []: - if isinstance(result, Request): - result_type = 1 - # 给request的 parser_name 赋值 - result.parser_name = result.parser_name or parser.name - - # 判断是同步的callback还是异步的 - if result.request_sync: # 同步 - request_dict = { - "request_obj": result, - "request_redis": None, - } - requests.append(request_dict) - else: # 异步 - # 将next_request 入库 - self._request_buffer.put_request(result) - del_request_redis_after_request_to_db = True + # 校验 + if parser.validate(request, response) == False: + break - elif isinstance(result, Item): - result_type = 2 - # 将item入库 - self._item_buffer.put_item(result) - # 需删除正在做的request - del_request_redis_after_item_to_db = True + else: + response = None - elif callable(result): # result为可执行的无参函数 - if ( - result_type == 2 - ): # item 的 callback，buffer里的item均入库后再执行 - self._item_buffer.put_item(result) - del_request_redis_after_item_to_db = True + if request.callback: # 如果有parser的回调函数，则用回调处理 + callback_parser = ( + request.callback + if callable(request.callback) + else tools.get_method(parser, request.callback) + ) + results = callback_parser(request, response) + else: # 否则默认用parser处理 + results = parser.parse(request, response) - else: # result_type == 1: # request 的 callback，buffer里的request均入库后再执行。可能有的parser直接返回callback - self._request_buffer.put_request(result) - del_request_redis_after_request_to_db = True + if results and not isinstance(results, Iterable): + raise Exception( + "%s.%s返回值必须可迭代" % (parser.name, request.callback or "parse") + ) - # else: - # raise TypeError('Expect Request、Item、callback func, bug get type: {}'.format(type(result))) + # 标识上一个result是什么 + result_type = 0 # 0\1\2 (初始值\request\item) + # 此处判断是request 还是 item + for result in results or []: + if isinstance(result, Request): + result_type = 1 + # 给request的 parser_name 赋值 + result.parser_name = result.parser_name or parser.name + + # 判断是同步的callback还是异步的 + if result.request_sync: # 同步 + request_dict = { + "request_obj": result, + "request_redis": None, + } + self.deal_request(request_dict) + else: # 异步 + # 将next_request 入库 + self._request_buffer.put_request(result) + del_request_redis_after_request_to_db = True - except Exception as e: - exception_type = ( - str(type(e)).replace("", "") - ) - if exception_type.startswith("requests"): - # 记录下载失败的文档 - self.record_download_status( - PaserControl.DOWNLOAD_EXCEPTION, parser.name - ) + elif isinstance(result, Item): + result_type = 2 + # 将item入库 + self._item_buffer.put_item(result) + # 需删除正在做的request + del_request_redis_after_item_to_db = True - else: - # 记录解析程序异常 - self.record_download_status( - PaserControl.PAESERS_EXCEPTION, parser.name - ) + elif callable(result): # result为可执行的无参函数 + if result_type == 2: # item 的 callback，buffer里的item均入库后再执行 + self._item_buffer.put_item(result) + del_request_redis_after_item_to_db = True - if setting.LOG_LEVEL == "DEBUG": # 只有debug模式下打印， 超时的异常篇幅太多 - log.exception(e) + else: # result_type == 1: # request 的 callback，buffer里的request均入库后再执行。可能有的parser直接返回callback + self._request_buffer.put_request(result) + del_request_redis_after_request_to_db = True - log.error( - """ - -------------- %s.%s error ------------- - error %s - response %s - deal request %s - """ - % ( + elif result is not None: + function_name = "{}.{}".format( parser.name, ( request.callback @@ -229,219 +223,301 @@ def deal_requests(self, requests): and getattr(request.callback, "__name__") or request.callback ) - or "parser", - str(e), - response, - tools.dumps_json(request.to_dict, indent=28) - if setting.LOG_LEVEL == "DEBUG" - else request, + or "parse", + ) + raise TypeError( + f"{function_name} result expect Request、Item or callback, bug get type: {type(result)}" ) - ) - request.error_msg = "%s: %s" % (exception_type, e) - request.response = str(response) + except Exception as e: + exception_type = ( + str(type(e)).replace("", "") + ) + if exception_type.startswith("requests"): + # 记录下载失败的文档 + self.record_download_status( + ParserControl.DOWNLOAD_EXCEPTION, parser.name + ) + if request.retry_times % setting.PROXY_MAX_FAILED_TIMES == 0: + request.del_proxy() - if "Invalid URL" in str(e): - request.is_abandoned = True + else: + # 记录解析程序异常 + self.record_download_status( + ParserControl.PAESERS_EXCEPTION, parser.name + ) - requests = parser.exception_request(request, response) or [ - request - ] - if not isinstance(requests, Iterable): - raise Exception( - "%s.%s返回值必须可迭代" % (parser.name, "exception_request") + if setting.LOG_LEVEL == "DEBUG": # 只有debug模式下打印， 超时的异常篇幅太多 + log.exception(e) + + log.error( + """ + -------------- %s.%s error ------------- + error %s + response %s + deal request %s + """ + % ( + parser.name, + ( + request.callback + and callable(request.callback) + and getattr(request.callback, "__name__") + or request.callback ) - for request in requests: - if callable(request): - self._request_buffer.put_request(request) - continue + or "parse", + str(e), + response, + tools.dumps_json(request.to_dict, indent=28) + if setting.LOG_LEVEL == "DEBUG" + else request, + ) + ) - if not isinstance(request, Request): - raise Exception("exception_request 需return request") + request.error_msg = "%s: %s" % (exception_type, e) + request.response = str(response) - if ( - request.retry_times + 1 > setting.SPIDER_MAX_RETRY_TIMES - or request.is_abandoned - ): - self.__class__._failed_task_count += 1 # 记录失败任务数 - - # 处理failed_request的返回值 request 或 func - results = parser.failed_request(request, response) or [ - request - ] - if not isinstance(results, Iterable): - raise Exception( - "%s.%s返回值必须可迭代" - % (parser.name, "failed_request") - ) + if "Invalid URL" in str(e): + request.is_abandoned = True - for result in results: - if isinstance(result, Request): - if setting.SAVE_FAILED_REQUEST: - if used_download_midware_enable: - # 去掉download_midware 添加的属性 - original_request = ( - Request.from_dict( - eval(request_redis) - ) - if request_redis - else result - ) - original_request.error_msg = ( - request.error_msg - ) - original_request.response = ( - request.response - ) - - self._request_buffer.put_failed_request( - original_request - ) - else: - self._request_buffer.put_failed_request( - result - ) - - elif callable(result): - self._request_buffer.put_request(result) - - elif isinstance(result, Item): - self._item_buffer.put_item(result) + requests = parser.exception_request(request, response, e) or [ + request + ] + if not isinstance(requests, Iterable): + raise Exception( + "%s.%s返回值必须可迭代" % (parser.name, "exception_request") + ) + for request in requests: + if callable(request): + self._request_buffer.put_request(request) + continue + + if not isinstance(request, Request): + raise Exception("exception_request 需 yield request") + + if ( + request.retry_times + 1 > setting.SPIDER_MAX_RETRY_TIMES + or request.is_abandoned + ): + self.__class__._failed_task_count += 1 # 记录失败任务数 + + # 处理failed_request的返回值 request 或 func + results = parser.failed_request(request, response, e) or [ + request + ] + if not isinstance(results, Iterable): + raise Exception( + "%s.%s返回值必须可迭代" % (parser.name, "failed_request") + ) - del_request_redis_after_request_to_db = True + for result in results: + if isinstance(result, Request): + if setting.SAVE_FAILED_REQUEST: + if used_download_midware_enable: + # 去掉download_midware 添加的属性 + original_request = ( + Request.from_dict(eval(request_redis)) + if request_redis + else result + ) + original_request.error_msg = ( + request.error_msg + ) + original_request.response = request.response + + self._request_buffer.put_failed_request( + original_request + ) + else: + self._request_buffer.put_failed_request( + result + ) + + elif callable(result): + self._request_buffer.put_request(result) - else: - # 将 requests 重新入库 爬取 - request.retry_times += 1 - request.filter_repeat = False - log.info( - """ - 入库 等待重试 - url %s - 重试次数 %s - 最大允许重试次数 %s""" - % ( - request.url, - request.retry_times, - setting.SPIDER_MAX_RETRY_TIMES, - ) - ) - if used_download_midware_enable: - # 去掉download_midware 添加的属性 使用原来的requests - original_request = ( - Request.from_dict(eval(request_redis)) - if request_redis - else request - ) - if hasattr(request, "error_msg"): - original_request.error_msg = request.error_msg - if hasattr(request, "response"): - original_request.response = request.response - original_request.retry_times = request.retry_times - original_request.filter_repeat = ( - request.filter_repeat - ) + elif isinstance(result, Item): + self._item_buffer.put_item(result) - self._request_buffer.put_request(original_request) - else: - self._request_buffer.put_request(request) - del_request_redis_after_request_to_db = True + del_request_redis_after_request_to_db = True - else: - # 记录下载成功的文档 - self.record_download_status( - PaserControl.DOWNLOAD_SUCCESS, parser.name - ) - # 记录成功任务数 - self.__class__._success_task_count += 1 - - # 缓存下载成功的文档 - if setting.RESPONSE_CACHED_ENABLE: - request.save_cached( - response=response, - expire_time=setting.RESPONSE_CACHED_EXPIRE_TIME, + else: + # 将 requests 重新入库 爬取 + request.retry_times += 1 + request.filter_repeat = False + log.info( + """ + 入库 等待重试 + url %s + 重试次数 %s + 最大允许重试次数 %s""" + % ( + request.url, + request.retry_times, + setting.SPIDER_MAX_RETRY_TIMES, + ) ) + if used_download_midware_enable: + # 去掉download_midware 添加的属性 使用原来的requests + original_request = ( + Request.from_dict(eval(request_redis)) + if request_redis + else request + ) + if hasattr(request, "error_msg"): + original_request.error_msg = request.error_msg + if hasattr(request, "response"): + original_request.response = request.response + original_request.retry_times = request.retry_times + original_request.filter_repeat = request.filter_repeat + + self._request_buffer.put_request(original_request) + else: + self._request_buffer.put_request(request) + del_request_redis_after_request_to_db = True + + else: + # 记录下载成功的文档 + self.record_download_status( + ParserControl.DOWNLOAD_SUCCESS, parser.name + ) + # 记录成功任务数 + self.__class__._success_task_count += 1 + + # 缓存下载成功的文档 + if setting.RESPONSE_CACHED_ENABLE: + request.save_cached( + response=response, + expire_time=setting.RESPONSE_CACHED_EXPIRE_TIME, + ) - break + finally: + # 释放浏览器 + if response and getattr(response, "browser", None): + request.render_downloader.put_back(response.browser) - # 删除正在做的request 跟随item优先 - if request_redis: - if del_request_redis_after_item_to_db: - self._item_buffer.put_item(request_redis) + break - elif del_request_redis_after_request_to_db: - self._request_buffer.put_del_request(request_redis) + # 删除正在做的request 跟随item优先 + if request_redis: + if del_request_redis_after_item_to_db: + self._item_buffer.put_item(request_redis) - else: - self._request_buffer.put_del_request(request_redis) + elif del_request_redis_after_request_to_db: + self._request_buffer.put_del_request(request_redis) + + else: + self._request_buffer.put_del_request(request_redis) if setting.SPIDER_SLEEP_TIME: - time.sleep(setting.SPIDER_SLEEP_TIME) + if ( + isinstance(setting.SPIDER_SLEEP_TIME, (tuple, list)) + and len(setting.SPIDER_SLEEP_TIME) == 2 + ): + sleep_time = random.randint( + int(setting.SPIDER_SLEEP_TIME[0]), int(setting.SPIDER_SLEEP_TIME[1]) + ) + time.sleep(sleep_time) + else: + time.sleep(setting.SPIDER_SLEEP_TIME) def record_download_status(self, status, spider): """ 记录html等文档下载状态 @return: """ - pass + + metrics.emit_counter(f"{spider}:{status}", 1, classify="document") def stop(self): self._thread_stop = True + self._started.clear() + + def add_parser(self, parser: BaseParser): + # 动态增加parser.exception_request和parser.failed_request的参数, 兼容旧版本 + if parser not in self.__class__._hook_parsers: + self.__class__._hook_parsers.add(parser) + if len(inspect.getfullargspec(parser.exception_request).args) == 3: + _exception_request = parser.exception_request + parser.exception_request = ( + lambda request, response, e: _exception_request(request, response) + ) + + if len(inspect.getfullargspec(parser.failed_request).args) == 3: + _failed_request = parser.failed_request + parser.failed_request = lambda request, response, e: _failed_request( + request, response + ) - def add_parser(self, parser): self._parsers.append(parser) -class AirSpiderParserControl(PaserControl): +class AirSpiderParserControl(ParserControl): is_show_tip = False # 实时统计已做任务数及失败任务数，若失败任务数/已做任务数>0.5 则报警 _success_task_count = 0 _failed_task_count = 0 - def __init__(self, memory_db: MemoryDB): - super(PaserControl, self).__init__() + def __init__( + self, + *, + memory_db: MemoryDB, + request_buffer: AirSpiderRequestBuffer, + item_buffer: ItemBuffer, + ): + super(ParserControl, self).__init__() self._parsers = [] self._memory_db = memory_db self._thread_stop = False - self._wait_task_time = 0 + self._request_buffer = request_buffer + self._item_buffer = item_buffer def run(self): while not self._thread_stop: try: - requests = self._memory_db.get() - if not requests: + request = self._memory_db.get() + if not request: if not self.is_show_tip: - log.info("parser 等待任务 ...") + log.debug("等待任务...") self.is_show_tip = True - - time.sleep(1) - self._wait_task_time += 1 continue self.is_show_tip = False - self.deal_requests([requests]) + self.deal_request(request) except Exception as e: log.exception(e) - def deal_requests(self, requests): - for request in requests: - - response = None - - for parser in self._parsers: - if parser.name == request.parser_name: - try: - # 记录需下载的文档 - self.record_download_status( - PaserControl.DOWNLOAD_TOTAL, parser.name - ) - - # 解析request - if request.auto_request: - request_temp = None - if request.download_midware: + def deal_request(self, request): + response = None + + for parser in self._parsers: + if parser.name == request.parser_name: + try: + self.__class__._total_task_count += 1 + # 记录需下载的文档 + self.record_download_status( + ParserControl.DOWNLOAD_TOTAL, parser.name + ) + + # 解析request + if request.auto_request: + request_temp = None + response = None + + # 下载中间件 + if request.download_midware: + if isinstance(request.download_midware, (list, tuple)): + request_temp = request + for download_midware in request.download_midware: + download_midware = ( + download_midware + if callable(download_midware) + else tools.get_method(parser, download_midware) + ) + request_temp = download_midware(request_temp) + else: download_midware = ( request.download_midware if callable(request.download_midware) @@ -450,83 +526,71 @@ def deal_requests(self, requests): ) ) request_temp = download_midware(request) - elif request.download_midware != False: - request_temp = parser.download_midware(request) - - if request_temp: - if not isinstance(request_temp, Request): - raise Exception( - "download_midware need return a request, but received type: {}".format( - type(request_temp) - ) + elif request.download_midware != False: + request_temp = parser.download_midware(request) + + # 请求 + if request_temp: + if ( + isinstance(request_temp, (tuple, list)) + and len(request_temp) == 2 + ): + request_temp, response = request_temp + + if not isinstance(request_temp, Request): + raise Exception( + "download_midware need return a request, but received type: {}".format( + type(request_temp) ) - request = request_temp + ) + request = request_temp + if response is None: response = ( request.get_response() if not setting.RESPONSE_CACHED_USED else request.get_response_from_cached(save_cached=False) ) - else: - response = None + # 校验 + if parser.validate(request, response) == False: + break - if request.callback: # 如果有parser的回调函数，则用回调处理 - callback_parser = ( - request.callback - if callable(request.callback) - else tools.get_method(parser, request.callback) - ) - results = callback_parser(request, response) - else: # 否则默认用parser处理 - results = parser.parse(request, response) - - if results and not isinstance(results, Iterable): - raise Exception( - "%s.%s返回值必须可迭代" - % (parser.name, request.callback or "parser") - ) + else: + response = None - # 此处判断是request 还是 item - for result in results or []: - if isinstance(result, Request): - # 给request的 parser_name 赋值 - result.parser_name = result.parser_name or parser.name - - # 判断是同步的callback还是异步的 - if result.request_sync: # 同步 - requests.append(result) - else: # 异步 - # 将next_request 入库 - self._memory_db.add(result) - - except Exception as e: - exception_type = ( - str(type(e)).replace("", "") + if request.callback: # 如果有parser的回调函数，则用回调处理 + callback_parser = ( + request.callback + if callable(request.callback) + else tools.get_method(parser, request.callback) ) - if exception_type.startswith("requests"): - # 记录下载失败的文档 - self.record_download_status( - PaserControl.DOWNLOAD_EXCEPTION, parser.name - ) - - else: - # 记录解析程序异常 - self.record_download_status( - PaserControl.PAESERS_EXCEPTION, parser.name - ) + results = callback_parser(request, response) + else: # 否则默认用parser处理 + results = parser.parse(request, response) - if setting.LOG_LEVEL == "DEBUG": # 只有debug模式下打印， 超时的异常篇幅太多 - log.exception(e) + if results and not isinstance(results, Iterable): + raise Exception( + "%s.%s返回值必须可迭代" % (parser.name, request.callback or "parse") + ) - log.error( - """ - -------------- %s.%s error ------------- - error %s - response %s - deal request %s - """ - % ( + # 此处判断是request 还是 item + for result in results or []: + if isinstance(result, Request): + # 给request的 parser_name 赋值 + result.parser_name = result.parser_name or parser.name + + # 判断是同步的callback还是异步的 + if result.request_sync: # 同步 + self.deal_request(result) + else: # 异步 + # 将next_request 入库 + self._request_buffer.put_request(result) + + elif isinstance(result, Item): + self._item_buffer.put_item(result) + elif result is not None: + function_name = "{}.{}".format( parser.name, ( request.callback @@ -534,95 +598,150 @@ def deal_requests(self, requests): and getattr(request.callback, "__name__") or request.callback ) - or "parser", - str(e), - response, - tools.dumps_json(request.to_dict, indent=28) - if setting.LOG_LEVEL == "DEBUG" - else request, + or "parse", ) + raise TypeError( + f"{function_name} result expect Request or Item, bug get type: {type(result)}" + ) + + except Exception as e: + exception_type = ( + str(type(e)).replace("", "") + ) + if exception_type.startswith("requests"): + # 记录下载失败的文档 + self.record_download_status( + ParserControl.DOWNLOAD_EXCEPTION, parser.name ) + if request.retry_times % setting.PROXY_MAX_FAILED_TIMES == 0: + request.del_proxy() - request.error_msg = "%s: %s" % (exception_type, e) - request.response = str(response) + else: + # 记录解析程序异常 + self.record_download_status( + ParserControl.PAESERS_EXCEPTION, parser.name + ) - if "Invalid URL" in str(e): - request.is_abandoned = True + if setting.LOG_LEVEL == "DEBUG": # 只有debug模式下打印， 超时的异常篇幅太多 + log.exception(e) - requests = parser.exception_request(request, response) or [ - request - ] - if not isinstance(requests, Iterable): - raise Exception( - "%s.%s返回值必须可迭代" % (parser.name, "exception_request") + log.error( + """ + -------------- %s.%s error ------------- + error %s + response %s + deal request %s + """ + % ( + parser.name, + ( + request.callback + and callable(request.callback) + and getattr(request.callback, "__name__") + or request.callback ) - for request in requests: - if not isinstance(request, Request): - raise Exception("exception_request 需return request") + or "parse", + str(e), + response, + tools.dumps_json(request.to_dict, indent=28) + if setting.LOG_LEVEL == "DEBUG" + else request, + ) + ) - if ( - request.retry_times + 1 > setting.SPIDER_MAX_RETRY_TIMES - or request.is_abandoned - ): - self.__class__._failed_task_count += 1 # 记录失败任务数 - - # 处理failed_request的返回值 request 或 func - results = parser.failed_request(request, response) or [ - request - ] - if not isinstance(results, Iterable): - raise Exception( - "%s.%s返回值必须可迭代" - % (parser.name, "failed_request") - ) + request.error_msg = "%s: %s" % (exception_type, e) + request.response = str(response) - log.info( - """ - 任务超过最大重试次数，丢弃 + if "Invalid URL" in str(e): + request.is_abandoned = True + + requests = parser.exception_request(request, response, e) or [ + request + ] + if not isinstance(requests, Iterable): + raise Exception( + "%s.%s返回值必须可迭代" % (parser.name, "exception_request") + ) + for request in requests: + if not isinstance(request, Request): + raise Exception("exception_request 需 yield request") + + if ( + request.retry_times + 1 > setting.SPIDER_MAX_RETRY_TIMES + or request.is_abandoned + ): + self.__class__._failed_task_count += 1 # 记录失败任务数 + + # 处理failed_request的返回值 request 或 func + results = parser.failed_request(request, response, e) or [ + request + ] + if not isinstance(results, Iterable): + raise Exception( + "%s.%s返回值必须可迭代" % (parser.name, "failed_request") + ) + + log.info( + """ + 任务超过最大重试次数，丢弃 + url %s + 重试次数 %s + 最大允许重试次数 %s""" + % ( + request.url, + request.retry_times, + setting.SPIDER_MAX_RETRY_TIMES, + ) + ) + + else: + # 将 requests 重新入库 爬取 + request.retry_times += 1 + request.filter_repeat = False + log.info( + """ + 入库 等待重试 url %s 重试次数 %s 最大允许重试次数 %s""" - % ( - request.url, - request.retry_times, - setting.SPIDER_MAX_RETRY_TIMES, - ) + % ( + request.url, + request.retry_times, + setting.SPIDER_MAX_RETRY_TIMES, ) + ) + self._request_buffer.put_request(request) - else: - # 将 requests 重新入库 爬取 - request.retry_times += 1 - request.filter_repeat = False - log.info( - """ - 入库 等待重试 - url %s - 重试次数 %s - 最大允许重试次数 %s""" - % ( - request.url, - request.retry_times, - setting.SPIDER_MAX_RETRY_TIMES, - ) - ) - self._memory_db.add(request) - - else: - # 记录下载成功的文档 - self.record_download_status( - PaserControl.DOWNLOAD_SUCCESS, parser.name + else: + # 记录下载成功的文档 + self.record_download_status( + ParserControl.DOWNLOAD_SUCCESS, parser.name + ) + # 记录成功任务数 + self.__class__._success_task_count += 1 + + # 缓存下载成功的文档 + if setting.RESPONSE_CACHED_ENABLE: + request.save_cached( + response=response, + expire_time=setting.RESPONSE_CACHED_EXPIRE_TIME, ) - # 记录成功任务数 - self.__class__._success_task_count += 1 - - # 缓存下载成功的文档 - if setting.RESPONSE_CACHED_ENABLE: - request.save_cached( - response=response, - expire_time=setting.RESPONSE_CACHED_EXPIRE_TIME, - ) - break + finally: + # 释放浏览器 + if response and getattr(response, "browser", None): + request.render_downloader.put_back(response.browser) + + break if setting.SPIDER_SLEEP_TIME: - time.sleep(setting.SPIDER_SLEEP_TIME) + if ( + isinstance(setting.SPIDER_SLEEP_TIME, (tuple, list)) + and len(setting.SPIDER_SLEEP_TIME) == 2 + ): + sleep_time = random.randint( + int(setting.SPIDER_SLEEP_TIME[0]), int(setting.SPIDER_SLEEP_TIME[1]) + ) + time.sleep(sleep_time) + else: + time.sleep(setting.SPIDER_SLEEP_TIME) diff --git a/feapder/core/scheduler.py b/feapder/core/scheduler.py index f12fa539..0177d185 100644 --- a/feapder/core/scheduler.py +++ b/feapder/core/scheduler.py @@ -5,11 +5,11 @@ @summary: 组装parser、 parser_control 和 collector --------- @author: Boris -@email: boris@bzkj.tech +@email: boris_liu@foxmail.com """ import threading import time -from collections import Iterable +from collections.abc import Iterable import feapder.setting as setting import feapder.utils.tools as tools @@ -17,20 +17,24 @@ from feapder.buffer.request_buffer import RequestBuffer from feapder.core.base_parser import BaseParser from feapder.core.collector import Collector +from feapder.core.handle_failed_items import HandleFailedItems from feapder.core.handle_failed_requests import HandleFailedRequests -from feapder.core.parser_control import PaserControl +from feapder.core.parser_control import ParserControl from feapder.db.redisdb import RedisDB from feapder.network.item import Item from feapder.network.request import Request +from feapder.utils import metrics from feapder.utils.log import log from feapder.utils.redis_lock import RedisLock +from feapder.utils.tail_thread import TailThread SPIDER_START_TIME_KEY = "spider_start_time" SPIDER_END_TIME_KEY = "spider_end_time" SPIDER_LAST_TASK_COUNT_RECORD_TIME_KEY = "last_task_count_record_time" +HEARTBEAT_TIME_KEY = "heartbeat_time" -class Scheduler(threading.Thread): +class Scheduler(TailThread): __custom_setting__ = {} def __init__( @@ -39,28 +43,27 @@ def __init__( thread_count=None, begin_callback=None, end_callback=None, - delete_tabs=(), - process_num=None, - auto_stop_when_spider_done=None, + delete_keys=(), + keep_alive=None, auto_start_requests=None, - send_run_time=True, batch_interval=0, wait_lock=True, + task_table=None, + **kwargs, ): """ @summary: 调度器 --------- - @param redis_key: 爬虫request及item存放reis中的文件夹 + @param redis_key: 爬虫request及item存放redis中的文件夹 @param thread_count: 线程数，默认为配置文件中的线程数 @param begin_callback: 爬虫开始回调函数 @param end_callback: 爬虫结束回调函数 - @param delete_tabs: 爬虫启动时删除的表，类型: 元组/bool/string。 支持正则 - @param process_num: 进程数 - @param auto_stop_when_spider_done: 爬虫抓取完毕后是否自动结束或等待任务，默认自动结束 + @param delete_keys: 爬虫启动时删除的key，类型: 元组/bool/string。 支持正则 + @param keep_alive: 爬虫是否常驻，默认否 @param auto_start_requests: 爬虫是否自动添加任务 - @param send_run_time: 发送运行时间 @param batch_interval: 抓取时间间隔 默认为0 天为单位 多次启动时，只有当前时间与第一次抓取结束的时间间隔大于指定的时间间隔时，爬虫才启动 @param wait_lock: 下发任务时否等待锁，若不等待锁，可能会存在多进程同时在下发一样的任务，因此分布式环境下请将该值设置True + @param task_table: 任务表， 批次爬虫传递 --------- @result: """ @@ -68,7 +71,10 @@ def __init__( super(Scheduler, self).__init__() for key, value in self.__class__.__custom_setting__.items(): - setattr(setting, key, value) + if key == "AUTO_STOP_WHEN_SPIDER_DONE": # 兼容老版本的配置 + setattr(setting, "KEEP_ALIVE", not value) + else: + setattr(setting, key, value) self._redis_key = redis_key or setting.REDIS_KEY if not self._redis_key: @@ -81,24 +87,25 @@ def __init__( ) self._request_buffer = RequestBuffer(redis_key) - self._item_buffer = ItemBuffer(redis_key) + self._item_buffer = ItemBuffer(redis_key, task_table) - self._collector = Collector(redis_key, process_num) + self._collector = Collector(redis_key) self._parsers = [] self._parser_controls = [] - self._parser_control_obj = PaserControl + self._parser_control_obj = ParserControl - self._auto_stop_when_spider_done = ( - auto_stop_when_spider_done - if auto_stop_when_spider_done is not None - else setting.AUTO_STOP_WHEN_SPIDER_DONE - ) + # 兼容老版本的参数 + if "auto_stop_when_spider_done" in kwargs: + self._keep_alive = not kwargs.get("auto_stop_when_spider_done") + else: + self._keep_alive = ( + keep_alive if keep_alive is not None else setting.KEEP_ALIVE + ) self._auto_start_requests = ( auto_start_requests if auto_start_requests is not None else setting.SPIDER_AUTO_START_REQUESTS ) - self._send_run_time = send_run_time self._batch_interval = batch_interval self._begin_callback = ( @@ -112,44 +119,53 @@ def __init__( else lambda: log.info("\n********** feapder end **********") ) - self._thread_count = ( - setting.SPIDER_THREAD_COUNT if not thread_count else thread_count - ) + if thread_count: + setattr(setting, "SPIDER_THREAD_COUNT", thread_count) + self._thread_count = setting.SPIDER_THREAD_COUNT - self._spider_name = redis_key - self._project_name = redis_key.split(":")[0] + self._spider_name = self.name + self._task_table = task_table - self._tab_spider_time = setting.TAB_SPIDER_TIME.format(redis_key=redis_key) self._tab_spider_status = setting.TAB_SPIDER_STATUS.format(redis_key=redis_key) - self._tab_requests = setting.TAB_REQUSETS.format(redis_key=redis_key) - self._tab_failed_requests = setting.TAB_FAILED_REQUSETS.format( + self._tab_requests = setting.TAB_REQUESTS.format(redis_key=redis_key) + self._tab_failed_requests = setting.TAB_FAILED_REQUESTS.format( redis_key=redis_key ) - self._is_notify_end = False # 是否已经通知结束 self._last_task_count = 0 # 最近一次任务数量 + self._last_check_task_count_time = 0 + self._stop_heartbeat = False # 是否停止心跳 self._redisdb = RedisDB() - self._project_total_state_table = "{}_total_state".format(self._project_name) - self._is_exist_project_total_state_table = False - # Request 缓存设置 Request.cached_redis_key = redis_key Request.cached_expire_time = setting.RESPONSE_CACHED_EXPIRE_TIME - delete_tabs = delete_tabs or setting.DELETE_TABS - if delete_tabs: - self.delete_tables(delete_tabs) + delete_keys = delete_keys or setting.DELETE_KEYS + if delete_keys: + self.delete_tables(delete_keys) self._last_check_task_status_time = 0 self.wait_lock = wait_lock - def add_parser(self, parser): - parser = parser() # parser 实例化 + self.init_metrics() + # 重置丢失的任务 + self.reset_task() + + self._stop_spider = False + + def init_metrics(self): + """ + 初始化打点系统 + """ + metrics.init(**setting.METRICS_OTHER_ARGS) + + def add_parser(self, parser, **kwargs): + parser = parser(**kwargs) # parser 实例化 if isinstance(parser, BaseParser): self._parsers.append(parser) else: - raise ValueError("parser 必须继承spider.core.base_parser.BaseParser") + raise ValueError("类型错误，爬虫需继承feapder.BaseParser或feapder.BatchParser") def run(self): if not self.is_reach_next_spider_time(): @@ -158,46 +174,34 @@ def run(self): self._start() while True: - if self.all_thread_is_done(): - if not self._is_notify_end: - self.spider_end() # 跑完一轮 - self.record_spider_state( - spider_type=1, - state=1, - spider_end_time=tools.get_current_date(), - batch_interval=self._batch_interval, - ) + try: + if self._stop_spider or self.all_thread_is_done(): + if not self._is_notify_end: + self.spider_end() # 跑完一轮 + self._is_notify_end = True - self._is_notify_end = True + if not self._keep_alive: + self._stop_all_thread() + break - if self._auto_stop_when_spider_done: - self._stop_all_thread() - break + else: + self._is_notify_end = False - else: - self._is_notify_end = False + self.check_task_status() - self.check_task_status() + except Exception as e: + log.exception(e) tools.delay_time(1) # 1秒钟检查一次爬虫状态 def __add_task(self): - # 启动parser 的 start_requests - self.spider_begin() # 不自动结束的爬虫此处只能执行一遍 - self.record_spider_state( - spider_type=1, - state=0, - batch_date=tools.get_current_date(), - spider_start_time=tools.get_current_date(), - batch_interval=self._batch_interval, - ) - # 判断任务池中属否还有任务，若有接着抓取 todo_task_count = self._collector.get_requests_count() if todo_task_count: - log.info("检查到有待做任务 %s 条，不重下发新任务。将接着上回异常终止处继续抓取" % todo_task_count) + log.info("检查到有待做任务 %s 条，不重下发新任务，将接着上回异常终止处继续抓取" % todo_task_count) else: for parser in self._parsers: + # 启动parser 的 start_requests results = parser.start_requests() # 添加request到请求队列，由请求队列统一入库 if results and not isinstance(results, Iterable): @@ -230,6 +234,19 @@ def __add_task(self): self._item_buffer.flush() def _start(self): + self.spider_begin() + + # 将失败的item入库 + if setting.RETRY_FAILED_ITEMS: + handle_failed_items = HandleFailedItems( + redis_key=self._redis_key, + task_table=self._task_table, + item_buffer=self._item_buffer, + ) + handle_failed_items.reput_failed_items_to_db() + + # 心跳开始 + self.heartbeat_start() # 启动request_buffer self._request_buffer.start() # 启动item_buffer @@ -262,19 +279,15 @@ def _start(self): if self._auto_start_requests: # 自动下发 if self.wait_lock: # 将添加任务处加锁，防止多进程之间添加重复的任务 - with RedisLock( - key=self._spider_name, - timeout=3600, - wait_timeout=60, - redis_cli=RedisDB().get_redis_obj(), - ) as lock: + with RedisLock(key=self._spider_name) as lock: if lock.locked: self.__add_task() else: self.__add_task() def all_thread_is_done(self): - for i in range(3): # 降低偶然性, 因为各个环节不是并发的，很有可能当时状态为假，但检测下一条时该状态为真。一次检测很有可能遇到这种偶然性 + # 降低偶然性, 因为各个环节不是并发的，很有可能当时状态为假，但检测下一条时该状态为真。一次检测很有可能遇到这种偶然性 + for i in range(3): # 检测 collector 状态 if ( self._collector.is_collector_task() @@ -317,76 +330,24 @@ def check_task_status(self): else: return - # 检查redis中任务状态，若连续20分钟内任务数量未发生变化（parser可能卡死），则发出报警信息 - task_count = self._redisdb.zget_count(self._tab_requests) - - if task_count: - if task_count != self._last_task_count: - self._last_task_count = task_count - self._redisdb.hset( - self._tab_spider_time, - SPIDER_LAST_TASK_COUNT_RECORD_TIME_KEY, - tools.get_current_timestamp(), - ) # 多进程会重复发消息， 使用reids记录上次统计时间 - else: - # 判断时间间隔是否超过20分钟 - lua = """ - local key = KEYS[1] - local field = ARGV[1] - local current_timestamp = ARGV[2] - - -- 取值 - local last_timestamp = redis.call('hget', key, field) - if last_timestamp and current_timestamp - last_timestamp >= 1200 then - return current_timestamp - last_timestamp -- 返回任务停滞时间 秒 - end - - if not last_timestamp then - redis.call('hset', key, field, current_timestamp) - end - - return 0 - - """ - redis_obj = self._redisdb.get_redis_obj() - cmd = redis_obj.register_script(lua) - overtime = cmd( - keys=[self._tab_spider_time], - args=[ - SPIDER_LAST_TASK_COUNT_RECORD_TIME_KEY, - tools.get_current_timestamp(), - ], - ) - - if overtime: - # 发送报警 - msg = "《{}》爬虫任务停滞 {}，请检查爬虫是否正常".format( - self._spider_name, tools.format_seconds(overtime) - ) - log.error(msg) - self.send_msg( - msg, - level="error", - message_prefix="《{}》爬虫任务停滞".format(self._spider_name), - ) - - else: - self._last_task_count = 0 - # 检查失败任务数量 超过1000 报警， failed_count = self._redisdb.zget_count(self._tab_failed_requests) if failed_count > setting.WARNING_FAILED_COUNT: # 发送报警 - msg = "《%s》爬虫当前失败任务 %s, 请检查爬虫是否正常" % (self._spider_name, failed_count) + msg = "《%s》爬虫当前失败任务数：%s, 请检查爬虫是否正常" % (self._spider_name, failed_count) log.error(msg) self.send_msg( msg, level="error", - message_prefix="《%s》爬虫当前失败任务数预警" % (self._spider_name), + message_prefix="《%s》爬虫当前失败任务数报警" % (self._spider_name), ) - # parser_control实时统计已做任务数及失败任务数，若失败数大于10且失败任务数/已做任务数>=0.5 则报警 - failed_task_count, success_task_count = PaserControl.get_task_status_count() + # parser_control实时统计已做任务数及失败任务数，若成功率<0.5 则报警 + ( + failed_task_count, + success_task_count, + total_task_count, + ) = ParserControl.get_task_status_count() total_count = success_task_count + failed_task_count if total_count > 0: task_success_rate = success_task_count / total_count @@ -399,29 +360,63 @@ def check_task_status(self): task_success_rate, ) log.error(msg) - # 统计下上次发消息的时间，若时间大于1小时，则报警（此处为多进程，需要考虑别报重复） self.send_msg( msg, level="error", - message_prefix="《%s》爬虫当前任务成功率" % (self._spider_name), + message_prefix="《%s》爬虫当前任务成功率报警" % (self._spider_name), + ) + + # 判断任务数是否变化 + current_time = tools.get_current_timestamp() + if ( + current_time - self._last_check_task_count_time + > setting.WARNING_CHECK_TASK_COUNT_INTERVAL + ): + if ( + self._last_task_count + and self._last_task_count == total_task_count + and self._redisdb.zget_count(self._tab_requests) > 0 + ): + # 发送报警 + msg = "《{}》爬虫停滞 {}，请检查爬虫是否正常".format( + self._spider_name, + tools.format_seconds( + current_time - self._last_check_task_count_time + ), + ) + log.error(msg) + self.send_msg( + msg, + level="error", + message_prefix="《{}》爬虫停滞".format(self._spider_name), ) + else: + self._last_task_count = total_task_count + self._last_check_task_count_time = current_time - def delete_tables(self, delete_tables_list): - if isinstance(delete_tables_list, bool): - delete_tables_list = [self._redis_key + "*"] - elif not isinstance(delete_tables_list, (list, tuple)): - delete_tables_list = [delete_tables_list] + # 检查入库失败次数 + if self._item_buffer.export_falied_times > setting.EXPORT_DATA_MAX_FAILED_TIMES: + msg = "《{}》爬虫导出数据失败，失败次数：{}， 请检查爬虫是否正常".format( + self._spider_name, self._item_buffer.export_falied_times + ) + log.error(msg) + self.send_msg( + msg, level="error", message_prefix="《%s》爬虫导出数据失败" % (self._spider_name) + ) - redis = RedisDB() - for delete_tab in delete_tables_list: - if delete_tab == "*": - delete_tab = self._redis_key + "*" + def delete_tables(self, delete_keys): + if delete_keys == True: + delete_keys = [self._redis_key + "*"] + elif not isinstance(delete_keys, (list, tuple)): + delete_keys = [delete_keys] - tables = redis.getkeys(delete_tab) - for table in tables: - if table != self._tab_spider_time: - log.info("正在删除表 %s" % table) - redis.clear(table) + for delete_key in delete_keys: + if not delete_key.startswith(self._redis_key): + delete_key = self._redis_key + delete_key + keys = self._redisdb.getkeys(delete_key) + for key in keys: + log.debug("正在删除key %s" % key) + self._redisdb.clear(key) def _stop_all_thread(self): self._request_buffer.stop() @@ -431,18 +426,12 @@ def _stop_all_thread(self): # 停止 parser_controls for parser_control in self._parser_controls: parser_control.stop() + self.heartbeat_stop() + self._started.clear() def send_msg(self, msg, level="debug", message_prefix=""): - """ - @summary: 叮叮 发送消息 - --------- - @param msg: 消息 - @param developers: 开发者姓名 - --------- - @result: - """ - - tools.dingding_warning(msg, rate_limit=3600, message_prefix=message_prefix) + # log.debug("发送报警 level:{} msg{}".format(level, msg)) + tools.send_msg(msg=msg, level=level, message_prefix=message_prefix) def spider_begin(self): """ @@ -459,14 +448,14 @@ def spider_begin(self): parser.start_callback() # 记录开始时间 - if not self._redisdb.hexists(self._tab_spider_time, SPIDER_START_TIME_KEY): + if not self._redisdb.hexists(self._tab_spider_status, SPIDER_START_TIME_KEY): current_timestamp = tools.get_current_timestamp() self._redisdb.hset( - self._tab_spider_time, SPIDER_START_TIME_KEY, current_timestamp + self._tab_spider_status, SPIDER_START_TIME_KEY, current_timestamp ) # 发送消息 - # self.send_msg('《%s》爬虫开始'%self._spider_name) + self.send_msg("《%s》爬虫开始" % self._spider_name) def spider_end(self): self.record_end_time() @@ -475,28 +464,38 @@ def spider_end(self): self._end_callback() for parser in self._parsers: - parser.close() + if not self._keep_alive: + parser.close() parser.end_callback() - # 计算抓取时常 + if not self._keep_alive: + # 关闭webdirver + Request.render_downloader and Request.render_downloader.close_all() + + # 关闭打点 + metrics.close() + else: + metrics.flush() + + # 计算抓取时长 data = self._redisdb.hget( - self._tab_spider_time, SPIDER_START_TIME_KEY, is_pop=True + self._tab_spider_status, SPIDER_START_TIME_KEY, is_pop=True ) if data: begin_timestamp = int(data) spand_time = tools.get_current_timestamp() - begin_timestamp - msg = "《%s》爬虫结束，耗时 %s" % ( + msg = "《%s》爬虫%s，采集耗时 %s" % ( self._spider_name, + "被终止" if self._stop_spider else "结束", tools.format_seconds(spand_time), ) log.info(msg) - if self._send_run_time: - self.send_msg(msg) + self.send_msg(msg) - if not self._auto_stop_when_spider_done: + if self._keep_alive: log.info("爬虫不自动结束， 等待下一轮任务...") else: self.delete_tables(self._tab_spider_status) @@ -506,7 +505,7 @@ def record_end_time(self): if self._batch_interval: current_timestamp = tools.get_current_timestamp() self._redisdb.hset( - self._tab_spider_time, SPIDER_END_TIME_KEY, current_timestamp + self._tab_spider_status, SPIDER_END_TIME_KEY, current_timestamp ) def is_reach_next_spider_time(self): @@ -514,7 +513,7 @@ def is_reach_next_spider_time(self): return True last_spider_end_time = self._redisdb.hget( - self._tab_spider_time, SPIDER_END_TIME_KEY + self._tab_spider_status, SPIDER_END_TIME_KEY ) if last_spider_end_time: last_spider_end_time = int(last_spider_end_time) @@ -533,13 +532,60 @@ def is_reach_next_spider_time(self): return True - def record_spider_state( - self, - spider_type, - state, - batch_date=None, - spider_start_time=None, - spider_end_time=None, - batch_interval=None, - ): - pass + def join(self, timeout=None): + """ + 重写线程的join + """ + if not self._started.is_set(): + return + + super().join() + + def heartbeat(self): + while not self._stop_heartbeat: + try: + self._redisdb.hset( + self._tab_spider_status, + HEARTBEAT_TIME_KEY, + tools.get_current_timestamp(), + ) + except Exception as e: + log.error("心跳异常: {}".format(e)) + time.sleep(5) + + def heartbeat_start(self): + threading.Thread(target=self.heartbeat).start() + + def heartbeat_stop(self): + self._stop_heartbeat = True + + def have_alive_spider(self, heartbeat_interval=10): + heartbeat_time = self._redisdb.hget(self._tab_spider_status, HEARTBEAT_TIME_KEY) + if heartbeat_time: + heartbeat_time = int(heartbeat_time) + current_timestamp = tools.get_current_timestamp() + if current_timestamp - heartbeat_time < heartbeat_interval: + return True + return False + + def reset_task(self, heartbeat_interval=10): + """ + 重置丢失的任务 + Returns: + + """ + if self.have_alive_spider(heartbeat_interval=heartbeat_interval): + current_timestamp = tools.get_current_timestamp() + datas = self._redisdb.zrangebyscore_set_score( + self._tab_requests, + priority_min=current_timestamp, + priority_max=current_timestamp + setting.REQUEST_LOST_TIMEOUT, + score=300, + count=None, + ) + lose_count = len(datas) + if lose_count: + log.info("重置丢失任务完毕，共{}条".format(len(datas))) + + def stop_spider(self): + self._stop_spider = True diff --git a/feapder/core/spiders/__init__.py b/feapder/core/spiders/__init__.py index 13a438c2..a32ba668 100644 --- a/feapder/core/spiders/__init__.py +++ b/feapder/core/spiders/__init__.py @@ -5,11 +5,12 @@ @summary: --------- @author: Boris -@email: boris@bzkj.tech +@email: boris_liu@foxmail.com """ -__all__ = ["AirSpider", "Spider", "BatchSpider"] +__all__ = ["AirSpider", "TaskSpider", "Spider", "BatchSpider"] from feapder.core.spiders.air_spider import AirSpider from feapder.core.spiders.spider import Spider +from feapder.core.spiders.task_spider import TaskSpider from feapder.core.spiders.batch_spider import BatchSpider diff --git a/feapder/core/spiders/air_spider.py b/feapder/core/spiders/air_spider.py index 231ce48a..70c30112 100644 --- a/feapder/core/spiders/air_spider.py +++ b/feapder/core/spiders/air_spider.py @@ -5,21 +5,23 @@ @summary: 基于内存队列的爬虫，不支持分布式 --------- @author: Boris -@email: boris@bzkj.tech +@email: boris_liu@foxmail.com """ -from threading import Thread - import feapder.setting as setting import feapder.utils.tools as tools +from feapder.buffer.item_buffer import ItemBuffer +from feapder.buffer.request_buffer import AirSpiderRequestBuffer from feapder.core.base_parser import BaseParser from feapder.core.parser_control import AirSpiderParserControl -from feapder.db.memory_db import MemoryDB +from feapder.db.memorydb import MemoryDB from feapder.network.request import Request +from feapder.utils import metrics from feapder.utils.log import log +from feapder.utils.tail_thread import TailThread -class AirSpider(BaseParser, Thread): +class AirSpider(BaseParser, TailThread): __custom_setting__ = {} def __init__(self, thread_count=None): @@ -32,12 +34,19 @@ def __init__(self, thread_count=None): for key, value in self.__class__.__custom_setting__.items(): setattr(setting, key, value) - self._thread_count = ( - setting.SPIDER_THREAD_COUNT if not thread_count else thread_count - ) + if thread_count: + setattr(setting, "SPIDER_THREAD_COUNT", thread_count) + self._thread_count = setting.SPIDER_THREAD_COUNT self._memory_db = MemoryDB() self._parser_controls = [] + self._item_buffer = ItemBuffer(redis_key=self.name) + self._request_buffer = AirSpiderRequestBuffer( + db=self._memory_db, dedup_name=self.name + ) + + self._stop_spider = False + metrics.init(**setting.METRICS_OTHER_ARGS) def distribute_task(self): for request in self.start_requests(): @@ -45,7 +54,7 @@ def distribute_task(self): raise ValueError("仅支持 yield Request") request.parser_name = request.parser_name or self.name - self._memory_db.add(request) + self._request_buffer.put_request(request, ignore_max_size=False) def all_thread_is_done(self): for i in range(3): # 降低偶然性, 因为各个环节不是并发的，很有可能当时状态为假，但检测下一条时该状态为真。一次检测很有可能遇到这种偶然性 @@ -58,24 +67,72 @@ def all_thread_is_done(self): if not self._memory_db.empty(): return False + # 检测 item_buffer 状态 + if ( + self._item_buffer.get_items_count() > 0 + or self._item_buffer.is_adding_to_db() + ): + return False + tools.delay_time(1) return True def run(self): - self.distribute_task() + self.start_callback() for i in range(self._thread_count): - parser_control = AirSpiderParserControl(self._memory_db) + parser_control = AirSpiderParserControl( + memory_db=self._memory_db, + request_buffer=self._request_buffer, + item_buffer=self._item_buffer, + ) parser_control.add_parser(self) parser_control.start() self._parser_controls.append(parser_control) + self._item_buffer.start() + + self.distribute_task() + while True: - if self.all_thread_is_done(): - # 停止 parser_controls - for parser_control in self._parser_controls: - parser_control.stop() + try: + if self._stop_spider or self.all_thread_is_done(): + # 停止 parser_controls + for parser_control in self._parser_controls: + parser_control.stop() + + # 关闭item_buffer + self._item_buffer.stop() + + # 关闭webdirver + Request.render_downloader and Request.render_downloader.close_all() + + if self._stop_spider: + log.info("爬虫被终止") + else: + log.info("无任务，爬虫结束") + break + + except Exception as e: + log.exception(e) + + tools.delay_time(1) # 1秒钟检查一次爬虫状态 + + self.end_callback() + # 为了线程可重复start + self._started.clear() + # 关闭打点 + metrics.close() + + def join(self, timeout=None): + """ + 重写线程的join + """ + if not self._started.is_set(): + return + + super().join() - log.debug("无任务，爬虫结束") - break + def stop_spider(self): + self._stop_spider = True diff --git a/feapder/core/spiders/batch_spider.py b/feapder/core/spiders/batch_spider.py index 4ac67c1c..6b2ae092 100644 --- a/feapder/core/spiders/batch_spider.py +++ b/feapder/core/spiders/batch_spider.py @@ -5,19 +5,17 @@ @summary: --------- @author: Boris -@email: boris@bzkj.tech +@email: boris_liu@foxmail.com """ import datetime import os import time import warnings -from collections import Iterable +from collections.abc import Iterable import feapder.setting as setting import feapder.utils.tools as tools -from feapder.buffer.item_buffer import MAX_ITEM_COUNT -from feapder.buffer.request_buffer import RequestBuffer from feapder.core.base_parser import BatchParser from feapder.core.scheduler import Scheduler from feapder.db.mysqldb import MysqlDB @@ -26,8 +24,11 @@ from feapder.network.item import UpdateItem from feapder.network.request import Request from feapder.utils.log import log +from feapder.utils.perfect_dict import PerfectDict from feapder.utils.redis_lock import RedisLock +CONSOLE_PIPELINE_PATH = "feapder.pipelines.console_pipeline.ConsolePipeline" + class BatchSpider(BatchParser, Scheduler): def __init__( @@ -49,10 +50,10 @@ def __init__( thread_count=None, begin_callback=None, end_callback=None, - delete_tabs=(), - process_num=None, - auto_stop_when_spider_done=None, - send_run_time=False, + delete_keys=(), + keep_alive=None, + auto_start_next_batch=True, + **kwargs, ): """ @summary: 批次爬虫 @@ -68,43 +69,32 @@ def __init__( `parser_name` varchar(255) DEFAULT NULL COMMENT '任务解析器的脚本类名', PRIMARY KEY (`id`), UNIQUE KEY `nui` (`param`) USING BTREE - ) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8; - - 2、需有批次记录表 不存在自动创建 - - 此表节结构固定，参考建表语句如下： - CREATE TABLE `xxx_batch_record` ( - `id` int(11) NOT NULL AUTO_INCREMENT, - `batch_date` date DEFAULT NULL, - `done_count` int(11) DEFAULT NULL, - `total_count` int(11) DEFAULT NULL, - PRIMARY KEY (`id`) ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8; + 2、需有批次记录表 不存在自动创建 --------- @param task_table: mysql中的任务表 @param batch_record_table: mysql 中的批次记录表 @param batch_name: 批次采集程序名称 @param batch_interval: 批次间隔 天为单位。 如想一小时一批次，可写成1/24 @param task_keys: 需要获取的任务字段 列表 [] 如需指定解析的parser，则需将parser_name字段取出来。 - @param task_state: mysql中任务表的state字段名 + @param task_state: mysql中任务表的任务状态字段 @param min_task_count: redis 中最少任务数, 少于这个数量会从mysql的任务表取任务 @param check_task_interval: 检查是否还有任务的时间间隔； - @param task_limit: 数据库中取任务的数量 + @param task_limit: 从数据库中取任务的数量 @param redis_key: 任务等数据存放在redis中的key前缀 @param thread_count: 线程数，默认为配置文件中的线程数 @param begin_callback: 爬虫开始回调函数 @param end_callback: 爬虫结束回调函数 - @param delete_tabs: 爬虫启动时删除的表（redis里的key），元组类型。 支持正则; 常用于清空任务队列，否则重启时会断点续爬 - @param process_num: 进程数 - @param auto_stop_when_spider_done: 爬虫抓取完毕后是否自动结束或等待任务，默认自动结束 - @param send_run_time: 发送运行时间 - @param related_redis_key: 有关联的其他爬虫任务表（redis） - @param related_batch_record: 有关联的其他爬虫批次表（mysql）注意：要避免环路 如 A -> B & B -> A 。 环路可用related_redis_key指定 - related_redis_key 与 related_batch_record 选其一配置即可。 + @param delete_keys: 爬虫启动时删除的key，类型: 元组/bool/string。 支持正则; 常用于清空任务队列，否则重启时会断点续爬 + @param keep_alive: 爬虫是否常驻，默认否 + @param auto_start_next_batch: 本批次结束后，且下一批次时间已到达时，是否自动启动下一批次，默认是 + @param related_redis_key: 有关联的其他爬虫任务表（redis）注意：要避免环路 如 A -> B & B -> A 。 + @param related_batch_record: 有关联的其他爬虫批次表（mysql）注意：要避免环路 如 A -> B & B -> A 。 + related_redis_key 与 related_batch_record 选其一配置即可；用于相关联的爬虫没结束时，本爬虫也不结束 若相关连的爬虫为批次爬虫，推荐以related_batch_record配置， 若相关连的爬虫为普通爬虫，无批次表，可以以related_redis_key配置 - @param task_condition: 任务条件 用于从一个大任务表中挑选出数据自己爬虫的任务，及where后的条件语句 + @param task_condition: 任务条件 用于从一个大任务表中挑选出数据自己爬虫的任务，即where后的条件语句 @param task_order_by: 取任务时的排序条件 如 id desc --------- @result: @@ -115,19 +105,17 @@ def __init__( thread_count=thread_count, begin_callback=begin_callback, end_callback=end_callback, - delete_tabs=delete_tabs, - process_num=process_num, - auto_stop_when_spider_done=auto_stop_when_spider_done, + delete_keys=delete_keys, + keep_alive=keep_alive, auto_start_requests=False, - send_run_time=send_run_time, batch_interval=batch_interval, + task_table=task_table, + **kwargs, ) self._redisdb = RedisDB() self._mysqldb = MysqlDB() - self._request_buffer = RequestBuffer(self._redis_key) - self._task_table = task_table # mysql中的任务表 self._batch_record_table = batch_record_table # mysql 中的批次记录表 self._batch_name = batch_name # 批次采集程序名称 @@ -138,11 +126,11 @@ def __init__( self._check_task_interval = check_task_interval self._task_limit = task_limit # mysql中一次取的任务数量 self._related_task_tables = [ - setting.TAB_REQUSETS.format(redis_key=redis_key) + setting.TAB_REQUESTS.format(redis_key=redis_key) ] # 自己的task表也需要检查是否有任务 if related_redis_key: self._related_task_tables.append( - setting.TAB_REQUSETS.format(redis_key=related_redis_key) + setting.TAB_REQUESTS.format(redis_key=related_redis_key) ) self._related_batch_record = related_batch_record @@ -154,6 +142,7 @@ def __init__( task_condition ) self._task_order_by = task_order_by and " order by {}".format(task_order_by) + self._auto_start_next_batch = auto_start_next_batch self._batch_date_cache = None if self._batch_interval >= 1: @@ -163,33 +152,35 @@ def __init__( else: self._date_format = "%Y-%m-%d %H:%M" - # 报警相关 - self._send_msg_interval = datetime.timedelta(hours=1) # 每隔1小时发送一次报警 - self._last_send_msg_time = None + self._is_more_parsers = True # 多模版类爬虫 + # 初始化每个配置的属性 self._spider_last_done_time = None # 爬虫最近已做任务数量时间 - self._spider_last_done_count = 0 # 爬虫最近已做任务数量 + self._spider_last_done_count = None # 爬虫最近已做任务数量 self._spider_deal_speed_cached = None + self._batch_timeout = False # 批次是否超时或将要超时 - self._is_more_parsers = True # 多模版类爬虫 + # 重置任务 + self.reset_task() - def init_property(self): + def init_batch_property(self): """ 每个批次开始时需要重置的属性 @return: """ - self._last_send_msg_time = None - + self._spider_deal_speed_cached = None self._spider_last_done_time = None - self._spider_last_done_count = 0 # 爬虫刚开始启动时已做任务数量 + self._spider_last_done_count = None # 爬虫刚开始启动时已做任务数量 + self._batch_timeout = False - def add_parser(self, parser): + def add_parser(self, parser, **kwargs): parser = parser( self._task_table, self._batch_record_table, self._task_state, self._date_format, self._mysqldb, + **kwargs, ) # parser 实例化 self._parsers.append(parser) @@ -217,7 +208,7 @@ def start_monitor_task(self): while True: try: if self.check_batch(is_first_check): # 该批次已经做完 - if not self._auto_stop_when_spider_done: + if self._keep_alive: is_first_check = True log.info("爬虫所有任务已做完，不自动结束，等待新任务...") time.sleep(self._check_task_interval) @@ -228,7 +219,7 @@ def start_monitor_task(self): is_first_check = False # 检查redis中是否有任务 任务小于_min_task_count 则从mysql中取 - tab_requests = setting.TAB_REQUSETS.format(redis_key=self._redis_key) + tab_requests = setting.TAB_REQUESTS.format(redis_key=self._redis_key) todo_task_count = self._redisdb.zget_count(tab_requests) tasks = [] @@ -305,8 +296,8 @@ def create_batch_record_table(self): CREATE TABLE `{table_name}` ( `id` int(11) UNSIGNED NOT NULL AUTO_INCREMENT, `batch_date` {batch_date} DEFAULT NULL COMMENT '批次时间', - `done_count` int(11) DEFAULT NULL COMMENT '完成数 (1,-1)', `total_count` int(11) DEFAULT NULL COMMENT '任务总数', + `done_count` int(11) DEFAULT NULL COMMENT '完成数 (1,-1)', `fail_count` int(11) DEFAULT NULL COMMENT '失败任务数 (-1)', `interval` float(11) DEFAULT NULL COMMENT '批次间隔', `interval_unit` varchar(20) DEFAULT NULL COMMENT '批次间隔单位 day, hour', @@ -317,7 +308,7 @@ def create_batch_record_table(self): ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8; """.format( table_name=self._batch_record_table, - batch_date="date" if self._date_format == "%Y-%m-%d" else "datetime", + batch_date="datetime", ) self._mysqldb.execute(sql) @@ -334,6 +325,9 @@ def distribute_task(self, tasks): for task in tasks: for parser in self._parsers: # 寻找task对应的parser if parser.name in task: + task = PerfectDict( + _dict=dict(zip(self._task_keys, task)), _values=list(task) + ) requests = parser.start_requests(task) if requests and not isinstance(requests, Iterable): raise Exception( @@ -353,7 +347,7 @@ def distribute_task(self, tasks): if ( self._item_buffer.get_items_count() - >= MAX_ITEM_COUNT + >= setting.ITEM_MAX_CACHED_COUNT ): self._item_buffer.flush() @@ -365,7 +359,7 @@ def distribute_task(self, tasks): if ( self._item_buffer.get_items_count() - >= MAX_ITEM_COUNT + >= setting.ITEM_MAX_CACHED_COUNT ): self._item_buffer.flush() @@ -381,6 +375,9 @@ def distribute_task(self, tasks): else: # task没对应的parser 则将task下发到所有的parser for task in tasks: for parser in self._parsers: + task = PerfectDict( + _dict=dict(zip(self._task_keys, task)), _values=list(task) + ) requests = parser.start_requests(task) if requests and not isinstance(requests, Iterable): raise Exception( @@ -398,7 +395,10 @@ def distribute_task(self, tasks): self._item_buffer.put_item(request) result_type = 2 - if self._item_buffer.get_items_count() >= MAX_ITEM_COUNT: + if ( + self._item_buffer.get_items_count() + >= setting.ITEM_MAX_CACHED_COUNT + ): self._item_buffer.flush() elif callable(request): # callbale的request可能是更新数据库操作的函数 @@ -409,7 +409,7 @@ def distribute_task(self, tasks): if ( self._item_buffer.get_items_count() - >= MAX_ITEM_COUNT + >= setting.ITEM_MAX_CACHED_COUNT ): self._item_buffer.flush() @@ -476,8 +476,9 @@ def get_todo_task_from_mysql(self): """ # TODO 分批取数据 每批最大取 1000000个，防止内存占用过大 # 查询任务 + task_keys = ", ".join([f"`{key}`" for key in self._task_keys]) sql = "select %s from %s where %s = 0%s%s limit %s" % ( - ", ".join(self._task_keys), + task_keys, self._task_table, self._task_state, self._task_condition_prefix_and, @@ -510,8 +511,9 @@ def get_doing_task_from_mysql(self): """ # 查询任务 + task_keys = ", ".join([f"`{key}`" for key in self._task_keys]) sql = "select %s from %s where %s = 2%s%s limit %s" % ( - ", ".join(self._task_keys), + task_keys, self._task_table, self._task_state, self._task_condition_prefix_and, @@ -557,14 +559,12 @@ def get_deal_speed(self, total_count, done_count, last_batch_date): 或 None """ - if not self._spider_last_done_count: - now_date = datetime.datetime.now() + now_date = datetime.datetime.now() + if self._spider_last_done_count is None: self._spider_last_done_count = done_count self._spider_last_done_time = now_date - if done_count > self._spider_last_done_count: - now_date = datetime.datetime.now() - + elif done_count > self._spider_last_done_count: time_interval = (now_date - self._spider_last_done_time).total_seconds() deal_speed = ( done_count - self._spider_last_done_count @@ -616,14 +616,14 @@ def check_batch(self, is_first_check=False): @result: 完成返回True 否则False """ - sql = 'select date_format(batch_date, "{date_format}"), total_count, done_count from {batch_record_table} order by id desc limit 1'.format( + sql = 'select date_format(batch_date, "{date_format}"), total_count, done_count, is_done from {batch_record_table} order by id desc limit 1'.format( date_format=self._date_format.replace(":%M", ":%i"), batch_record_table=self._batch_record_table, ) - batch_info = self._mysqldb.find(sql) # (('2018-08-19', 49686, 0),) + batch_info = self._mysqldb.find(sql) # (('批次时间', 总量, 完成量, 批次是否完成),) if batch_info: - batch_date, total_count, done_count = batch_info[0] + batch_date, total_count, done_count, is_done = batch_info[0] now_date = datetime.datetime.now() last_batch_date = datetime.datetime.strptime(batch_date, self._date_format) @@ -639,47 +639,54 @@ def check_batch(self, is_first_check=False): done_count = task_count.get("done_count") if total_count == done_count: - # 检查相关联的爬虫是否完成 - releated_spider_is_done = self.related_spider_is_done() - if releated_spider_is_done == False: - msg = "《{}》本批次未完成, 正在等待依赖爬虫 {} 结束. 批次时间 {} 批次进度 {}/{}".format( - self._batch_name, - self._related_batch_record or self._related_task_tables, - batch_date, - done_count, - total_count, - ) - log.info(msg) - # 检查是否超时 超时发出报警 - if time_difference >= datetime.timedelta( - days=self._batch_interval - ): # 已经超时 - if ( - not self._last_send_msg_time - or now_date - self._last_send_msg_time - >= self._send_msg_interval - ): - self._last_send_msg_time = now_date - self.send_msg(msg, level="error") - - return False + if not is_done: + # 检查相关联的爬虫是否完成 + related_spider_is_done = self.related_spider_is_done() + if related_spider_is_done is False: + msg = "《{}》本批次未完成, 正在等待依赖爬虫 {} 结束. 批次时间 {} 批次进度 {}/{}".format( + self._batch_name, + self._related_batch_record or self._related_task_tables, + batch_date, + done_count, + total_count, + ) + log.info(msg) + # 检查是否超时 超时发出报警 + if time_difference >= datetime.timedelta( + days=self._batch_interval + ): # 已经超时 + self.send_msg( + msg, + level="error", + message_prefix="《{}》本批次未完成, 正在等待依赖爬虫 {} 结束".format( + self._batch_name, + self._related_batch_record + or self._related_task_tables, + ), + ) + self._batch_timeout = True - elif releated_spider_is_done == True: - # 更新is_done 状态 - self.update_is_done() + return False - else: - self.update_is_done() + else: + self.update_is_done() msg = "《{}》本批次完成 批次时间 {} 共处理 {} 条任务".format( self._batch_name, batch_date, done_count ) log.info(msg) if not is_first_check: - self.send_msg(msg) + if self._batch_timeout: # 之前报警过已超时，现在已完成，发出恢复消息 + self._batch_timeout = False + self.send_msg(msg, level="error") + else: + self.send_msg(msg) # 判断下一批次是否到 if time_difference >= datetime.timedelta(days=self._batch_interval): + if not is_first_check and not self._auto_start_next_batch: + return True # 下一批次不开始。因为设置了不自动开始下一批次 + msg = "《{}》下一批次开始".format(self._batch_name) log.info(msg) self.send_msg(msg) @@ -687,14 +694,20 @@ def check_batch(self, is_first_check=False): # 初始化任务表状态 if self.init_task() != False: # 更新失败返回False 其他返回True/None # 初始化属性 - self.init_property() + self.init_batch_property() is_success = ( self.record_batch() ) # 有可能插入不成功，但是任务表已经重置了，不过由于当前时间为下一批次的时间，检查批次是否结束时不会检查任务表，所以下次执行时仍然会重置 if is_success: - log.info("插入新批次记录成功 1分钟后开始下发任务") # 防止work批次时间没来得及更新 - tools.delay_time(60) + # 看是否有等待任务的worker，若有则需要等会再下发任务，防止work批次时间没来得及更新 + if self.have_alive_spider(): + log.info( + f"插入新批次记录成功，检测到有爬虫进程在等待任务，本批任务1分钟后开始下发, 防止爬虫端缓存的批次时间没来得及更新" + ) + tools.delay_time(60) + else: + log.info("插入新批次记录成功") return False # 下一批次开始 @@ -734,9 +747,12 @@ def check_batch(self, is_first_check=False): last_batch_date=last_batch_date, ) if result: - deal_speed, need_time, overflow_time, calculate_speed_time = ( - result - ) + ( + deal_speed, + need_time, + overflow_time, + calculate_speed_time, + ) = result msg += ", 任务处理速度于{}统计, 约 {}条/小时, 预计还需 {}".format( calculate_speed_time, deal_speed, @@ -749,14 +765,12 @@ def check_batch(self, is_first_check=False): ) log.info(msg) - - if ( - not self._last_send_msg_time - or now_date - self._last_send_msg_time - >= self._send_msg_interval - ): - self._last_send_msg_time = now_date - self.send_msg(msg, level="error") + self.send_msg( + msg, + level="error", + message_prefix="《{}》批次超时".format(self._batch_name), + ) + self._batch_timeout = True else: # 未超时 remaining_time = ( @@ -791,9 +805,12 @@ def check_batch(self, is_first_check=False): last_batch_date=last_batch_date, ) if result: - deal_speed, need_time, overflow_time, calculate_speed_time = ( - result - ) + ( + deal_speed, + need_time, + overflow_time, + calculate_speed_time, + ) = result msg += ", 任务处理速度于{}统计, 约 {}条/小时, 预计还需 {}".format( calculate_speed_time, deal_speed, @@ -805,13 +822,12 @@ def check_batch(self, is_first_check=False): tools.format_seconds(overflow_time) ) # 发送警报 - if ( - not self._last_send_msg_time - or now_date - self._last_send_msg_time - >= self._send_msg_interval - ): - self._last_send_msg_time = now_date - self.send_msg(msg, level="error") + self.send_msg( + msg, + level="error", + message_prefix="《{}》批次可能超时".format(self._batch_name), + ) + self._batch_timeout = True elif overflow_time < 0: msg += ", 该批次预计提前 {} 完成".format( @@ -848,7 +864,7 @@ def related_spider_is_done(self): if is_done is None: log.warning("相关联的批次表不存在或无批次信息") - return None + return True if not is_done: return False @@ -872,18 +888,15 @@ def record_batch(self): batch_date = tools.get_current_date(self._date_format) - sql = ( - "insert into %s (batch_date, done_count, total_count, `interval`, interval_unit, create_time) values ('%s', %s, %s, %s, '%s', CURRENT_TIME)" - % ( - self._batch_record_table, - batch_date, - 0, - total_task_count, - self._batch_interval - if self._batch_interval >= 1 - else self._batch_interval * 24, - "day" if self._batch_interval >= 1 else "hour", - ) + sql = "insert into %s (batch_date, done_count, total_count, `interval`, interval_unit, create_time) values ('%s', %s, %s, %s, '%s', CURRENT_TIME)" % ( + self._batch_record_table, + batch_date, + 0, + total_task_count, + self._batch_interval + if self._batch_interval >= 1 + else self._batch_interval * 24, + "day" if self._batch_interval >= 1 else "hour", ) affect_count = self._mysqldb.add(sql) # None / 0 / 1 (1 为成功) @@ -895,13 +908,6 @@ def record_batch(self): # 爬虫开始 self.spider_begin() - self.record_spider_state( - spider_type=2, - state=0, - batch_date=batch_date, - spider_start_time=tools.get_current_date(), - batch_interval=self._batch_interval, - ) else: log.error("插入新批次失败") @@ -948,12 +954,7 @@ def task_is_done(self): if is_done: # 检查任务表中是否有没做的任务 若有则is_done 为 False # 比较耗时 加锁防止多进程同时查询 - with RedisLock( - key=self._spider_name, - timeout=3600, - wait_timeout=0, - redis_cli=RedisDB().get_redis_obj(), - ) as lock: + with RedisLock(key=self._spider_name) as lock: if lock.locked: log.info("批次表标记已完成，正在检查任务表是否有未完成的任务") @@ -1000,34 +1001,33 @@ def run(self): self._start() while True: - if ( - self.task_is_done() and self.all_thread_is_done() - ): # redis全部的任务已经做完 并且mysql中的任务已经做完（检查各个线程all_thread_is_done，防止任务没做完，就更新任务状态，导致程序结束的情况） - if not self._is_notify_end: - self.spider_end() - self.record_spider_state( - spider_type=2, - state=1, - batch_date=self._batch_date_cache, - spider_end_time=tools.get_current_date(), - batch_interval=self._batch_interval, - ) + try: + if self._stop_spider or ( + self.task_is_done() and self.all_thread_is_done() + ): # redis全部的任务已经做完 并且mysql中的任务已经做完（检查各个线程all_thread_is_done，防止任务没做完，就更新任务状态，导致程序结束的情况） + if not self._is_notify_end: + self.spider_end() + self._is_notify_end = True + + if not self._keep_alive: + self._stop_all_thread() + break + else: + self._is_notify_end = False - self._is_notify_end = True + self.check_task_status() - if self._auto_stop_when_spider_done: - self._stop_all_thread() - break - else: - self._is_notify_end = False + except Exception as e: + log.exception(e) - self.check_task_status() tools.delay_time(10) # 10秒钟检查一次爬虫状态 except Exception as e: msg = "《%s》主线程异常 爬虫结束 exception: %s" % (self._batch_name, e) log.error(msg) - self.send_msg(msg) + self.send_msg( + msg, level="error", message_prefix="《%s》爬虫异常结束".format(self._batch_name) + ) os._exit(137) # 使退出码为35072 方便爬虫管理器重启 @@ -1045,15 +1045,12 @@ class DebugBatchSpider(BatchSpider): """ __debug_custom_setting__ = dict( - COLLECTOR_SLEEP_TIME=1, COLLECTOR_TASK_COUNT=1, # SPIDER SPIDER_THREAD_COUNT=1, SPIDER_SLEEP_TIME=0, - SPIDER_TASK_COUNT=1, SPIDER_MAX_RETRY_TIMES=10, - REQUEST_TIME_OUT=600, # 10分钟 - ADD_ITEM_TO_MYSQL=False, + REQUEST_LOST_TIMEOUT=600, # 10分钟 PROXY_ENABLE=False, RETRY_FAILED_REQUESTS=False, # 保存失败的request @@ -1062,7 +1059,7 @@ class DebugBatchSpider(BatchSpider): ITEM_FILTER_ENABLE=False, REQUEST_FILTER_ENABLE=False, OSS_UPLOAD_TABLES=(), - DELETE_TABS=True, + DELETE_KEYS=True, ) def __init__( @@ -1070,15 +1067,15 @@ def __init__( task_id=None, task=None, save_to_db=False, - update_stask=False, + update_task=False, *args, - **kwargs + **kwargs, ): """ @param task_id: 任务id @param task: 任务 task 与 task_id 二者选一即可 @param save_to_db: 数据是否入库 默认否 - @param update_stask: 是否更新任务 默认否 + @param update_task: 是否更新任务 默认否 @param args: @param kwargs: """ @@ -1090,8 +1087,11 @@ def __init__( raise Exception("task_id 与 task 不能同时为null") kwargs["redis_key"] = kwargs["redis_key"] + "_debug" - if save_to_db: - self.__class__.__debug_custom_setting__.update(ADD_ITEM_TO_MYSQL=True) + if not save_to_db: + self.__class__.__debug_custom_setting__["ITEM_PIPELINES"] = [ + CONSOLE_PIPELINE_PATH + ] + self.__class__.__custom_setting__.update( self.__class__.__debug_custom_setting__ ) @@ -1100,7 +1100,7 @@ def __init__( self._task_id = task_id self._task = task - self._update_task = update_stask + self._update_task = update_task def start_monitor_task(self): """ @@ -1136,8 +1136,9 @@ def get_todo_task_from_mysql(self): """ # 查询任务 + task_keys = ", ".join([f"`{key}`" for key in self._task_keys]) sql = "select %s from %s where id=%s" % ( - ", ".join(self._task_keys), + task_keys, self._task_table, self._task_id, ) @@ -1192,22 +1193,6 @@ def update_task_batch(self, task_id, state=1, *args, **kwargs): return update_item - def delete_tables(self, delete_tables_list): - if isinstance(delete_tables_list, bool): - delete_tables_list = [self._redis_key + "*"] - elif not isinstance(delete_tables_list, (list, tuple)): - delete_tables_list = [delete_tables_list] - - redis = RedisDB() - for delete_tab in delete_tables_list: - if delete_tab == "*": - delete_tab = self._redis_key + "*" - - tables = redis.getkeys(delete_tab) - for table in tables: - log.info("正在删除表 %s" % table) - redis.clear(table) - def run(self): self.start_monitor_task() @@ -1217,21 +1202,14 @@ def run(self): self._start() while True: - if self.all_thread_is_done(): - self._stop_all_thread() - break + try: + if self.all_thread_is_done(): + self._stop_all_thread() + break + + except Exception as e: + log.exception(e) tools.delay_time(1) # 1秒钟检查一次爬虫状态 self.delete_tables([self._redis_key + "*"]) - - def record_spider_state( - self, - spider_type, - state, - batch_date=None, - spider_start_time=None, - spider_end_time=None, - batch_interval=None, - ): - pass diff --git a/feapder/core/spiders/spider.py b/feapder/core/spiders/spider.py index 60663682..a1097559 100644 --- a/feapder/core/spiders/spider.py +++ b/feapder/core/spiders/spider.py @@ -5,12 +5,12 @@ @summary: --------- @author: Boris -@email: boris@bzkj.tech +@email: boris_liu@foxmail.com """ import time import warnings -from collections import Iterable +from collections.abc import Iterable import feapder.setting as setting import feapder.utils.tools as tools @@ -21,6 +21,8 @@ from feapder.network.request import Request from feapder.utils.log import log +CONSOLE_PIPELINE_PATH = "feapder.pipelines.console_pipeline.ConsolePipeline" + class Spider( BaseParser, Scheduler @@ -38,13 +40,12 @@ def __init__( thread_count=None, begin_callback=None, end_callback=None, - delete_tabs=(), - process_num=None, - auto_stop_when_spider_done=None, + delete_keys=(), + keep_alive=None, auto_start_requests=None, - send_run_time=False, batch_interval=0, - wait_lock=True + wait_lock=True, + **kwargs ): """ @summary: 爬虫 @@ -55,11 +56,9 @@ def __init__( @param thread_count: 线程数，默认为配置文件中的线程数 @param begin_callback: 爬虫开始回调函数 @param end_callback: 爬虫结束回调函数 - @param delete_tabs: 爬虫启动时删除的表（redis里的key），元组类型。 支持正则; 常用于清空任务队列，否则重启时会断点续爬 - @param process_num: 进程数 - @param auto_stop_when_spider_done: 爬虫抓取完毕后是否自动结束或等待任务，默认自动结束 + @param delete_keys: 爬虫启动时删除的key，类型: 元组/bool/string。 支持正则; 常用于清空任务队列，否则重启时会断点续爬 + @param keep_alive: 爬虫是否常驻 @param auto_start_requests: 爬虫是否自动添加任务 - @param send_run_time: 发送运行时间 @param batch_interval: 抓取时间间隔 默认为0 天为单位 多次启动时，只有当前时间与第一次抓取结束的时间间隔大于指定的时间间隔时，爬虫才启动 @param wait_lock: 下发任务时否等待锁，若不等待锁，可能会存在多进程同时在下发一样的任务，因此分布式环境下请将该值设置True --------- @@ -70,13 +69,12 @@ def __init__( thread_count=thread_count, begin_callback=begin_callback, end_callback=end_callback, - delete_tabs=delete_tabs, - process_num=process_num, - auto_stop_when_spider_done=auto_stop_when_spider_done, + delete_keys=delete_keys, + keep_alive=keep_alive, auto_start_requests=auto_start_requests, - send_run_time=send_run_time, batch_interval=batch_interval, wait_lock=wait_lock, + **kwargs ) self._min_task_count = min_task_count @@ -98,9 +96,7 @@ def start_monitor_task(self, *args, **kws): while True: try: # 检查redis中是否有任务 - tab_requests = setting.TAB_REQUSETS.format( - redis_key=self._redis_key - ) + tab_requests = setting.TAB_REQUESTS.format(redis_key=self._redis_key) todo_task_count = redisdb.zget_count(tab_requests) if todo_task_count < self._min_task_count: # 添加任务 @@ -113,7 +109,7 @@ def start_monitor_task(self, *args, **kws): except Exception as e: log.exception(e) - if self._auto_stop_when_spider_done: + if not self._keep_alive: break time.sleep(self._check_task_interval) @@ -164,13 +160,6 @@ def distribute_task(self, *args, **kws): if self._is_distributed_task: # 有任务时才提示启动爬虫 # begin self.spider_begin() - self.record_spider_state( - spider_type=1, - state=0, - batch_date=tools.get_current_date(), - spider_start_time=tools.get_current_date(), - batch_interval=self._batch_interval, - ) # 重置已经提示无任务状态为False self._is_show_not_task = False @@ -194,26 +183,22 @@ def run(self): self._start() while True: - if self.all_thread_is_done(): - if not self._is_notify_end: - self.spider_end() # 跑完一轮 - self.record_spider_state( - spider_type=1, - state=1, - spider_end_time=tools.get_current_date(), - batch_interval=self._batch_interval, - ) - - self._is_notify_end = True + try: + if self._stop_spider or self.all_thread_is_done(): + if not self._is_notify_end: + self.spider_end() # 跑完一轮 + self._is_notify_end = True - if self._auto_stop_when_spider_done: - self._stop_all_thread() - break + if not self._keep_alive: + self._stop_all_thread() + break - else: - self._is_notify_end = False + else: + self._is_notify_end = False - self.check_task_status() + self.check_task_status() + except Exception as e: + log.exception(e) tools.delay_time(1) # 1秒钟检查一次爬虫状态 @@ -231,15 +216,12 @@ class DebugSpider(Spider): """ __debug_custom_setting__ = dict( - COLLECTOR_SLEEP_TIME=1, COLLECTOR_TASK_COUNT=1, # SPIDER SPIDER_THREAD_COUNT=1, SPIDER_SLEEP_TIME=0, - SPIDER_TASK_COUNT=1, SPIDER_MAX_RETRY_TIMES=10, - REQUEST_TIME_OUT=600, # 10分钟 - ADD_ITEM_TO_MYSQL=False, + REQUEST_LOST_TIMEOUT=600, # 10分钟 PROXY_ENABLE=False, RETRY_FAILED_REQUESTS=False, # 保存失败的request @@ -248,13 +230,16 @@ class DebugSpider(Spider): ITEM_FILTER_ENABLE=False, REQUEST_FILTER_ENABLE=False, OSS_UPLOAD_TABLES=(), - DELETE_TABS=True, + DELETE_KEYS=True, ) - def __init__(self, request=None, request_dict=None, *args, **kwargs): + def __init__( + self, request=None, request_dict=None, save_to_db=False, *args, **kwargs + ): """ @param request: request 类对象 @param request_dict: request 字典。 request 与 request_dict 二者选一即可 + @param save_to_db: 数据是否入库 默认否 @param kwargs: """ warnings.warn( @@ -265,6 +250,10 @@ def __init__(self, request=None, request_dict=None, *args, **kwargs): raise Exception("request 与 request_dict 不能同时为null") kwargs["redis_key"] = kwargs["redis_key"] + "_debug" + if not save_to_db: + self.__class__.__debug_custom_setting__["ITEM_PIPELINES"] = [ + CONSOLE_PIPELINE_PATH + ] self.__class__.__custom_setting__.update( self.__class__.__debug_custom_setting__ ) @@ -276,22 +265,6 @@ def __init__(self, request=None, request_dict=None, *args, **kwargs): def save_cached(self, request, response, table): pass - def delete_tables(self, delete_tables_list): - if isinstance(delete_tables_list, bool): - delete_tables_list = [self._redis_key + "*"] - elif not isinstance(delete_tables_list, (list, tuple)): - delete_tables_list = [delete_tables_list] - - redis = RedisDB() - for delete_tab in delete_tables_list: - if delete_tab == "*": - delete_tab = self._redis_key + "*" - - tables = redis.getkeys(delete_tab) - for table in tables: - log.info("正在删除表 %s" % table) - redis.clear(table) - def __start_requests(self): yield self._request @@ -334,13 +307,6 @@ def distribute_task(self): if self._is_distributed_task: # 有任务时才提示启动爬虫 # begin self.spider_begin() - self.record_spider_state( - spider_type=1, - state=0, - batch_date=tools.get_current_date(), - spider_start_time=tools.get_current_date(), - batch_interval=self._batch_interval, - ) # 重置已经提示无任务状态为False self._is_show_not_task = False @@ -354,17 +320,6 @@ def distribute_task(self): self._is_show_not_task = True - def record_spider_state( - self, - spider_type, - state, - batch_date=None, - spider_start_time=None, - spider_end_time=None, - batch_interval=None, - ): - pass - def _start(self): # 启动parser 的 start_requests self.spider_begin() # 不自动结束的爬虫此处只能执行一遍 @@ -426,9 +381,12 @@ def run(self): self._start() while True: - if self.all_thread_is_done(): - self._stop_all_thread() - break + try: + if self.all_thread_is_done(): + self._stop_all_thread() + break + except Exception as e: + log.exception(e) tools.delay_time(1) # 1秒钟检查一次爬虫状态 diff --git a/feapder/core/spiders/task_spider.py b/feapder/core/spiders/task_spider.py new file mode 100644 index 00000000..41cb3596 --- /dev/null +++ b/feapder/core/spiders/task_spider.py @@ -0,0 +1,733 @@ +# -*- coding: utf-8 -*- +""" +Created on 2020/4/22 12:06 AM +--------- +@summary: +--------- +@author: Boris +@email: boris_liu@foxmail.com +""" + +import os +import time +import warnings +from collections.abc import Iterable +from typing import List, Tuple, Dict, Union + +import feapder.setting as setting +import feapder.utils.tools as tools +from feapder.core.base_parser import TaskParser +from feapder.core.scheduler import Scheduler +from feapder.db.mysqldb import MysqlDB +from feapder.db.redisdb import RedisDB +from feapder.network.item import Item +from feapder.network.item import UpdateItem +from feapder.network.request import Request +from feapder.utils.log import log +from feapder.utils.perfect_dict import PerfectDict + +CONSOLE_PIPELINE_PATH = "feapder.pipelines.console_pipeline.ConsolePipeline" + + +class TaskSpider(TaskParser, Scheduler): + def __init__( + self, + redis_key, + task_table, + task_table_type="mysql", + task_keys=None, + task_state="state", + min_task_count=10000, + check_task_interval=5, + task_limit=10000, + related_redis_key=None, + related_batch_record=None, + task_condition="", + task_order_by="", + thread_count=None, + begin_callback=None, + end_callback=None, + delete_keys=(), + keep_alive=None, + batch_interval=0, + use_mysql=True, + **kwargs, + ): + """ + @summary: 任务爬虫 + 必要条件 需要指定任务表，可以是redis表或者mysql表作为任务种子 + redis任务种子表：zset类型。值为 {"xxx":xxx, "xxx2":"xxx2"}；若为集成模式，需指定parser_name字段，如{"xxx":xxx, "xxx2":"xxx2", "parser_name":"TestTaskSpider"} + mysql任务表： + 任务表中必须有id及任务状态字段 如 state, 其他字段可根据爬虫需要的参数自行扩充。若为集成模式，需指定parser_name字段。 + + 参考建表语句如下： + CREATE TABLE `table_name` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `param` varchar(1000) DEFAULT NULL COMMENT '爬虫需要的抓取数据需要的参数', + `state` int(11) DEFAULT NULL COMMENT '任务状态', + `parser_name` varchar(255) DEFAULT NULL COMMENT '任务解析器的脚本类名', + PRIMARY KEY (`id`), + UNIQUE KEY `nui` (`param`) USING BTREE + ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8; + + --------- + @param task_table: mysql中的任务表 或 redis中存放任务种子的key，zset类型 + @param task_table_type: 任务表类型 支持 redis 、mysql + @param task_keys: 需要获取的任务字段 列表 [] 如需指定解析的parser，则需将parser_name字段取出来。 + @param task_state: mysql中任务表的任务状态字段 + @param min_task_count: redis 中最少任务数, 少于这个数量会从种子表中取任务 + @param check_task_interval: 检查是否还有任务的时间间隔； + @param task_limit: 每次从数据库中取任务的数量 + @param redis_key: 任务等数据存放在redis中的key前缀 + @param thread_count: 线程数，默认为配置文件中的线程数 + @param begin_callback: 爬虫开始回调函数 + @param end_callback: 爬虫结束回调函数 + @param delete_keys: 爬虫启动时删除的key，类型: 元组/bool/string。 支持正则; 常用于清空任务队列，否则重启时会断点续爬 + @param keep_alive: 爬虫是否常驻，默认否 + @param related_redis_key: 有关联的其他爬虫任务表（redis）注意：要避免环路 如 A -> B & B -> A 。 + @param related_batch_record: 有关联的其他爬虫批次表（mysql）注意：要避免环路 如 A -> B & B -> A 。 + related_redis_key 与 related_batch_record 选其一配置即可；用于相关联的爬虫没结束时，本爬虫也不结束 + 若相关连的爬虫为批次爬虫，推荐以related_batch_record配置， + 若相关连的爬虫为普通爬虫，无批次表，可以以related_redis_key配置 + @param task_condition: 任务条件 用于从一个大任务表中挑选出数据自己爬虫的任务，即where后的条件语句 + @param task_order_by: 取任务时的排序条件 如 id desc + @param batch_interval: 抓取时间间隔 默认为0 天为单位 多次启动时，只有当前时间与第一次抓取结束的时间间隔大于指定的时间间隔时，爬虫才启动 + @param use_mysql: 是否使用mysql数据库 + --------- + @result: + """ + Scheduler.__init__( + self, + redis_key=redis_key, + thread_count=thread_count, + begin_callback=begin_callback, + end_callback=end_callback, + delete_keys=delete_keys, + keep_alive=keep_alive, + auto_start_requests=False, + batch_interval=batch_interval, + task_table=task_table, + **kwargs, + ) + + self._redisdb = RedisDB() + self._mysqldb = MysqlDB() if use_mysql else None + + self._task_table = task_table # mysql中的任务表 + self._task_keys = task_keys # 需要获取的任务字段 + self._task_table_type = task_table_type + + if self._task_table_type == "mysql" and not self._task_keys: + raise Exception("需指定任务字段 使用task_keys") + + self._task_state = task_state # mysql中任务表的state字段名 + self._min_task_count = min_task_count # redis 中最少任务数 + self._check_task_interval = check_task_interval + self._task_limit = task_limit # mysql中一次取的任务数量 + self._related_task_tables = [ + setting.TAB_REQUESTS.format(redis_key=redis_key) + ] # 自己的task表也需要检查是否有任务 + if related_redis_key: + self._related_task_tables.append( + setting.TAB_REQUESTS.format(redis_key=related_redis_key) + ) + + self._related_batch_record = related_batch_record + self._task_condition = task_condition + self._task_condition_prefix_and = task_condition and " and {}".format( + task_condition + ) + self._task_condition_prefix_where = task_condition and " where {}".format( + task_condition + ) + self._task_order_by = task_order_by and " order by {}".format(task_order_by) + + self._is_more_parsers = True # 多模版类爬虫 + self.reset_task() + + def add_parser(self, parser, **kwargs): + parser = parser( + self._task_table, + self._task_state, + self._mysqldb, + **kwargs, + ) # parser 实例化 + self._parsers.append(parser) + + def start_monitor_task(self): + """ + @summary: 监控任务状态 + --------- + --------- + @result: + """ + if not self._parsers: # 不是多模版模式， 将自己注入到parsers，自己为模版 + self._is_more_parsers = False + self._parsers.append(self) + + elif len(self._parsers) <= 1: + self._is_more_parsers = False + + # 添加任务 + for parser in self._parsers: + parser.add_task() + + while True: + try: + # 检查redis中是否有任务 任务小于_min_task_count 则从mysql中取 + tab_requests = setting.TAB_REQUESTS.format(redis_key=self._redis_key) + todo_task_count = self._redisdb.zget_count(tab_requests) + + tasks = [] + if todo_task_count < self._min_task_count: + tasks = self.get_task(todo_task_count) + if not tasks: + if not todo_task_count: + if self._keep_alive: + log.info("任务均已做完，爬虫常驻, 等待新任务") + time.sleep(self._check_task_interval) + continue + elif self.have_alive_spider(): + log.info("任务均已做完，但还有爬虫在运行，等待爬虫结束") + time.sleep(self._check_task_interval) + continue + elif not self.related_spider_is_done(): + continue + else: + log.info("任务均已做完，爬虫结束") + break + + else: + log.info("redis 中尚有%s条积压任务，暂时不派发新任务" % todo_task_count) + + if not tasks: + if todo_task_count >= self._min_task_count: + # log.info('任务正在进行 redis中剩余任务 %s' % todo_task_count) + pass + else: + log.info("无待做种子 redis中剩余任务 %s" % todo_task_count) + else: + # make start requests + self.distribute_task(tasks) + log.info(f"添加任务到redis成功 共{len(tasks)}条") + + except Exception as e: + log.exception(e) + + time.sleep(self._check_task_interval) + + def get_task(self, todo_task_count) -> List[Union[Tuple, Dict]]: + """ + 获取任务 + Args: + todo_task_count: redis里剩余的任务数 + + Returns: + + """ + tasks = [] + if self._task_table_type == "mysql": + # 从mysql中取任务 + log.info("redis 中剩余任务%s 数量过小 从mysql中取任务追加" % todo_task_count) + tasks = self.get_todo_task_from_mysql() + if not tasks: # 状态为0的任务已经做完，需要检查状态为2的任务是否丢失 + # redis 中无待做任务，此时mysql中状态为2的任务为丢失任务。需重新做 + if todo_task_count == 0: + log.info("无待做任务，尝试取丢失的任务") + tasks = self.get_doing_task_from_mysql() + elif self._task_table_type == "redis": + log.info("redis 中剩余任务%s 数量过小 从redis种子任务表中取任务追加" % todo_task_count) + tasks = self.get_task_from_redis() + else: + raise Exception( + f"task_table_type expect mysql or redis，bug got {self._task_table_type}" + ) + + return tasks + + def distribute_task(self, tasks): + """ + @summary: 分发任务 + --------- + @param tasks: + --------- + @result: + """ + if self._is_more_parsers: # 为多模版类爬虫，需要下发指定的parser + for task in tasks: + for parser in self._parsers: # 寻找task对应的parser + if parser.name in task: + if isinstance(task, dict): + task = PerfectDict(_dict=task) + else: + task = PerfectDict( + _dict=dict(zip(self._task_keys, task)), + _values=list(task), + ) + requests = parser.start_requests(task) + if requests and not isinstance(requests, Iterable): + raise Exception( + "%s.%s返回值必须可迭代" % (parser.name, "start_requests") + ) + + result_type = 1 + for request in requests or []: + if isinstance(request, Request): + request.parser_name = request.parser_name or parser.name + self._request_buffer.put_request(request) + result_type = 1 + + elif isinstance(request, Item): + self._item_buffer.put_item(request) + result_type = 2 + + if ( + self._item_buffer.get_items_count() + >= setting.ITEM_MAX_CACHED_COUNT + ): + self._item_buffer.flush() + + elif callable(request): # callbale的request可能是更新数据库操作的函数 + if result_type == 1: + self._request_buffer.put_request(request) + else: + self._item_buffer.put_item(request) + + if ( + self._item_buffer.get_items_count() + >= setting.ITEM_MAX_CACHED_COUNT + ): + self._item_buffer.flush() + + else: + raise TypeError( + "start_requests yield result type error, expect Request、Item、callback func, bug get type: {}".format( + type(requests) + ) + ) + + break + + else: # task没对应的parser 则将task下发到所有的parser + for task in tasks: + for parser in self._parsers: + if isinstance(task, dict): + task = PerfectDict(_dict=task) + else: + task = PerfectDict( + _dict=dict(zip(self._task_keys, task)), _values=list(task) + ) + requests = parser.start_requests(task) + if requests and not isinstance(requests, Iterable): + raise Exception( + "%s.%s返回值必须可迭代" % (parser.name, "start_requests") + ) + + result_type = 1 + for request in requests or []: + if isinstance(request, Request): + request.parser_name = request.parser_name or parser.name + self._request_buffer.put_request(request) + result_type = 1 + + elif isinstance(request, Item): + self._item_buffer.put_item(request) + result_type = 2 + + if ( + self._item_buffer.get_items_count() + >= setting.ITEM_MAX_CACHED_COUNT + ): + self._item_buffer.flush() + + elif callable(request): # callbale的request可能是更新数据库操作的函数 + if result_type == 1: + self._request_buffer.put_request(request) + else: + self._item_buffer.put_item(request) + + if ( + self._item_buffer.get_items_count() + >= setting.ITEM_MAX_CACHED_COUNT + ): + self._item_buffer.flush() + + self._request_buffer.flush() + self._item_buffer.flush() + + def get_task_from_redis(self): + tasks = self._redisdb.zget(self._task_table, count=self._task_limit) + tasks = [eval(task) for task in tasks] + return tasks + + def get_todo_task_from_mysql(self): + """ + @summary: 取待做的任务 + --------- + --------- + @result: + """ + # TODO 分批取数据 每批最大取 1000000个，防止内存占用过大 + # 查询任务 + task_keys = ", ".join([f"`{key}`" for key in self._task_keys]) + sql = "select %s from %s where %s = 0%s%s limit %s" % ( + task_keys, + self._task_table, + self._task_state, + self._task_condition_prefix_and, + self._task_order_by, + self._task_limit, + ) + tasks = self._mysqldb.find(sql) + + if tasks: + # 更新任务状态 + for i in range(0, len(tasks), 10000): # 10000 一批量更新 + task_ids = str( + tuple([task[0] for task in tasks[i : i + 10000]]) + ).replace(",)", ")") + sql = "update %s set %s = 2 where id in %s" % ( + self._task_table, + self._task_state, + task_ids, + ) + self._mysqldb.update(sql) + + return tasks + + def get_doing_task_from_mysql(self): + """ + @summary: 取正在做的任务 + --------- + --------- + @result: + """ + + # 查询任务 + task_keys = ", ".join([f"`{key}`" for key in self._task_keys]) + sql = "select %s from %s where %s = 2%s%s limit %s" % ( + task_keys, + self._task_table, + self._task_state, + self._task_condition_prefix_and, + self._task_order_by, + self._task_limit, + ) + tasks = self._mysqldb.find(sql) + + return tasks + + def get_lose_task_count(self): + sql = "select count(1) from %s where %s = 2%s" % ( + self._task_table, + self._task_state, + self._task_condition_prefix_and, + ) + doing_count = self._mysqldb.find(sql)[0][0] + return doing_count + + def reset_lose_task_from_mysql(self): + """ + @summary: 重置丢失任务为待做 + --------- + --------- + @result: + """ + + sql = "update {table} set {state} = 0 where {state} = 2{task_condition}".format( + table=self._task_table, + state=self._task_state, + task_condition=self._task_condition_prefix_and, + ) + return self._mysqldb.update(sql) + + def related_spider_is_done(self): + """ + 相关连的爬虫是否跑完 + @return: True / False / None 表示无相关的爬虫 可由自身的total_count 和 done_count 来判断 + """ + + for related_redis_task_table in self._related_task_tables: + if self._redisdb.exists_key(related_redis_task_table): + log.info(f"依赖的爬虫还未结束，任务表为：{related_redis_task_table}") + return False + + if self._related_batch_record: + sql = "select is_done from {} order by id desc limit 1".format( + self._related_batch_record + ) + is_done = self._mysqldb.find(sql) + is_done = is_done[0][0] if is_done else None + + if is_done is None: + log.warning("相关联的批次表不存在或无批次信息") + return True + + if not is_done: + log.info(f"依赖的爬虫还未结束，批次表为：{self._related_batch_record}") + return False + + return True + + # -------- 批次结束逻辑 ------------ + + def task_is_done(self): + """ + @summary: 检查种子表是否做完 + --------- + --------- + @result: True / False （做完 / 未做完） + """ + is_done = False + if self._task_table_type == "mysql": + sql = "select 1 from %s where (%s = 0 or %s=2)%s limit 1" % ( + self._task_table, + self._task_state, + self._task_state, + self._task_condition_prefix_and, + ) + count = self._mysqldb.find(sql) # [(1,)] / [] + elif self._task_table_type == "redis": + count = self._redisdb.zget_count(self._task_table) + else: + raise Exception( + f"task_table_type expect mysql or redis，bug got {self._task_table_type}" + ) + + if not count: + log.info("种子表中任务均已完成") + is_done = True + + return is_done + + def run(self): + """ + @summary: 重写run方法 检查mysql中的任务是否做完， 做完停止 + --------- + --------- + @result: + """ + try: + if not self.is_reach_next_spider_time(): + return + + if not self._parsers: # 不是add_parser 模式 + self._parsers.append(self) + + self._start() + + while True: + try: + if self._stop_spider or ( + self.all_thread_is_done() + and self.task_is_done() + and self.related_spider_is_done() + ): # redis全部的任务已经做完 并且mysql中的任务已经做完（检查各个线程all_thread_is_done，防止任务没做完，就更新任务状态，导致程序结束的情况） + if not self._is_notify_end: + self.spider_end() + self._is_notify_end = True + + if not self._keep_alive: + self._stop_all_thread() + break + else: + log.info("常驻爬虫，等待新任务") + else: + self._is_notify_end = False + + self.check_task_status() + + except Exception as e: + log.exception(e) + + tools.delay_time(10) # 10秒钟检查一次爬虫状态 + + except Exception as e: + msg = "《%s》主线程异常 爬虫结束 exception: %s" % (self.name, e) + log.error(msg) + self.send_msg( + msg, level="error", message_prefix="《%s》爬虫异常结束".format(self.name) + ) + + os._exit(137) # 使退出码为35072 方便爬虫管理器重启 + + @classmethod + def to_DebugTaskSpider(cls, *args, **kwargs): + # DebugBatchSpider 继承 cls + DebugTaskSpider.__bases__ = (cls,) + DebugTaskSpider.__name__ = cls.__name__ + return DebugTaskSpider(*args, **kwargs) + + +class DebugTaskSpider(TaskSpider): + """ + Debug批次爬虫 + """ + + __debug_custom_setting__ = dict( + COLLECTOR_TASK_COUNT=1, + # SPIDER + SPIDER_THREAD_COUNT=1, + SPIDER_SLEEP_TIME=0, + SPIDER_MAX_RETRY_TIMES=10, + REQUEST_LOST_TIMEOUT=600, # 10分钟 + PROXY_ENABLE=False, + RETRY_FAILED_REQUESTS=False, + # 保存失败的request + SAVE_FAILED_REQUEST=False, + # 过滤 + ITEM_FILTER_ENABLE=False, + REQUEST_FILTER_ENABLE=False, + OSS_UPLOAD_TABLES=(), + DELETE_KEYS=True, + ) + + def __init__( + self, + task_id=None, + task=None, + save_to_db=False, + update_task=False, + *args, + **kwargs, + ): + """ + @param task_id: 任务id + @param task: 任务 task 与 task_id 二者选一即可。如 task = {"url":""} + @param save_to_db: 数据是否入库 默认否 + @param update_task: 是否更新任务 默认否 + @param args: + @param kwargs: + """ + warnings.warn( + "您正处于debug模式下，该模式下不会更新任务状态及数据入库，仅用于调试。正式发布前请更改为正常模式", category=Warning + ) + + if not task and not task_id: + raise Exception("task_id 与 task 不能同时为空") + + kwargs["redis_key"] = kwargs["redis_key"] + "_debug" + if not save_to_db: + self.__class__.__debug_custom_setting__["ITEM_PIPELINES"] = [ + CONSOLE_PIPELINE_PATH + ] + self.__class__.__custom_setting__.update( + self.__class__.__debug_custom_setting__ + ) + + super(DebugTaskSpider, self).__init__(*args, **kwargs) + + self._task_id = task_id + self._task = task + self._update_task = update_task + + def start_monitor_task(self): + """ + @summary: 监控任务状态 + --------- + --------- + @result: + """ + if not self._parsers: # 不是多模版模式， 将自己注入到parsers，自己为模版 + self._is_more_parsers = False + self._parsers.append(self) + + elif len(self._parsers) <= 1: + self._is_more_parsers = False + + if self._task: + self.distribute_task([self._task]) + else: + tasks = self.get_todo_task_from_mysql() + if not tasks: + raise Exception("未获取到任务 请检查 task_id: {} 是否存在".format(self._task_id)) + self.distribute_task(tasks) + + log.debug("下发任务完毕") + + def get_todo_task_from_mysql(self): + """ + @summary: 取待做的任务 + --------- + --------- + @result: + """ + + # 查询任务 + task_keys = ", ".join([f"`{key}`" for key in self._task_keys]) + sql = "select %s from %s where id=%s" % ( + task_keys, + self._task_table, + self._task_id, + ) + tasks = self._mysqldb.find(sql) + + return tasks + + def save_cached(self, request, response, table): + pass + + def update_task_state(self, task_id, state=1, *args, **kwargs): + """ + @summary: 更新任务表中任务状态，做完每个任务时代码逻辑中要主动调用。可能会重写 + 调用方法为 yield lambda : self.update_task_state(task_id, state) + --------- + @param task_id: + @param state: + --------- + @result: + """ + if self._update_task: + kwargs["id"] = task_id + kwargs[self._task_state] = state + + sql = tools.make_update_sql( + self._task_table, + kwargs, + condition="id = {task_id}".format(task_id=task_id), + ) + + if self._mysqldb.update(sql): + log.debug("置任务%s状态成功" % task_id) + else: + log.error("置任务%s状态失败 sql=%s" % (task_id, sql)) + + def update_task_batch(self, task_id, state=1, *args, **kwargs): + """ + 批量更新任务 多处调用，更新的字段必须一致 + 注意：需要 写成 yield update_task_batch(...) 否则不会更新 + @param task_id: + @param state: + @param kwargs: + @return: + """ + if self._update_task: + kwargs["id"] = task_id + kwargs[self._task_state] = state + + update_item = UpdateItem(**kwargs) + update_item.table_name = self._task_table + update_item.name_underline = self._task_table + "_item" + + return update_item + + def run(self): + self.start_monitor_task() + + if not self._parsers: # 不是add_parser 模式 + self._parsers.append(self) + + self._start() + + while True: + try: + if self.all_thread_is_done(): + self._stop_all_thread() + break + + except Exception as e: + log.exception(e) + + tools.delay_time(1) # 1秒钟检查一次爬虫状态 + + self.delete_tables([self._redis_key + "*"]) diff --git a/feapder/db/__init__.py b/feapder/db/__init__.py index 0566204e..d2f0fb9f 100644 --- a/feapder/db/__init__.py +++ b/feapder/db/__init__.py @@ -5,5 +5,5 @@ @summary: --------- @author: Boris -@email: boris@bzkj.tech +@email: boris_liu@foxmail.com """ \ No newline at end of file diff --git a/feapder/db/memory_db.py b/feapder/db/memorydb.py similarity index 52% rename from feapder/db/memory_db.py rename to feapder/db/memorydb.py index 24efbc6a..99c8c7d6 100644 --- a/feapder/db/memory_db.py +++ b/feapder/db/memorydb.py @@ -5,22 +5,29 @@ @summary: 基于内存的队列，代替redis --------- @author: Boris -@email: boris@bzkj.tech +@email: boris_liu@foxmail.com """ from queue import PriorityQueue +from feapder import setting + class MemoryDB: def __init__(self): - self.priority_queue = PriorityQueue() + self.priority_queue = PriorityQueue(maxsize=setting.TASK_MAX_CACHED_SIZE) - def add(self, item): + def add(self, item, ignore_max_size=False): """ 添加任务 :param item: 数据: 支持小于号比较的类 或者 （priority, item） + :param ignore_max_size: queue满时是否等待，为True时无视队列的maxsize，直接往里塞 :return: """ - self.priority_queue.put(item) + if ignore_max_size: + self.priority_queue._put(item) + self.priority_queue.unfinished_tasks += 1 + else: + self.priority_queue.put(item) def get(self): """ @@ -28,7 +35,7 @@ def get(self): :return: """ try: - item = self.priority_queue.get_nowait() + item = self.priority_queue.get(timeout=1) return item except: return diff --git a/feapder/db/mongodb.py b/feapder/db/mongodb.py new file mode 100644 index 00000000..791fe0d9 --- /dev/null +++ b/feapder/db/mongodb.py @@ -0,0 +1,501 @@ +# -*- coding: utf-8 -*- +""" +Created on 2021-04-18 14:12:21 +--------- +@summary: 操作mongo数据库 +--------- +@author: Mkdir700 +@email: mkdir700@gmail.com +""" +import re +from typing import List, Dict, Optional +from urllib import parse + +import pymongo +from pymongo import MongoClient, UpdateOne, UpdateMany +from pymongo.collection import Collection +from pymongo.database import Database +from pymongo.errors import DuplicateKeyError, BulkWriteError + +import feapder.setting as setting +from feapder.utils.log import log + + +class MongoDB: + def __init__( + self, + ip=None, + port=None, + db=None, + user_name=None, + user_pass=None, + url=None, + **kwargs, + ): + if not ip: + ip = setting.MONGO_IP + if not port: + port = setting.MONGO_PORT + if not db: + db = setting.MONGO_DB + if not user_name: + user_name = setting.MONGO_USER_NAME + if not user_pass: + user_pass = setting.MONGO_USER_PASS + if not url: + url = setting.MONGO_URL + + if url: + self.client = MongoClient(url, **kwargs) + else: + self.client = MongoClient( + host=ip, port=port, username=user_name, password=user_pass, **kwargs + ) + + self.db = self.get_database(db) + + # 缓存索引信息 + self.__index__cached = {} + + @classmethod + def from_url(cls, url, **kwargs): + """ + Args: + url: mongodb://[username:password@]host1[:port1][,host2[:port2],...[,hostN[:portN]]][/[database][?options]] + 参考：http://mongodb.github.io/mongo-java-driver/3.4/javadoc/com/mongodb/MongoClientURI.html + **kwargs: + + Returns: + + """ + url_parsed = parse.urlparse(url) + + db_type = url_parsed.scheme.strip() + if db_type != "mongodb": + raise Exception( + "url error, expect mongodb://[username:password@]host1[:port1][,host2[:port2],...[,hostN[:portN]]][/[database][?options]], but get {}".format( + url + ) + ) + + return cls(url=url, **kwargs) + + def get_database(self, database, **kwargs) -> Database: + """ + 获取数据库对象 + @param database: 数据库名 + @return: + """ + return self.client.get_database(database, **kwargs) + + def get_collection(self, coll_name, **kwargs) -> Collection: + """ + 根据集合名获取集合对象 + @param coll_name: 集合名 + @return: + """ + return self.db.get_collection(coll_name, **kwargs) + + def find( + self, coll_name: str, condition: Optional[Dict] = None, limit: int = 0, **kwargs + ) -> List[Dict]: + """ + @summary: + 无数据： 返回[] + 有数据： [{'_id': 'xx', ...}, ...] + --------- + @param coll_name: 集合名(表名) + @param condition: 查询条件 + @param limit: 结果数量 + @param kwargs: + 更多参数 https://docs.mongodb.com/manual/reference/command/find/#command-fields + + --------- + @result: + """ + condition = {} if condition is None else condition + command = {"find": coll_name, "filter": condition, "limit": limit} + command.update(kwargs) + result = self.run_command(command) + cursor = result["cursor"] + cursor_id = cursor["id"] + dataset = cursor["firstBatch"] + while True: + if cursor_id == 0: + break + result = self.run_command( + { + "getMore": cursor_id, + "collection": coll_name, + "batchSize": kwargs.get("batchSize", 100), + } + ) + cursor = result["cursor"] + cursor_id = cursor["id"] + dataset.extend(cursor["nextBatch"]) + return dataset + + def add( + self, + coll_name, + data: Dict, + replace=False, + update_columns=(), + update_columns_value=(), + insert_ignore=False, + ): + """ + 添加单条数据 + Args: + coll_name: 集合名 + data: 单条数据 + replace: 唯一索引冲突时直接覆盖旧数据，默认为False + update_columns: 更新指定的列（如果数据唯一索引冲突，则更新指定字段，如 update_columns = ["name", "title"] + update_columns_value: 指定更新的字段对应的值, 不指定则用数据本身的值更新 + insert_ignore: 索引冲突是否忽略 默认False + + Returns: 插入成功的行数 + + """ + affect_count = 1 + collection = self.get_collection(coll_name) + try: + collection.insert_one(data) + except DuplicateKeyError as e: + # 存在则更新 + if update_columns: + if not isinstance(update_columns, (tuple, list)): + update_columns = [update_columns] + + condition = self.__get_update_condition( + coll_name, data, e.details.get("errmsg") + ) + + # 更新指定的列 + if update_columns_value: + # 使用指定的值更新 + doc = { + key: value + for key, value in zip(update_columns, update_columns_value) + } + else: + # 使用数据本身的值更新 + doc = {key: data[key] for key in update_columns} + + collection.update_one(condition, {"$set": doc}) + + # 覆盖更新 + elif replace: + condition = self.__get_update_condition( + coll_name, data, e.details.get("errmsg") + ) + # 替换已存在的数据 + collection.replace_one(condition, data) + + elif not insert_ignore: + raise e + + return affect_count + + def add_batch( + self, + coll_name: str, + datas: List[Dict], + replace=False, + update_columns=(), + update_columns_value=(), + condition_fields: dict = None, + ): + """ + 批量添加数据 + Args: + coll_name: 集合名 + datas: 数据 [{'_id': 'xx'}, ... ] + replace: 唯一索引冲突时直接覆盖旧数据，默认为False + update_columns: 更新指定的列（如果数据的唯一索引存在，则更新指定字段，如 update_columns = ["name", "title"] + update_columns_value: 指定更新的字段对应的值, 不指定则用数据本身的值更新 + condition_fields: 用于条件查找的字段，不指定则用索引冲突中的字段查找 + + Returns: 添加行数，不包含更新 + + """ + add_count = 0 + + if not datas: + return add_count + + collection = self.get_collection(coll_name) + if not isinstance(update_columns, (tuple, list)): + update_columns = [update_columns] + + try: + add_count = len(datas) + collection.insert_many(datas, ordered=False) + except BulkWriteError as e: + write_errors = e.details.get("writeErrors") + for error in write_errors: + if error.get("code") == 11000: + # 数据重复 + # 获取重复的数据 + data = error.get("op") + + def get_condition(): + # 获取更新条件 + if condition_fields: + condition = { + condition_field: data[condition_field] + for condition_field in condition_fields + } + else: + # 根据重复的值获取更新条件 + condition = self.__get_update_condition( + coll_name, data, error.get("errmsg") + ) + + return condition + + if update_columns: + # 更新指定的列 + if update_columns_value: + # 使用指定的值更新 + doc = { + key: value + for key, value in zip( + update_columns, update_columns_value + ) + } + else: + # 使用数据本身的值更新 + doc = {key: data.get(key) for key in update_columns} + + collection.update_one(get_condition(), {"$set": doc}) + add_count -= 1 + + elif replace: + # 覆盖更新 + collection.replace_one(get_condition(), data) + add_count -= 1 + + else: + # log.error(error) + add_count -= 1 + + return add_count + + def count(self, coll_name, condition: Optional[Dict], limit=0, **kwargs): + """ + 计数 + @param coll_name: 集合名 + @param condition: 查询条件 + @param limit: 限制数量 + @param kwargs: + ---- + command = { + count: , + query: , + limit: , + skip: , + hint: , + readConcern: , + collation: , + comment: + } + https://docs.mongodb.com/manual/reference/command/count/#mongodb-dbcommand-dbcmd.count + @return: 数据数量 + """ + command = {"count": coll_name, "query": condition, "limit": limit, **kwargs} + result = self.run_command(command) + return result["n"] + + def update(self, coll_name, data: Dict, condition: Dict, upsert: bool = False): + """ + 更新 + Args: + coll_name: 集合名 + data: 单条数据 {"xxx":"xxx"} + condition: 更新条件 {"_id": "xxxx"} + upsert: 数据不存在则插入,默认为 False + + Returns: True / False + """ + try: + collection = self.get_collection(coll_name) + collection.update_one(condition, {"$set": data}, upsert=upsert) + except Exception as e: + log.error( + """ + error:{} + condition: {} + """.format( + e, condition + ) + ) + return False + else: + return True + + def update_many(self, coll_name, data: Dict, condition: Dict, upsert: bool = False): + """ + 批量更新 + Args: + coll_name: 集合名 + data: 单条数据 {"xxx":"xxx"} + condition: 更新条件 {"_id": "xxxx"} + upsert: 数据不存在则插入,默认为 False + + Returns: True / False + """ + try: + collection = self.get_collection(coll_name) + collection.update_many(condition, {"$set": data}, upsert=upsert) + except Exception as e: + log.error( + """ + error:{} + condition: {} + """.format( + e, condition + ) + ) + return False + else: + return True + + def update_batch( + self, + coll_name: str, + update_data_list: List[Dict], + condition_field: str, + upsert: bool = False, + ): + """ + 批量更新数据 + Args: + coll_name: 集合名 + update_data_list: 更新数据列表 + condition_field: 更新条件字段 + upsert: 数据不存在则插入，默认为 False + + Returns: 更新行数 + + """ + if not update_data_list: + return 0 + + collection = self.get_collection(coll_name) + bulk_operations = [] + + for update_data in update_data_list: + condition = {condition_field: update_data.get(condition_field)} + update_operation = UpdateMany( + condition, {"$set": update_data}, upsert=upsert + ) + bulk_operations.append(update_operation) + try: + result = collection.bulk_write(bulk_operations, ordered=False) + return result.modified_count + result.upserted_count + except BulkWriteError as e: + log.error(f"Bulk write error: {e.details}") + return 0 + + def delete(self, coll_name, condition: Dict) -> bool: + """ + 删除 + Args: + coll_name: 集合名 + condition: 查找条件 + Returns: True / False + + """ + try: + collection = self.get_collection(coll_name) + collection.delete_one(condition) + except Exception as e: + log.error( + """ + error:{} + condition: {} + """.format( + e, condition + ) + ) + return False + else: + return True + + def run_command(self, command: Dict): + """ + 运行指令 + 参考文档 https://www.geek-book.com/src/docs/mongodb/mongodb/docs.mongodb.com/manual/reference/command/index.html + @param command: + @return: + """ + return self.db.command(command) + + def create_index(self, coll_name, keys, unique=True): + collection = self.get_collection(coll_name) + _keys = [(key, pymongo.ASCENDING) for key in keys] + collection.create_index(_keys, unique=unique) + + def get_index(self, coll_name): + return self.get_collection(coll_name).index_information() + + def drop_collection(self, coll_name): + return self.db.drop_collection(coll_name) + + def get_index_key(self, coll_name, index_name): + """ + 获取参与索引的key + Args: + index_name: 索引名 + + Returns: + + """ + cache_key = f"{coll_name}:{index_name}" + + if cache_key in self.__index__cached: + return self.__index__cached.get(cache_key) + + index = self.get_index(coll_name) + index_detail = index.get(index_name) + if not index_detail: + errmsg = f"not found index {index_name} in collection {coll_name}" + raise Exception(errmsg) + + index_keys = [val[0] for val in index_detail.get("key")] + self.__index__cached[cache_key] = index_keys + return index_keys + + def __get_update_condition( + self, coll_name: str, data: dict, duplicate_errmsg: str + ) -> dict: + """ + 根据索引冲突的报错信息 获取更新条件 + Args: + duplicate_errmsg: E11000 duplicate key error collection: feapder.test index: a_1_b_1 dup key: { : 1, : "你好" } + data: {"a": 1, "b": "你好", "c": "嘻嘻"} + + Returns: {"a": 1, "b": "你好"} + + """ + index_name = re.search(r"index: (\w+)", duplicate_errmsg).group(1) + index_keys = self.get_index_key(coll_name, index_name) + + condition = {key: data.get(key) for key in index_keys} + return condition + + def __getattr__(self, name): + return getattr(self.db, name) + + +if __name__ == "__main__": + update_data_list = [{"_id": "1", "status": 1}, {"_id": "2", "status": 1}] + mongo = MongoDB() + updated_count = mongo.update_batch("your_table_name", update_data_list, "_id") + print(f"Updated {updated_count} documents.") + + id_list = ["1", "2"] + result = mongo.update_many( + "your_table_name", {"status": 1}, {"_id": {"$in": id_list}} + ) diff --git a/feapder/db/mysqldb.py b/feapder/db/mysqldb.py index 498ddeb6..9043bafe 100644 --- a/feapder/db/mysqldb.py +++ b/feapder/db/mysqldb.py @@ -2,14 +2,15 @@ """ Created on 2016-11-16 16:25 --------- -@summary: 操作oracle数据库 +@summary: 操作mysql数据库 --------- @author: Boris -@email: boris@bzkj.tech +@email: boris_liu@foxmail.com """ import datetime import json from urllib import parse +from typing import List, Dict import pymysql from dbutils.pooled_db import PooledDB @@ -40,7 +41,7 @@ def wapper(*args, **kwargs): class MysqlDB: def __init__( - self, ip=None, port=None, db=None, user_name=None, user_pass=None, **kwargs + self, ip=None, port=None, db=None, user_name=None, user_pass=None, charset="utf8mb4", set_session=None, **kwargs ): # 可能会改setting中的值，所以此处不能直接赋值为默认值，需要后加载赋值 if not ip: @@ -55,7 +56,6 @@ def __init__( user_pass = setting.MYSQL_USER_PASS try: - self.connect_pool = PooledDB( creator=pymysql, mincached=1, @@ -68,14 +68,16 @@ def __init__( user=user_name, passwd=user_pass, db=db, - charset="utf8mb4", + charset=charset, + setsession=set_session, cursorclass=cursors.SSCursor, + **kwargs ) # cursorclass 使用服务的游标，默认的在多线程下大批量插入数据会使内存递增 except Exception as e: log.error( """ - 连接数据失败： + 连接失败： ip: {} port: {} db: {} @@ -83,7 +85,7 @@ def __init__( user_pass: {} exception: {} """.format( - ip, port, db, user_name, user_pass, e + ip, port, db, user_name, user_pass, charset, e ) ) else: @@ -91,7 +93,15 @@ def __init__( @classmethod def from_url(cls, url, **kwargs): - # mysql://username:ip:port/db?charset=utf8mb4 + """ + + Args: + url: mysql://username:password@ip:port/db?charset=utf8mb4 + **kwargs: + + Returns: + + """ url_parsed = parse.urlparse(url) db_type = url_parsed.scheme.strip() @@ -102,13 +112,16 @@ def from_url(cls, url, **kwargs): ) ) - connect_params = {} - connect_params["ip"] = url_parsed.hostname.strip() - connect_params["port"] = url_parsed.port - connect_params["user_name"] = url_parsed.username.strip() - connect_params["user_pass"] = url_parsed.password.strip() - connect_params["db"] = url_parsed.path.strip("/").strip() - + connect_params = { + "ip": url_parsed.hostname.strip(), + "port": url_parsed.port, + "user_name": url_parsed.username.strip(), + "user_pass": url_parsed.password.strip(), + "db": url_parsed.path.strip("/").strip(), + } + # 解析 query 字符串参数，比如 ?charset=utf8 + query_params = dict(parse.parse_qsl(url_parsed.query)) + connect_params.update(query_params) connect_params.update(kwargs) return cls(**connect_params) @@ -136,8 +149,10 @@ def get_connection(self): return conn, cursor def close_connection(self, conn, cursor): - cursor.close() - conn.close() + if conn: + conn.close() + if cursor: + cursor.close() def size_of_connections(self): """ @@ -154,7 +169,7 @@ def size_of_connect_pool(self): return len(self.connect_pool._idle_cache) @auto_retry - def find(self, sql, limit=0, to_json=False, cursor=None): + def find(self, sql, limit=0, to_json=False, conver_col=True): """ @summary: 无数据： 返回() @@ -163,6 +178,8 @@ def find(self, sql, limit=0, to_json=False, cursor=None): --------- @param sql: @param limit: + @param to_json 是否将查询结果转为json + @param conver_col 是否处理查询结果，如date类型转字符串，json字符串转成json。仅当to_json=True时生效 --------- @result: """ @@ -177,37 +194,50 @@ def find(self, sql, limit=0, to_json=False, cursor=None): else: result = cursor.fetchall() - if to_json: + if to_json and result: columns = [i[0] for i in cursor.description] # 处理数据 - def fix_lob(row): - def convert(col): - if isinstance(col, (datetime.date, datetime.time)): - return str(col) - elif isinstance(col, str) and ( + def convert(col): + if isinstance(col, (datetime.date, datetime.time)): + return str(col) + elif isinstance(col, str) and ( col.startswith("{") or col.startswith("[") - ): - try: - # col = self.unescape_string(col) - return json.loads(col) - except: - return col - else: + ): + try: # col = self.unescape_string(col) + return json.loads(col) + except: return col - - return [convert(c) for c in row] - - result = [fix_lob(row) for row in result] - result = [dict(zip(columns, r)) for r in result] + else: + # col = self.unescape_string(col) + return col + + if limit == 1: + if conver_col: + result = [convert(col) for col in result] + result = dict(zip(columns, result)) + else: + if conver_col: + result = [[convert(col) for col in row] for row in result] + result = [dict(zip(columns, r)) for r in result] self.close_connection(conn, cursor) return result - def add(self, sql, exception_callfunc=""): + def add(self, sql, exception_callfunc=None): + """ + + Args: + sql: + exception_callfunc: 异常回调 + + Returns: 添加行数 + + """ affect_count = None + conn, cursor = None, None try: conn, cursor = self.get_connection() @@ -229,20 +259,32 @@ def add(self, sql, exception_callfunc=""): return affect_count - def add2(self, table, data, **kwargs): + def add_smart(self, table, data: Dict, **kwargs): + """ + 添加数据, 直接传递json格式的数据，不用拼sql + Args: + table: 表名 + data: 字典 {"xxx":"xxx"} + **kwargs: + + Returns: 添加行数 + + """ sql = make_insert_sql(table, data, **kwargs) return self.add(sql) - def add_batch(self, sql, datas): + def add_batch(self, sql, datas: List[List]): """ - @summary: + @summary: 批量添加数据 --------- - @ param sql: insert ignore into (xxx,xxx) values (%s, %s, %s) - # param datas:[[..], [...]] + @ param sql: insert ignore into (xxx,xxx,xxx) values (%s, %s, %s) + @ param datas: 列表 [[v1,v2,v3], [v1,v2,v3]] + 列表里的值要和插入的key的顺序对应上 --------- - @result: + @result: 添加行数 """ affect_count = None + conn, cursor = None, None try: conn, cursor = self.get_connection() @@ -262,16 +304,28 @@ def add_batch(self, sql, datas): return affect_count - def add_batch2(self, table, datas, **kwargs): + def add_batch_smart(self, table, datas: List[Dict], **kwargs) -> int: + """ + 批量添加数据, 直接传递list格式的数据，不用拼sql + Args: + table: 表名 + datas: 列表 [{}, {}, {}] + **kwargs: + + Returns: 添加行数 + + """ sql, datas = make_batch_sql(table, datas, **kwargs) return self.add_batch(sql, datas) - def update(self, sql): + def update(self, sql) -> int: + affect_count = None + conn, cursor = None, None + try: conn, cursor = self.get_connection() - cursor.execute(sql) + affect_count = cursor.execute(sql) conn.commit() - except Exception as e: log.error( """ @@ -280,22 +334,40 @@ def update(self, sql): """ % (e, sql) ) - return False - else: - return True finally: self.close_connection(conn, cursor) - def update2(self, table, data, condition): + return affect_count + + def update_smart(self, table, data: Dict, condition) -> int: + """ + 更新, 不用拼sql + Args: + table: 表名 + data: 数据 {"xxx":"xxx"} + condition: 更新条件 where后面的条件，如 condition='status=1' + + Returns: 影响行数 + + """ sql = make_update_sql(table, data, condition) return self.update(sql) - def delete(self, sql): + def delete(self, sql) -> int: + """ + 删除 + Args: + sql: + + Returns: 影响行数 + + """ + affect_count = None + conn, cursor = None, None try: conn, cursor = self.get_connection() - cursor.execute(sql) + affect_count = cursor.execute(sql) conn.commit() - except Exception as e: log.error( """ @@ -304,18 +376,25 @@ def delete(self, sql): """ % (e, sql) ) - return False - else: - return True finally: self.close_connection(conn, cursor) - def execute(self, sql): + return affect_count + + def execute(self, sql) -> int: + """ + + Args: + sql: + + Returns: 影响行数 + """ + affect_count = None + conn, cursor = None, None try: conn, cursor = self.get_connection() - cursor.execute(sql) + affect_count = cursor.execute(sql) conn.commit() - except Exception as e: log.error( """ @@ -324,25 +403,7 @@ def execute(self, sql): """ % (e, sql) ) - return False - else: - return True finally: self.close_connection(conn, cursor) - def set_unique_key(self, table, key): - try: - sql = "alter table %s add unique (%s)" % (table, key) - - conn, cursor = self.get_connection() - cursor.execute(sql) - conn.commit() - - except Exception as e: - log.error(table + " " + str(e) + " key = " + key) - return False - else: - log.debug("%s表创建唯一索引成功 索引为 %s" % (table, key)) - return True - finally: - self.close_connection(conn, cursor) + return affect_count diff --git a/feapder/db/redisdb.py b/feapder/db/redisdb.py index 984aef20..ee4e0cd9 100644 --- a/feapder/db/redisdb.py +++ b/feapder/db/redisdb.py @@ -6,38 +6,64 @@ --------- @author: Boris """ +import time +from typing import Union, List import redis +from redis.cluster import ClusterNode, RedisCluster +from redis.connection import Encoder as _Encoder +from redis.exceptions import ConnectionError, TimeoutError +from redis.exceptions import DataError from redis.sentinel import Sentinel -from rediscluster import RedisCluster import feapder.setting as setting from feapder.utils.log import log -# class Singleton(object): -# def __init__(self, cls): -# self._cls = cls -# self._instance = {} -# -# def __call__(self, *args, **kwargs): -# if self._cls not in self._instance: -# self._instance[self._cls] = self._cls(*args, **kwargs) -# return self._instance[self._cls] -# -# -# @Singleton +class Encoder(_Encoder): + def encode(self, value): + "Return a bytestring or bytes-like representation of the value" + if isinstance(value, (bytes, memoryview)): + return value + # elif isinstance(value, bool): + # # special case bool since it is a subclass of int + # raise DataError( + # "Invalid input of type: 'bool'. Convert to a " + # "bytes, string, int or float first." + # ) + elif isinstance(value, float): + value = repr(value).encode() + elif isinstance(value, int): + # python 2 repr() on longs is '123L', so use str() instead + value = str(value).encode() + elif isinstance(value, (list, dict, tuple)): + value = str(value) + elif not isinstance(value, str): + # a value we don't know how to deal with. throw an error + typename = type(value).__name__ + raise DataError( + "Invalid input of type: '%s'. Convert to a " + "bytes, string, int or float first." % typename + ) + if isinstance(value, str): + value = value.encode(self.encoding, self.encoding_errors) + return value + + +redis.connection.Encoder = Encoder + + class RedisDB: def __init__( self, ip_ports=None, - db=0, + db=None, user_pass=None, url=None, decode_responses=True, service_name=None, - max_connections=32, - **kwargs + max_connections=1000, + **kwargs, ): """ redis的封装 @@ -48,6 +74,7 @@ def __init__( url: decode_responses: service_name: 适用于redis哨兵模式 + max_connections: 同一个redis对象使用的并发数（连接池的最大连接数），超过这个数量会抛出redis.ConnectionError """ # 可能会改setting中的值，所以此处不能直接赋值为默认值，需要后加载赋值 @@ -59,92 +86,117 @@ def __init__( user_pass = setting.REDISDB_USER_PASS if service_name is None: service_name = setting.REDISDB_SERVICE_NAME + if kwargs is None: + kwargs = setting.REDISDB_KWARGS self._is_redis_cluster = False + self.__redis = None + self._url = url + self._ip_ports = ip_ports + self._db = db + self._user_pass = user_pass + self._decode_responses = decode_responses + self._service_name = service_name + self._max_connections = max_connections + self._kwargs = kwargs + self.get_connect() + + def __repr__(self): + if self._url: + return "".format(self._url) + + return "".format( + self._ip_ports, self._db, self._user_pass + ) + + @property + def _redis(self): try: - if not url: - if not ip_ports: - raise Exception("未设置redis连接信息") + if not self.__redis.ping(): + raise ConnectionError("unable to connect to redis") + except: + self._reconnect() + + return self.__redis + + @_redis.setter + def _redis(self, val): + self.__redis = val + + def get_connect(self): + # 获取数据库连接 + try: + if not self._url: + if not self._ip_ports: + raise ConnectionError("未设置 redis 连接信息") ip_ports = ( - ip_ports if isinstance(ip_ports, list) else ip_ports.split(",") + self._ip_ports + if isinstance(self._ip_ports, list) + else self._ip_ports.split(",") ) if len(ip_ports) > 1: startup_nodes = [] for ip_port in ip_ports: ip, port = ip_port.split(":") - startup_nodes.append({"host": ip, "port": port}) + startup_nodes.append(ClusterNode(ip, int(port))) - if service_name: + if self._service_name: # log.debug("使用redis哨兵模式") - hosts = [(node["host"], node["port"]) for node in startup_nodes] - sentinel = Sentinel(hosts, socket_timeout=3, **kwargs) + hosts = [(node.host, node.port) for node in startup_nodes] + sentinel = Sentinel(hosts, socket_timeout=3, **self._kwargs) self._redis = sentinel.master_for( - service_name, - password=user_pass, - db=db, - redis_class=redis.StrictRedis, - decode_responses=decode_responses, - max_connections=max_connections, - **kwargs + self._service_name, + password=self._user_pass, + db=self._db, + redis_class=redis.Redis, + decode_responses=self._decode_responses, + max_connections=self._max_connections, + **self._kwargs, ) + self._is_redis_cluster = False else: # log.debug("使用redis集群模式") self._redis = RedisCluster( startup_nodes=startup_nodes, - decode_responses=decode_responses, - password=user_pass, - max_connections=max_connections, - **kwargs + decode_responses=self._decode_responses, + password=self._user_pass, + max_connections=self._max_connections, + **self._kwargs, ) - - self._is_redis_cluster = True + self._is_redis_cluster = True else: ip, port = ip_ports[0].split(":") - self._redis = redis.StrictRedis( + self._redis = redis.Redis( host=ip, port=port, - db=db, - password=user_pass, - decode_responses=decode_responses, - max_connections=max_connections, - **kwargs + db=self._db, + password=self._user_pass, + decode_responses=self._decode_responses, + max_connections=self._max_connections, + **self._kwargs, ) + self._is_redis_cluster = False else: - self._redis = redis.StrictRedis.from_url( - url, decode_responses=decode_responses + self._redis = redis.Redis.from_url( + self._url, decode_responses=self._decode_responses, **self._kwargs ) + self._is_redis_cluster = False except Exception as e: - raise - else: - # if not url: - # log.debug("连接到redis数据库 %s db%s" % (ip_ports, db)) - # else: - # log.debug("连接到redis数据库 %s" % (url)) - pass + raise e - self._ip_ports = ip_ports - self._db = db - self._user_pass = user_pass - self._url = url - - def __repr__(self): - if self._url: - return "".format(self._url) - - return "".format( - self._ip_ports, self._db, self._user_pass - ) + # 不要写成self._redis.ping() 否则循环调用了 + return self.__redis.ping() @classmethod def from_url(cls, url): """ Args: - url: redis://[[username]:[password]]@localhost:6379/0 + url: redis://[[username]:[password]]@[host]:[port]/[db] Returns: @@ -162,9 +214,7 @@ def sadd(self, table, values): """ if isinstance(values, list): - pipe = self._redis.pipeline( - transaction=True - ) # redis-py默认在执行每次请求都会创建（连接池申请连接）和断开（归还连接池）一次连接操作，如果想要在一次请求中指定多个命令，则可以使用pipline实现一次请求指定多个命令，并且默认情况下一次pipline 是原子性操作。 + pipe = self._redis.pipeline() if not self._is_redis_cluster: pipe.multi() @@ -183,14 +233,13 @@ def sget(self, table, count=1, is_pop=True): @param is_pop: @return: """ + datas = [] if is_pop: count = count if count <= self.sget_count(table) else self.sget_count(table) if count: if count > 1: - pipe = self._redis.pipeline( - transaction=True - ) # redis-py默认在执行每次请求都会创建（连接池申请连接）和断开（归还连接池）一次连接操作，如果想要在一次请求中指定多个命令，则可以使用pipline实现一次请求指定多个命令，并且默认情况下一次pipline 是原子性操作。 + pipe = self._redis.pipeline() if not self._is_redis_cluster: pipe.multi() @@ -216,10 +265,9 @@ def srem(self, table, values): --------- @result: """ + if isinstance(values, list): - pipe = self._redis.pipeline( - transaction=True - ) # redis-py默认在执行每次请求都会创建（连接池申请连接）和断开（归还连接池）一次连接操作，如果想要在一次请求中指定多个命令，则可以使用pipline实现一次请求指定多个命令，并且默认情况下一次pipline 是原子性操作。 + pipe = self._redis.pipeline() if not self._is_redis_cluster: pipe.multi() @@ -242,6 +290,7 @@ def sdelete(self, table): --------- @result: """ + # 当 SCAN 命令的游标参数被设置为 0 时， 服务器将开始一次新的迭代， 而当服务器向用户返回值为 0 的游标时， 表示迭代已结束 cursor = "0" while cursor != 0: @@ -272,7 +321,7 @@ def zadd(self, table, values, prioritys=0): else: assert len(values) == len(prioritys), "values值要与prioritys值一一对应" - pipe = self._redis.pipeline(transaction=True) + pipe = self._redis.pipeline() if not self._is_redis_cluster: pipe.multi() @@ -297,12 +346,11 @@ def zget(self, table, count=1, is_pop=True): --------- @result: 列表 """ + start_pos = 0 # 包含 end_pos = count - 1 if count > 0 else count - pipe = self._redis.pipeline( - transaction=True - ) # redis-py默认在执行每次请求都会创建（连接池申请连接）和断开（归还连接池）一次连接操作，如果想要在一次请求中指定多个命令，则可以使用pipline实现一次请求指定多个命令，并且默认情况下一次pipline 是原子性操作。 + pipe = self._redis.pipeline() if not self._is_redis_cluster: pipe.multi() # 标记事务的开始 参考 http://www.runoob.com/redis/redis-transactions.html @@ -337,7 +385,7 @@ def zrangebyscore(self, table, priority_min, priority_max, count=None, is_pop=Tr # 使用lua脚本， 保证操作的原子性 lua = """ - local key = KEYS[1] + -- local key = KEYS[1] local min_score = ARGV[2] local max_score = ARGV[3] local is_pop = ARGV[4] @@ -346,15 +394,15 @@ def zrangebyscore(self, table, priority_min, priority_max, count=None, is_pop=Tr -- 取值 local datas = nil if count then - datas = redis.call('zrangebyscore', key, min_score, max_score, 'limit', 0, count) + datas = redis.call('zrangebyscore', KEYS[1], min_score, max_score, 'limit', 0, count) else - datas = redis.call('zrangebyscore', key, min_score, max_score) + datas = redis.call('zrangebyscore', KEYS[1], min_score, max_score) end -- 删除redis中刚取到的值 - if (is_pop) then + if (is_pop=='True' or is_pop=='1') then for i=1, #datas do - redis.call('zrem', key, datas[i]) + redis.call('zrem', KEYS[1], datas[i]) end end @@ -389,7 +437,7 @@ def zrangebyscore_increase_score( # 使用lua脚本， 保证操作的原子性 lua = """ - local key = KEYS[1] + -- local key = KEYS[1] local min_score = ARGV[1] local max_score = ARGV[2] local increase_score = ARGV[3] @@ -398,14 +446,14 @@ def zrangebyscore_increase_score( -- 取值 local datas = nil if count then - datas = redis.call('zrangebyscore', key, min_score, max_score, 'limit', 0, count) + datas = redis.call('zrangebyscore', KEYS[1], min_score, max_score, 'limit', 0, count) else - datas = redis.call('zrangebyscore', key, min_score, max_score) + datas = redis.call('zrangebyscore', KEYS[1], min_score, max_score) end --修改优先级 for i=1, #datas do - redis.call('zincrby', key, increase_score, datas[i]) + redis.call('zincrby', KEYS[1], increase_score, datas[i]) end return datas @@ -438,7 +486,7 @@ def zrangebyscore_set_score( # 使用lua脚本， 保证操作的原子性 lua = """ - local key = KEYS[1] + -- local key = KEYS[1] local min_score = ARGV[1] local max_score = ARGV[2] local set_score = ARGV[3] @@ -447,9 +495,9 @@ def zrangebyscore_set_score( -- 取值 local datas = nil if count then - datas = redis.call('zrangebyscore', key, min_score, max_score, 'withscores','limit', 0, count) + datas = redis.call('zrangebyscore', KEYS[1], min_score, max_score, 'withscores','limit', 0, count) else - datas = redis.call('zrangebyscore', key, min_score, max_score, 'withscores') + datas = redis.call('zrangebyscore', KEYS[1], min_score, max_score, 'withscores') end local real_datas = {} -- 数据 @@ -460,7 +508,7 @@ def zrangebyscore_set_score( table.insert(real_datas, data) -- 添加数据 - redis.call('zincrby', key, set_score - score, datas[i]) + redis.call('zincrby', KEYS[1], set_score - score, datas[i]) end return real_datas @@ -474,6 +522,9 @@ def zrangebyscore_set_score( return res + def zincrby(self, table, amount, value): + return self._redis.zincrby(table, amount, value) + def zget_count(self, table, priority_min=None, priority_max=None): """ @summary: 获取表数据的数量 @@ -499,16 +550,9 @@ def zrem(self, table, values): --------- @result: """ - if isinstance(values, list): - pipe = self._redis.pipeline( - transaction=True - ) # redis-py默认在执行每次请求都会创建（连接池申请连接）和断开（归还连接池）一次连接操作，如果想要在一次请求中指定多个命令，则可以使用pipline实现一次请求指定多个命令，并且默认情况下一次pipline 是原子性操作。 - if not self._is_redis_cluster: - pipe.multi() - for value in values: - pipe.zrem(table, value) - pipe.execute() + if isinstance(values, list): + self._redis.zrem(table, *values) else: self._redis.zrem(table, values) @@ -518,13 +562,13 @@ def zexists(self, table, values): @param values: @return: """ + is_exists = [] if isinstance(values, list): - pipe = self._redis.pipeline( - transaction=True - ) # redis-py默认在执行每次请求都会创建（连接池申请连接）和断开（归还连接池）一次连接操作，如果想要在一次请求中指定多个命令，则可以使用pipline实现一次请求指定多个命令，并且默认情况下一次pipline 是原子性操作。 - pipe.multi() + pipe = self._redis.pipeline() + if not self._is_redis_cluster: + pipe.multi() for value in values: pipe.zscore(table, value) is_exists_temp = pipe.execute() @@ -542,18 +586,16 @@ def zexists(self, table, values): def lpush(self, table, values): if isinstance(values, list): - pipe = self._redis.pipeline( - transaction=True - ) # redis-py默认在执行每次请求都会创建（连接池申请连接）和断开（归还连接池）一次连接操作，如果想要在一次请求中指定多个命令，则可以使用pipline实现一次请求指定多个命令，并且默认情况下一次pipline 是原子性操作。 + pipe = self._redis.pipeline() if not self._is_redis_cluster: pipe.multi() for value in values: - pipe.rpush(table, value) + pipe.lpush(table, value) pipe.execute() else: - return self._redis.rpush(table, values) + return self._redis.lpush(table, values) def lpop(self, table, count=1): """ @@ -564,15 +606,14 @@ def lpop(self, table, count=1): --------- @result: count>1时返回列表 """ - datas = None - count = count if count <= self.lget_count(table) else self.lget_count(table) + datas = None + lcount = self.lget_count(table) + count = count if count <= lcount else lcount if count: if count > 1: - pipe = self._redis.pipeline( - transaction=True - ) # redis-py默认在执行每次请求都会创建（连接池申请连接）和断开（归还连接池）一次连接操作，如果想要在一次请求中指定多个命令，则可以使用pipline实现一次请求指定多个命令，并且默认情况下一次pipline 是原子性操作。 + pipe = self._redis.pipeline() if not self._is_redis_cluster: pipe.multi() @@ -615,7 +656,6 @@ def lrem(self, table, value, num=0): --------- @result: 删除的条数 """ - return self._redis.lrem(table, num, value) def lrange(self, table, start=0, end=-1): @@ -633,7 +673,6 @@ def hset(self, table, key, value): --------- @result: 1 新插入； 0 覆盖 """ - return self._redis.hset(table, key, value) def hset_batch(self, table, datas): @@ -645,7 +684,7 @@ def hset_batch(self, table, datas): Returns: """ - pipe = self._redis.pipeline(transaction=True) + pipe = self._redis.pipeline() if not self._is_redis_cluster: pipe.multi() @@ -661,13 +700,13 @@ def hget(self, table, key, is_pop=False): return self._redis.hget(table, key) else: lua = """ - local key = KEYS[1] + -- local key = KEYS[1] local field = ARGV[1] -- 取值 - local datas = redis.call('hget', key, field) + local datas = redis.call('hget', KEYS[1], field) -- 删除值 - redis.call('hdel', key, field) + redis.call('hdel', KEYS[1], field) return datas @@ -692,35 +731,50 @@ def hdel(self, table, *keys): --------- @result: """ - self._redis.hdel(table, *keys) def hget_count(self, table): return self._redis.hlen(table) - def setbit(self, table, offsets, values): + def hkeys(self, table): + return self._redis.hkeys(table) + + def hvals(self, key): + return self._redis.hvals(key) + + def setbit( + self, table, offsets: Union[int, List[int]], values: Union[int, List[int]] + ): """ - 设置字符串数组某一位的值， 返回之前的值 - @param table: + 设置字符串数组某一位的值，返回之前的值 + @param table: Redis key @param offsets: 支持列表或单个值 @param values: 支持列表或单个值 @return: list / 单个值 """ if isinstance(offsets, list): - if not isinstance(values, list): - values = [values] * len(offsets) + if isinstance(values, int): + # 使用lua脚本，数据是一起传给redis的，降低了网络开销，但redis会阻塞 + script = """ + local value = table.remove(ARGV, 1) + local offsets = ARGV + local results = {} + for i, offset in ipairs(offsets) do + results[i] = redis.call('SETBIT', KEYS[1], offset, value) + end + return results + """ + return self._redis.eval(script, 1, table, values, *offsets) else: assert len(offsets) == len(values), "offsets值要与values值一一对应" + pipe = self._redis.pipeline() + if not self._is_redis_cluster: + pipe.multi() - pipe = self._redis.pipeline( - transaction=True - ) # redis-py默认在执行每次请求都会创建（连接池申请连接）和断开（归还连接池）一次连接操作，如果想要在一次请求中指定多个命令，则可以使用pipline实现一次请求指定多个命令，并且默认情况下一次pipline 是原子性操作。 - pipe.multi() + for offset, value in zip(offsets, values): + pipe.setbit(table, offset, value) - for offset, value in zip(offsets, values): - pipe.setbit(table, offset, value) - - return pipe.execute() + return pipe.execute() else: return self._redis.setbit(table, offsets, values) @@ -733,10 +787,9 @@ def getbit(self, table, offsets): @return: list / 单个值 """ if isinstance(offsets, list): - pipe = self._redis.pipeline( - transaction=True - ) # redis-py默认在执行每次请求都会创建（连接池申请连接）和断开（归还连接池）一次连接操作，如果想要在一次请求中指定多个命令，则可以使用pipline实现一次请求指定多个命令，并且默认情况下一次pipline 是原子性操作。 - pipe.multi() + pipe = self._redis.pipeline() + if not self._is_redis_cluster: + pipe.multi() for offset in offsets: pipe.getbit(table, offset) @@ -749,6 +802,20 @@ def bitcount(self, table): return self._redis.bitcount(table) def strset(self, table, value, **kwargs): + """ + 设置键值 + Args: + table: + value: + **kwargs: + ex: Union[None, int, timedelta] = ..., 设置键的过期时间为 second 秒 + px: Union[None, int, timedelta] = ..., 设置键的过期时间为 millisecond 毫秒 + nx: bool = ..., 只有键不存在时，才对键进行设置操作 + xx: bool = ..., 只有键已经存在时，才对键进行设置操作 + keepttl: bool = ..., 保留键的过期时间 + Returns: + + """ return self._redis.set(table, value, **kwargs) def str_incrby(self, table, value): @@ -775,9 +842,19 @@ def set_expire(self, key, seconds): --------- @result: """ - self._redis.expire(key, seconds) + def get_expire(self, key): + """ + @summary: 查询过期时间 + --------- + @param key: + @param seconds: 秒 + --------- + @result: + """ + return self._redis.ttl(key) + def clear(self, table): try: self._redis.delete(table) @@ -786,3 +863,93 @@ def clear(self, table): def get_redis_obj(self): return self._redis + + def _reconnect(self): + # 检测连接状态, 当数据库重启或设置 timeout 导致断开连接时自动重连 + retry_count = 0 + while True: + try: + retry_count += 1 + log.error(f"redis 连接断开, 重新连接 {retry_count}") + if self.get_connect(): + log.info(f"redis 连接成功") + return True + except (ConnectionError, TimeoutError) as e: + log.error(f"连接失败 e: {e}") + + time.sleep(2) + + def __getattr__(self, name): + return getattr(self._redis, name) + + def current_status(self, show_key=True, filter_key_by_used_memory=10 * 1024 * 1024): + """ + 统计redis当前使用情况 + Args: + show_key: 是否统计每个key的内存 + filter_key_by_used_memory: 根据内存的使用量过滤key 只显示使用量大于指定内存的key + + Returns: + + """ + from prettytable import PrettyTable + from tqdm import tqdm + + status_msg = "" + + print("正在查询最大连接数...") + clients_count = self._redis.execute_command("info clients") + max_clients_count = self._redis.execute_command("config get maxclients") + status_msg += ": ".join(max_clients_count) + "\n" + status_msg += clients_count + "\n" + + print("正在查询整体内存使用情况...") + total_status = self._redis.execute_command("info memory") + status_msg += total_status + "\n" + + if show_key: + print("正在查询每个key占用内存情况等信息...") + table = PrettyTable( + field_names=[ + "type", + "key", + "value_count", + "used_memory_human", + "used_memory", + ], + sortby="used_memory", + reversesort=True, + header_style="title", + ) + + keys = self._redis.execute_command("keys *") + for key in tqdm(keys): + key_type = self._redis.execute_command("type {}".format(key)) + if key_type == "set": + value_count = self._redis.scard(key) + elif key_type == "zset": + value_count = self._redis.zcard(key) + elif key_type == "list": + value_count = self._redis.llen(key) + elif key_type == "hash": + value_count = self._redis.hlen(key) + elif key_type == "string": + value_count = self._redis.strlen(key) + elif key_type == "none": + continue + else: + raise TypeError("尚不支持 {} 类型的key".format(key_type)) + + used_memory = self._redis.execute_command("memory usage {}".format(key)) + if used_memory >= filter_key_by_used_memory: + used_memory_human = ( + "%0.2fMB" % (used_memory / 1024 / 1024) if used_memory else 0 + ) + + table.add_row( + [key_type, key, value_count, used_memory_human, used_memory] + ) + + status_msg += str(table) + + return status_msg diff --git a/feapder/dedup/__init__.py b/feapder/dedup/__init__.py index e5025b80..6b67ca4a 100644 --- a/feapder/dedup/__init__.py +++ b/feapder/dedup/__init__.py @@ -5,7 +5,7 @@ @summary: --------- @author: Boris -@email: boris@bzkj.tech +@email: boris_liu@foxmail.com """ import copy @@ -14,26 +14,30 @@ from feapder.utils.tools import get_md5 from .bloomfilter import BloomFilter, ScalableBloomFilter from .expirefilter import ExpireFilter +from .litefilter import LiteFilter class Dedup: BloomFilter = 1 MemoryFilter = 2 ExpireFilter = 3 + LiteFilter = 4 def __init__(self, filter_type: int = BloomFilter, to_md5: bool = True, **kwargs): """ - 去重过滤器 集成BloomFilter、MemoryFilter、ExpireFilter + 去重过滤器 集成BloomFilter、MemoryFilter、ExpireFilter、MemoryLiteFilter Args: filter_type: 过滤器类型 BloomFilter name: 过滤器名称 该名称会默认以dedup作为前缀 dedup:expire_set:[name]/dedup:bloomfilter:[name]。 默认ExpireFilter name=过期时间; BloomFilter name=dedup:bloomfilter:bloomfilter - absolute_name: 过滤器绝对名称 不会加dedup前缀 + absolute_name: 过滤器绝对名称 不会加dedup前缀，当此值不为空时name参数无效 expire_time: ExpireFilter的过期时间 单位为秒，其他两种过滤器不用指定 error_rate: BloomFilter/MemoryFilter的误判率 默认为0.00001 to_md5: 去重前是否将数据转为MD5，默认是 redis_url: redis://[[username]:[password]]@localhost:6379/0 BloomFilter 与 ExpireFilter 使用 默认会读取setting中的redis配置，若无setting，则需要专递redis_url + initial_capacity: 单个布隆过滤器去重容量 默认100000000，当布隆过滤器容量满时会扩展下一个布隆过滤器 + error_rate：布隆过滤器的误判率 默认0.00001 **kwargs: """ @@ -55,6 +59,9 @@ def __init__(self, filter_type: int = BloomFilter, to_md5: bool = True, **kwargs redis_url=kwargs.get("redis_url"), ) + elif filter_type == Dedup.LiteFilter: + self.dedup = LiteFilter() + else: initial_capacity = kwargs.get("initial_capacity", 100000000) error_rate = kwargs.get("error_rate", 0.00001) diff --git a/feapder/dedup/basefilter.py b/feapder/dedup/basefilter.py new file mode 100644 index 00000000..f221ba1d --- /dev/null +++ b/feapder/dedup/basefilter.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- +""" +Created on 2022/9/21 11:17 AM +--------- +@summary: +--------- +@author: Boris +@email: boris_liu@foxmail.com +""" +import abc +from typing import List, Union + + +class BaseFilter: + @abc.abstractmethod + def add( + self, keys: Union[List[str], str], *args, **kwargs + ) -> Union[List[bool], bool]: + """ + + Args: + keys: list / 单个值 + *args: + **kwargs: + + Returns: + list / 单个值 (如果数据已存在 返回 0 否则返回 1, 可以理解为是否添加成功) + """ + pass + + @abc.abstractmethod + def get(self, keys: Union[List[str], str]) -> Union[List[bool], bool]: + """ + 检查数据是否存在 + Args: + keys: list / 单个值 + + Returns: + list / 单个值 (如果数据已存在 返回 1 否则返回 0) + """ + pass diff --git a/feapder/dedup/bitarray.py b/feapder/dedup/bitarray.py index b24c444d..348ceb46 100644 --- a/feapder/dedup/bitarray.py +++ b/feapder/dedup/bitarray.py @@ -5,12 +5,11 @@ @summary: --------- @author: Boris -@email: boris@bzkj.tech +@email: boris_liu@foxmail.com """ from __future__ import absolute_import -import bitarray from feapder.db.redisdb import RedisDB @@ -45,6 +44,13 @@ def count(self, value=True): class MemoryBitArray(BitArray): def __init__(self, num_bits): + try: + import bitarray + except Exception as e: + raise Exception( + '需要安装feapder完整版\ncommand: pip install "feapder[all]"\n若安装出错，参考：https://feapder.com/#/question/%E5%AE%89%E8%A3%85%E9%97%AE%E9%A2%98' + ) + self.num_bits = num_bits self.bitarray = bitarray.bitarray(num_bits, endian="little") @@ -121,7 +127,18 @@ def set(self, offsets, values): @param values: 支持列表或单个值 @return: list / 单个值 """ - return self.redis_db.setbit(self.name, offsets, values) + # 对offsets进行分片，最大100000个 + results = [] + batch_size = 170000 + for i in range(0, len(offsets), batch_size): + results.extend( + self.redis_db.setbit( + self.name, + offsets[i : i + batch_size], + values[i : i + batch_size] if isinstance(values, list) else values, + ) + ) + return results def get(self, offsets): return self.redis_db.getbit(self.name, offsets) @@ -132,6 +149,6 @@ def count(self, value=True): if count: return int(count) else: - count = self.redis_db.bitcount(self.name) + count = self.redis_db.bitcount(self.name) # 被设置为 1 的比特位的数量 self.redis_db.strset(self.count_cached_name, count, ex=1800) # 半小时过期 return count diff --git a/feapder/dedup/bloomfilter.py b/feapder/dedup/bloomfilter.py index 9f5c02b9..0e1af813 100644 --- a/feapder/dedup/bloomfilter.py +++ b/feapder/dedup/bloomfilter.py @@ -5,7 +5,7 @@ @summary: --------- @author: Boris -@email: boris@bzkj.tech +@email: boris_liu@foxmail.com """ import hashlib @@ -14,7 +14,7 @@ import time from struct import unpack, pack -from feapder.db.redisdb import RedisDB +from feapder.dedup.basefilter import BaseFilter from feapder.utils.redis_lock import RedisLock from . import bitarray @@ -146,24 +146,18 @@ def is_at_capacity(self): 比较耗时 半小时检查一次 @return: """ - # if self._is_at_capacity: - # return self._is_at_capacity - # - # if not self._check_capacity_time or time.time() - self._check_capacity_time > 1800: - # bit_count = self.bitarray.count() - # if bit_count and bit_count / self.num_bits > 0.5: - # self._is_at_capacity = True - # - # self._check_capacity_time = time.time() - # - # return self._is_at_capacity - if self._is_at_capacity: return self._is_at_capacity - bit_count = self.bitarray.count() - if bit_count and bit_count / self.num_bits > 0.5: - self._is_at_capacity = True + if ( + not self._check_capacity_time + or time.time() - self._check_capacity_time > 1800 + ): + bit_count = self.bitarray.count() + if bit_count and bit_count / self.num_bits > 0.5: + self._is_at_capacity = True + + self._check_capacity_time = time.time() return self._is_at_capacity @@ -174,8 +168,8 @@ def add(self, keys): @param keys: list or one key @return: """ - if self.is_at_capacity: - raise IndexError("BloomFilter is at capacity") + # if self.is_at_capacity: + # raise IndexError("BloomFilter is at capacity") is_list = isinstance(keys, list) @@ -197,7 +191,7 @@ def add(self, keys): return is_added if is_list else is_added[0] -class ScalableBloomFilter(object): +class ScalableBloomFilter(BaseFilter): """ 自动扩展空间的bloomfilter, 当一个filter满一半的时候，创建下一个 """ @@ -267,12 +261,13 @@ def check_filter_capacity(self): self._check_capacity_time = time.time() else: - with RedisLock( - key="ScalableBloomFilter", - timeout=300, - wait_timeout=300, - redis_cli=RedisDB(url=self.redis_url).get_redis_obj(), - ) as lock: # 全局锁 同一时间只有一个进程在真正的创建新的filter，等这个进程创建完，其他进程只是把刚创建的filter append进来 + # 全局锁 同一时间只有一个进程在真正的创建新的filter，等这个进程创建完，其他进程只是把刚创建的filter append进来 + key = ( + f"ScalableBloomFilter:{self.name}" + if self.name + else "ScalableBloomFilter" + ) + with RedisLock(key=key, redis_url=self.redis_url) as lock: if lock.locked: while True: if self.filters[-1].is_at_capacity: diff --git a/feapder/dedup/expirefilter.py b/feapder/dedup/expirefilter.py index 2b7b7aef..12a4b12d 100644 --- a/feapder/dedup/expirefilter.py +++ b/feapder/dedup/expirefilter.py @@ -5,15 +5,16 @@ @summary: 带有有效期的去重集合 --------- @author: Boris -@email: boris@bzkj.tech +@email: boris_liu@foxmail.com """ import time from feapder.db.redisdb import RedisDB +from feapder.dedup.basefilter import BaseFilter -class ExpireFilter: +class ExpireFilter(BaseFilter): redis_db = None def __init__( @@ -30,6 +31,7 @@ def __init__( self.name = name self.expire_time = expire_time self.expire_time_record_key = expire_time_record_key + self.del_expire_key_time = None self.record_expire_time() @@ -47,16 +49,30 @@ def add(self, keys, *args, **kwargs): @param keys: 检查关键词在zset中是否存在，支持列表批量 @return: list / 单个值 """ + if self.current_timestamp - self.del_expire_key_time > self.expire_time: + self.del_expire_key() + is_added = self.redis_db.zadd(self.name, keys, self.current_timestamp) return is_added def get(self, keys): - return self.redis_db.zexists(self.name, keys) + is_exist = self.redis_db.zexists(self.name, keys) + if isinstance(keys, list): + # 判断数据本身是否重复 + temp_set = set() + for i, key in enumerate(keys): + if key in temp_set: + is_exist[i] = 1 + else: + temp_set.add(key) + + return is_exist def del_expire_key(self): self.redis_db.zremrangebyscore( self.name, "-inf", self.current_timestamp - self.expire_time ) + self.del_expire_key_time = self.current_timestamp def record_expire_time(self): if self.expire_time_record_key: diff --git a/feapder/dedup/litefilter.py b/feapder/dedup/litefilter.py new file mode 100644 index 00000000..da664190 --- /dev/null +++ b/feapder/dedup/litefilter.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- +""" +Created on 2022/9/21 11:28 AM +--------- +@summary: +--------- +@author: Boris +@email: boris_liu@foxmail.com +""" +from typing import List, Union, Set + +from feapder.dedup.basefilter import BaseFilter + + +class LiteFilter(BaseFilter): + def __init__(self): + self.datas: Set[str] = set() + + def add( + self, keys: Union[List[str], str], *args, **kwargs + ) -> Union[List[int], int]: + """ + + Args: + keys: list / 单个值 + *args: + **kwargs: + + Returns: + list / 单个值 (如果数据已存在 返回 0 否则返回 1, 可以理解为是否添加成功) + """ + if isinstance(keys, list): + is_add = [] + for key in keys: + if key not in self.datas: + self.datas.add(key) + is_add.append(1) + else: + is_add.append(0) + else: + if keys not in self.datas: + is_add = 1 + self.datas.add(keys) + else: + is_add = 0 + return is_add + + def get(self, keys: Union[List[str], str]) -> Union[List[int], int]: + """ + 检查数据是否存在 + Args: + keys: list / 单个值 + + Returns: + list / 单个值 (如果数据已存在 返回 1 否则返回 0) + """ + if isinstance(keys, list): + temp_set = set() + is_exist = [] + for key in keys: + # 数据本身重复或者数据在去重库里 + if key in temp_set or key in self.datas: + is_exist.append(1) + else: + is_exist.append(0) + temp_set.add(key) + + return is_exist + else: + return int(keys in self.datas) diff --git a/feapder/network/downloader/__init__.py b/feapder/network/downloader/__init__.py new file mode 100644 index 00000000..f036271e --- /dev/null +++ b/feapder/network/downloader/__init__.py @@ -0,0 +1,12 @@ +from ._requests import RequestsDownloader +from ._requests import RequestsSessionDownloader + +# 下面是非必要依赖 +try: + from ._selenium import SeleniumDownloader +except ModuleNotFoundError: + pass +try: + from ._playwright import PlaywrightDownloader +except ModuleNotFoundError: + pass diff --git a/feapder/network/downloader/_playwright.py b/feapder/network/downloader/_playwright.py new file mode 100644 index 00000000..facc75cd --- /dev/null +++ b/feapder/network/downloader/_playwright.py @@ -0,0 +1,105 @@ +# -*- coding: utf-8 -*- +""" +Created on 2022/9/7 4:05 PM +--------- +@summary: +--------- +@author: Boris +@email: boris_liu@foxmail.com +""" + +import feapder.setting as setting +import feapder.utils.tools as tools +from feapder.network.downloader.base import RenderDownloader +from feapder.network.response import Response +from feapder.utils.webdriver import WebDriverPool, PlaywrightDriver + + +class PlaywrightDownloader(RenderDownloader): + webdriver_pool: WebDriverPool = None + + @property + def _webdriver_pool(self): + if not self.__class__.webdriver_pool: + self.__class__.webdriver_pool = WebDriverPool( + **setting.PLAYWRIGHT, driver_cls=PlaywrightDriver, thread_safe=True + ) + + return self.__class__.webdriver_pool + + def download(self, request) -> Response: + # 代理优先级 自定义 > 配置文件 > 随机 + if request.custom_proxies: + proxy = request.get_proxy() + elif setting.PLAYWRIGHT.get("proxy"): + proxy = setting.PLAYWRIGHT.get("proxy") + else: + proxy = request.get_proxy() + + # user_agent优先级 自定义 > 配置文件 > 随机 + if request.custom_ua: + user_agent = request.get_user_agent() + elif setting.PLAYWRIGHT.get("user_agent"): + user_agent = setting.PLAYWRIGHT.get("user_agent") + else: + user_agent = request.get_user_agent() + + cookies = request.get_cookies() + url = request.url + render_time = request.render_time or setting.PLAYWRIGHT.get("render_time") + wait_until = setting.PLAYWRIGHT.get("wait_until") or "domcontentloaded" + if request.get_params(): + url = tools.joint_url(url, request.get_params()) + + driver: PlaywrightDriver = self._webdriver_pool.get( + user_agent=user_agent, proxy=proxy + ) + try: + if cookies: + driver.url = url + driver.cookies = cookies + http_response = driver.page.goto(url, wait_until=wait_until) + status_code = http_response.status + + if render_time: + tools.delay_time(render_time) + + html = driver.page.content() + response = Response.from_dict( + { + "url": driver.page.url, + "cookies": driver.cookies, + "_content": html.encode(), + "status_code": status_code, + "elapsed": 666, + "headers": { + "User-Agent": driver.user_agent, + "Cookie": tools.cookies2str(driver.cookies), + }, + } + ) + + response.driver = driver + response.browser = driver + return response + except Exception as e: + self._webdriver_pool.remove(driver) + raise e + + def close(self, driver): + if driver: + self._webdriver_pool.remove(driver) + + def put_back(self, driver): + """ + 释放浏览器对象 + """ + self._webdriver_pool.put(driver) + + def close_all(self): + """ + 关闭所有浏览器 + """ + # 不支持 + # self._webdriver_pool.close() + pass diff --git a/feapder/network/downloader/_requests.py b/feapder/network/downloader/_requests.py new file mode 100644 index 00000000..15342f93 --- /dev/null +++ b/feapder/network/downloader/_requests.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- +""" +Created on 2022/4/10 5:57 下午 +--------- +@summary: +--------- +@author: Boris +@email: boris_liu@foxmail.com +""" + +import requests +from requests.adapters import HTTPAdapter + +from feapder.network.downloader.base import Downloader +from feapder.network.response import Response + + +class RequestsDownloader(Downloader): + def download(self, request) -> Response: + response = requests.request( + request.method, request.url, **request.requests_kwargs + ) + response = Response(response) + return response + + +class RequestsSessionDownloader(Downloader): + session = None + + @property + def _session(self): + if not self.__class__.session: + self.__class__.session = requests.Session() + # pool_connections – 缓存的 urllib3 连接池个数 pool_maxsize – 连接池中保存的最大连接数 + http_adapter = HTTPAdapter(pool_connections=1000, pool_maxsize=1000) + # 任何使用该session会话的 HTTP 请求，只要其 URL 是以给定的前缀开头，该传输适配器就会被使用到。 + self.__class__.session.mount("http", http_adapter) + + return self.__class__.session + + def download(self, request) -> Response: + response = self._session.request( + request.method, request.url, **request.requests_kwargs + ) + response = Response(response) + return response diff --git a/feapder/network/downloader/_selenium.py b/feapder/network/downloader/_selenium.py new file mode 100644 index 00000000..682158da --- /dev/null +++ b/feapder/network/downloader/_selenium.py @@ -0,0 +1,102 @@ +# -*- coding: utf-8 -*- +""" +Created on 2022/7/26 4:28 下午 +--------- +@summary: +--------- +@author: Boris +@email: boris_liu@foxmail.com +""" + +import feapder.setting as setting +import feapder.utils.tools as tools +from feapder.network.downloader.base import RenderDownloader +from feapder.network.response import Response +from feapder.utils.webdriver import WebDriverPool, SeleniumDriver + + +class SeleniumDownloader(RenderDownloader): + webdriver_pool: WebDriverPool = None + + @property + def _webdriver_pool(self): + if not self.__class__.webdriver_pool: + self.__class__.webdriver_pool = WebDriverPool( + **setting.WEBDRIVER, driver=SeleniumDriver + ) + + return self.__class__.webdriver_pool + + def download(self, request) -> Response: + # 代理优先级 自定义 > 配置文件 > 随机 + if request.custom_proxies: + proxy = request.get_proxy() + elif setting.WEBDRIVER.get("proxy"): + proxy = setting.WEBDRIVER.get("proxy") + else: + proxy = request.get_proxy() + + # user_agent优先级 自定义 > 配置文件 > 随机 + if request.custom_ua: + user_agent = request.get_user_agent() + elif setting.WEBDRIVER.get("user_agent"): + user_agent = setting.WEBDRIVER.get("user_agent") + else: + user_agent = request.get_user_agent() + + cookies = request.get_cookies() + url = request.url + render_time = request.render_time or setting.WEBDRIVER.get("render_time") + if request.get_params(): + url = tools.joint_url(url, request.get_params()) + + browser: SeleniumDriver = self._webdriver_pool.get( + user_agent=user_agent, proxy=proxy + ) + try: + browser.get(url) + if cookies: + browser.cookies = cookies + # 刷新使cookie生效 + browser.get(url) + + if render_time: + tools.delay_time(render_time) + + html = browser.page_source + response = Response.from_dict( + { + "url": browser.current_url, + "cookies": browser.cookies, + "_content": html.encode(), + "status_code": 200, + "elapsed": 666, + "headers": { + "User-Agent": browser.user_agent, + "Cookie": tools.cookies2str(browser.cookies), + }, + } + ) + + response.driver = browser + response.browser = browser + return response + except Exception as e: + self._webdriver_pool.remove(browser) + raise e + + def close(self, driver): + if driver: + self._webdriver_pool.remove(driver) + + def put_back(self, driver): + """ + 释放浏览器对象 + """ + self._webdriver_pool.put(driver) + + def close_all(self): + """ + 关闭所有浏览器 + """ + self._webdriver_pool.close() diff --git a/feapder/network/downloader/base.py b/feapder/network/downloader/base.py new file mode 100644 index 00000000..ff0fc3b4 --- /dev/null +++ b/feapder/network/downloader/base.py @@ -0,0 +1,41 @@ +import abc +from abc import ABC + +from feapder.network.response import Response + + +class Downloader: + @abc.abstractmethod + def download(self, request) -> Response: + """ + + Args: + request: feapder.Request + + Returns: feapder.Response + + """ + raise NotImplementedError + + def close(self, response: Response): + pass + + +class RenderDownloader(Downloader, ABC): + def put_back(self, driver): + """ + 释放浏览器对象 + """ + pass + + def close(self, driver): + """ + 关闭浏览器 + """ + pass + + def close_all(self): + """ + 关闭所有浏览器 + """ + pass diff --git a/feapder/network/item.py b/feapder/network/item.py index ee7ca8cc..33eae79c 100644 --- a/feapder/network/item.py +++ b/feapder/network/item.py @@ -8,6 +8,9 @@ @email: boris_liu@foxmail.com """ +import re +from typing import List + import feapder.utils.tools as tools @@ -18,12 +21,14 @@ def __new__(cls, name, bases, attrs): attrs.setdefault("__name_underline__", None) attrs.setdefault("__update_key__", None) attrs.setdefault("__unique_key__", None) + attrs.setdefault("__pipelines__", None) return type.__new__(cls, name, bases, attrs) class Item(metaclass=ItemMetaclass): - __unique_key__ = [] + __unique_key__: List = [] + __pipelines__: List = None def __init__(self, **kwargs): self.__dict__ = kwargs @@ -37,21 +42,23 @@ def __getitem__(self, key): def __setitem__(self, key, value): self.__dict__[key] = value - def per_to_db(self): + def update(self, *args, **kwargs): """ - @summary: 入库前的处理 - --------- - --------- - @result: + 更新字段，与字典使用方法一致 """ - pass + self.__dict__.update(*args, **kwargs) + + def update_strict(self, *args, **kwargs): + """ + 更新严格更新，只更新item中有的字段 + """ + for key, value in dict(*args, **kwargs).items(): + if key in self.__dict__: + self.__dict__[key] = value - def after_to_db(self): + def pre_to_db(self): """ - @summary: 入库后的处理 (弃用) - --------- - --------- - @result: + 入库前的处理 """ pass @@ -60,10 +67,12 @@ def to_dict(self): propertys = {} for key, value in self.__dict__.items(): if key not in ( - "__name__", - "__table_name__", - "__name_underline__", - "__update_key__", + "__name__", + "__table_name__", + "__name_underline__", + "__update_key__", + "__unique_key__", + "__pipelines__", ): if key.startswith(f"_{self.__class__.__name__}"): key = key.replace(f"_{self.__class__.__name__}", "") @@ -83,12 +92,12 @@ def item_name(self): @item_name.setter def item_name(self, name): self.__name__ = name - self.__table_name__ = self.name_underline.replace("_item", "") + self.__table_name__ = re.sub("_item$", "", self.name_underline) @property def table_name(self): if not self.__table_name__: - self.__table_name__ = self.name_underline.replace("_item", "") + self.__table_name__ = re.sub("_item$", "", self.name_underline) return self.__table_name__ @table_name.setter @@ -118,13 +127,24 @@ def unique_key(self, keys): else: self.__unique_key__ = (keys,) + @property + def pipelines(self): + return self.__pipelines__ or self.__class__.__pipelines__ + + @pipelines.setter + def pipelines(self, pipelines): + if isinstance(pipelines, (tuple, list)): + self.__pipelines__ = pipelines + else: + self.__pipelines__ = (pipelines,) + @property def fingerprint(self): args = [] for key, value in self.to_dict.items(): if value: if (self.unique_key and key in self.unique_key) or not self.unique_key: - args.append(str(value)) + args.append(key + str(value)) if args: args = sorted(args) diff --git a/feapder/network/proxy_pool/__init__.py b/feapder/network/proxy_pool/__init__.py new file mode 100644 index 00000000..0a6935b6 --- /dev/null +++ b/feapder/network/proxy_pool/__init__.py @@ -0,0 +1,11 @@ +# -*- coding: utf-8 -*- +""" +Created on 2023/7/25 10:16 +--------- +@summary: +--------- +@author: Boris +@email: boris_liu@foxmail.com +""" +from .base import BaseProxyPool +from .proxy_pool import ProxyPool diff --git a/feapder/network/proxy_pool/base.py b/feapder/network/proxy_pool/base.py new file mode 100644 index 00000000..0a2dc590 --- /dev/null +++ b/feapder/network/proxy_pool/base.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- +""" +Created on 2023/7/25 10:03 +--------- +@summary: +--------- +@author: Boris +@email: boris_liu@foxmail.com +""" + +import abc + +from feapder.utils.log import log + + +class BaseProxyPool: + @abc.abstractmethod + def get_proxy(self): + """ + 获取代理 + Returns: + {"http": "xxx", "https": "xxx"} + """ + raise NotImplementedError + + @abc.abstractmethod + def del_proxy(self, proxy): + """ + @summary: 删除代理 + --------- + @param proxy: ip:port + """ + raise NotImplementedError + + def tag_proxy(self, **kwargs): + """ + @summary: 标记代理 + --------- + @param kwargs: + @return: + """ + log.warning("暂不支持标记代理") + pass diff --git a/feapder/network/proxy_pool/proxy_pool.py b/feapder/network/proxy_pool/proxy_pool.py new file mode 100644 index 00000000..ce492633 --- /dev/null +++ b/feapder/network/proxy_pool/proxy_pool.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- +""" +Created on 2022/10/19 10:40 AM +--------- +@summary: +--------- +@author: Boris +@email: boris_liu@foxmail.com +""" +from queue import Queue + +import requests + +import feapder.setting as setting +from feapder.network.proxy_pool.base import BaseProxyPool +from feapder.utils import metrics +from feapder.utils import tools + + +class ProxyPool(BaseProxyPool): + """ + 通过API提取代理，存储在内存中，无代理时会自动提取 + API返回的代理以 \r\n 分隔 + """ + + def __init__(self, proxy_api=None, **kwargs): + self.proxy_api = proxy_api or setting.PROXY_EXTRACT_API + self.proxy_queue = Queue() + + def format_proxy(self, proxy): + return {"http": "http://" + proxy, "https": "http://" + proxy} + + @tools.retry(3, interval=5) + def pull_proxies(self): + resp = requests.get(self.proxy_api) + proxies = resp.text.strip() + resp.close() + if "{" in proxies or not proxies: + raise Exception("获取代理失败", proxies) + # 使用 /r/n 分隔 + return proxies.split("\r\n") + + def get_proxy(self): + try: + if self.proxy_queue.empty(): + proxies = self.pull_proxies() + for proxy in proxies: + self.proxy_queue.put_nowait(proxy) + metrics.emit_counter("total", 1, classify="proxy") + + proxy = self.proxy_queue.get_nowait() + self.proxy_queue.put_nowait(proxy) + + metrics.emit_counter("used_times", 1, classify="proxy") + + return self.format_proxy(proxy) + except Exception as e: + tools.send_msg("获取代理失败", level="error") + raise Exception("获取代理失败", e) + + def del_proxy(self, proxy): + """ + @summary: 删除代理 + --------- + @param proxy: ip:port + """ + if proxy in self.proxy_queue.queue: + self.proxy_queue.queue.remove(proxy) + metrics.emit_counter("invalid", 1, classify="proxy") diff --git a/feapder/network/proxy_pool.py b/feapder/network/proxy_pool_old.py similarity index 94% rename from feapder/network/proxy_pool.py rename to feapder/network/proxy_pool_old.py index ad7af706..fddaabc1 100644 --- a/feapder/network/proxy_pool.py +++ b/feapder/network/proxy_pool_old.py @@ -14,13 +14,13 @@ import requests from feapder import setting -from feapder.utils import log from feapder.utils import tools +from feapder.utils.log import log # 建立本地缓存代理文件夹 proxy_path = os.path.join(os.path.dirname(__file__), "proxy_file") if not os.path.exists(proxy_path): - os.mkdir(proxy_path) + os.makedirs(proxy_path, exist_ok=True) def get_proxies_by_host(host, port): @@ -31,18 +31,11 @@ def get_proxies_by_host(host, port): def get_proxies_by_id(proxy_id): proxies = { "http": "http://{}".format(proxy_id), - "https": "https://{}".format(proxy_id), + "https": "http://{}".format(proxy_id), } return proxies -# 代理类型定义 -class LimitProxy(object): - """提取次数有限制的代理""" - - pass - - def get_proxy_from_url(**kwargs): """ 获取指定url的代理 @@ -133,7 +126,7 @@ def get_proxy_from_file(filename, **kwargs): ip = "{}@{}".format(auth, ip) if not protocol: proxies = { - "https": "https://%s:%s" % (ip, port), + "https": "http://%s:%s" % (ip, port), "http": "http://%s:%s" % (ip, port), } else: @@ -151,10 +144,10 @@ def get_proxy_from_redis(proxy_source_url, **kwargs): ip:port ts @param kwargs: {"redis_proxies_key": "xxx"} - @return: [{'http':'http://xxx.xxx.xxx:xxx', 'https':'https://xxx.xxx.xxx.xxx:xxx'}] + @return: [{'http':'http://xxx.xxx.xxx:xxx', 'https':'http://xxx.xxx.xxx.xxx:xxx'}] """ - redis_conn = redis.StrictRedis.from_url(proxy_source_url) + redis_conn = redis.Redis.from_url(proxy_source_url) key = kwargs.get("redis_proxies_key") assert key, "从redis中获取代理 需要指定 redis_proxies_key" proxies = redis_conn.zrange(key, 0, -1) @@ -162,7 +155,7 @@ def get_proxy_from_redis(proxy_source_url, **kwargs): for proxy in proxies: proxy = proxy.decode() proxies_list.append( - {"https": "https://%s" % proxy, "http": "http://%s" % proxy} + {"https": "http://%s" % proxy, "http": "http://%s" % proxy} ) return proxies_list @@ -174,7 +167,7 @@ def check_proxy( type=0, timeout=5, logger=None, - show_error_log=False, + show_error_log=True, **kwargs, ): """ @@ -187,7 +180,7 @@ def check_proxy( :return: """ if not logger: - logger = log.get_logger(__file__) + logger = log ok = 0 if type == 0 and ip and port: # socket检测成功 不代表代理一定可用 Connection closed by foreign host. 这种情况就不行 @@ -205,23 +198,11 @@ def check_proxy( if not proxies: proxies = { "http": "http://{}:{}".format(ip, port), - "https": "https://{}:{}".format(ip, port), + "https": "http://{}:{}".format(ip, port), } - target_url = random.choice( - [ - "http://www.baidu.com", - # "http://httpbin.org/ip", - ] - ) try: r = requests.get( - target_url, - headers={ - "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36" - }, - proxies=proxies, - timeout=timeout, - stream=True, + "http://www.baidu.com", proxies=proxies, timeout=timeout, stream=True ) ok = 1 r.close() @@ -247,7 +228,6 @@ def __init__( max_proxy_use_num=10000, delay=30, use_interval=None, - logger=None, **kwargs, ): """ @@ -294,7 +274,7 @@ def __init__( self.proxy_id = self.proxy_ip_port # 日志处理器 - self.logger = logger or log.get_logger(__file__) + self.logger = log def get_proxies(self): self.use_num += 1 @@ -404,6 +384,9 @@ def __init__(self, **kwargs): :param logger: 日志处理器 默认 log.get_logger() :param kwargs: 其他的参数 """ + kwargs.setdefault("size", -1) + kwargs.setdefault("proxy_source_url", setting.PROXY_EXTRACT_API) + super(ProxyPool, self).__init__(**kwargs) # 队列最大长度 self.max_queue_size = kwargs.get("size", -1) @@ -418,7 +401,7 @@ def __init__(self, **kwargs): self.proxy_source_url = list(set(self.proxy_source_url)) kwargs.update({"proxy_source_url": self.proxy_source_url}) # 处理日志 - self.logger = kwargs.get("logger") or log.get_logger(__file__) + self.logger = kwargs.get("logger") or log kwargs["logger"] = self.logger if not self.proxy_source_url: self.logger.warn("need set proxy_source_url or proxy_instance") @@ -436,7 +419,7 @@ def __init__(self, **kwargs): self.proxy_dict = {} # 失效代理队列 self.invalid_proxy_dict = {} - # + self.kwargs = kwargs # 重置代理池锁 @@ -720,9 +703,3 @@ def all(self) -> list: :return: """ return get_proxy_from_url(**self.kwargs) - - -if not setting.PROXY_ENABLE or not setting.PROXY_EXTRACT_API: - proxy_pool = None -else: - proxy_pool = ProxyPool(size=-1, proxy_source_url=setting.PROXY_EXTRACT_API) diff --git a/feapder/network/request.py b/feapder/network/request.py index 81675fa0..95e51604 100644 --- a/feapder/network/request.py +++ b/feapder/network/request.py @@ -8,16 +8,20 @@ @email: boris_liu@foxmail.com """ +import copy +import os +import re + import requests -from requests.adapters import HTTPAdapter +from requests.cookies import RequestsCookieJar from requests.packages.urllib3.exceptions import InsecureRequestWarning import feapder.setting as setting import feapder.utils.tools as tools from feapder.db.redisdb import RedisDB from feapder.network import user_agent -from feapder.network.item import Item -from feapder.network.proxy_pool import proxy_pool +from feapder.network.downloader.base import Downloader, RenderDownloader +from feapder.network.proxy_pool import BaseProxyPool from feapder.network.response import Response from feapder.utils.log import log @@ -25,22 +29,22 @@ requests.packages.urllib3.disable_warnings(InsecureRequestWarning) -class Request(object): - session = None +class Request: user_agent_pool = user_agent - proxies_pool = proxy_pool + proxies_pool: BaseProxyPool = None cache_db = None # redis / pika - cached_redis_key = ( - None - ) # 缓存response的文件文件夹 response_cached:cached_redis_key:md5 + cached_redis_key = None # 缓存response的文件文件夹 response_cached:cached_redis_key:md5 cached_expire_time = 1200 # 缓存过期时间 - local_filepath = None - oss_handler = None + # 下载器 + downloader: Downloader = None + session_downloader: Downloader = None + render_downloader: RenderDownloader = None - __REQUEST_ATTRS__ = [ - # 'method', 'url', 必须传递 不加入**kwargs中 + __REQUEST_ATTRS__ = { + # "method", + # "url", "params", "data", "headers", @@ -55,10 +59,11 @@ class Request(object): "verify", "cert", "json", - ] + } - DEFAULT_KEY_VALUE = dict( + _DEFAULT_KEY_VALUE_ = dict( url="", + method=None, retry_times=0, priority=300, parser_name=None, @@ -70,8 +75,17 @@ class Request(object): random_user_agent=True, download_midware=None, is_abandoned=False, + render=False, + render_time=0, + make_absolute_links=None, ) + _CUSTOM_PROPERTIES_ = { + "requests_kwargs", + "custom_ua", + "custom_proxies", + } + def __init__( self, url="", @@ -86,6 +100,9 @@ def __init__( random_user_agent=True, download_midware=None, is_abandoned=False, + render=False, + render_time=0, + make_absolute_links=None, **kwargs, ): """ @@ -104,8 +121,11 @@ def __init__( @param random_user_agent: 是否随机User-Agent (True/False) 当setting中的RANDOM_HEADERS设置为True时该参数生效 默认True @param download_midware: 下载中间件。默认为parser中的download_midware @param is_abandoned: 当发生异常时是否放弃重试 True/False. 默认False + @param render: 是否用浏览器渲染 + @param render_time: 渲染时长，即打开网页等待指定时间后再获取源码 + @param make_absolute_links: 是否转成绝对连接，默认是 -- - 以下参数于requests参数使用方式一致 + 以下参数与requests参数使用方式一致 @param method: 请求方式，如POST或GET，默认根据data值是否为空来判断 @param params: 请求参数 @param data: 请求body @@ -121,12 +141,13 @@ def __init__( @param stream: 如果为 False，将会立即下载响应内容 @param cert: -- - @param **kwargs: 其他值: 如 Request(item=item) 则item可直接用 reqeust.item 取出 + @param **kwargs: 其他值: 如 Request(item=item) 则item可直接用 request.item 取出 --------- @result: """ self.url = url + self.method = None self.retry_times = retry_times self.priority = priority self.parser_name = parser_name @@ -138,7 +159,15 @@ def __init__( self.random_user_agent = random_user_agent self.download_midware = download_midware self.is_abandoned = is_abandoned + self.render = render + self.render_time = render_time + self.make_absolute_links = ( + make_absolute_links + if make_absolute_links is not None + else setting.MAKE_ABSOLUTE_LINKS + ) + # 自定义属性，不参与序列化 self.requests_kwargs = {} for key, value in kwargs.items(): if key in self.__class__.__REQUEST_ATTRS__: # 取requests参数 @@ -146,6 +175,9 @@ def __init__( self.__dict__[key] = value + self.custom_ua = False + self.custom_proxies = False + def __repr__(self): try: return "".format(self.url) @@ -164,24 +196,50 @@ def __setattr__(self, key, value): if key in self.__class__.__REQUEST_ATTRS__: self.requests_kwargs[key] = value + # def __getattr__(self, item): + # try: + # return self.__dict__[item] + # except: + # raise AttributeError("Request has no attribute %s" % item) + def __lt__(self, other): return self.priority < other.priority @property - def _session(self): - use_session = ( - setting.USE_SESSION if self.use_session is None else self.use_session - ) # self.use_session 优先级高 - if use_session and not self.__class__.session: - self.__class__.session = requests.Session() - http_adapter = HTTPAdapter( - pool_connections=1000, pool_maxsize=1000 - ) # pool_connections – 缓存的 urllib3 连接池个数 pool_maxsize – 连接池中保存的最大连接数 - self.__class__.session.mount( - "http", http_adapter - ) # 任何使用该session会话的 HTTP 请求，只要其 URL 是以给定的前缀开头，该传输适配器就会被使用到。 - - return self.__class__.session + def _proxies_pool(self): + if not self.__class__.proxies_pool: + self.__class__.proxies_pool = tools.import_cls(setting.PROXY_POOL)() + + return self.__class__.proxies_pool + + @property + def _downloader(self): + if not self.__class__.downloader: + self.__class__.downloader = tools.import_cls(setting.DOWNLOADER)() + + return self.__class__.downloader + + @property + def _session_downloader(self): + if not self.__class__.session_downloader: + self.__class__.session_downloader = tools.import_cls( + setting.SESSION_DOWNLOADER + )() + + return self.__class__.session_downloader + + @property + def _render_downloader(self): + if not self.__class__.render_downloader: + try: + self.__class__.render_downloader = tools.import_cls( + setting.RENDER_DOWNLOADER + )() + except AttributeError: + log.error('当前是渲染模式，请安装 pip install "feapder[render]"') + os._exit(0) + + return self.__class__.render_downloader @property def to_dict(self): @@ -192,40 +250,67 @@ def to_dict(self): if callable(self.callback) else self.callback ) - self.download_midware = ( - getattr(self.download_midware, "__name__") - if callable(self.download_midware) - else self.download_midware - ) + + if isinstance(self.download_midware, (tuple, list)): + self.download_midware = [ + getattr(download_midware, "__name__") + if callable(download_midware) + and download_midware.__class__.__name__ == "method" + else download_midware + for download_midware in self.download_midware + ] + else: + self.download_midware = ( + getattr(self.download_midware, "__name__") + if callable(self.download_midware) + and self.download_midware.__class__.__name__ == "method" + else self.download_midware + ) for key, value in self.__dict__.items(): if ( - key in self.__class__.DEFAULT_KEY_VALUE - and self.__class__.DEFAULT_KEY_VALUE.get(key) == value - or key == "requests_kwargs" + key in self.__class__._DEFAULT_KEY_VALUE_ + and self.__class__._DEFAULT_KEY_VALUE_.get(key) == value + or key in self.__class__._CUSTOM_PROPERTIES_ ): continue - if callable(value) or isinstance(value, Item): # 序列化 如item - value = tools.dumps_obj(value) + if value is not None: + if key in self.__class__.__REQUEST_ATTRS__: + if not isinstance( + value, (bool, float, int, str, tuple, list, dict) + ): + value = tools.dumps_obj(value) + else: + if not isinstance(value, (bool, float, int, str)): + value = tools.dumps_obj(value) request_dict[key] = value return request_dict - def get_response(self, save_cached=False): + @property + def callback_name(self): + return ( + getattr(self.callback, "__name__") + if callable(self.callback) + else self.callback + ) + + def make_requests_kwargs(self): """ - 获取带有selector功能的response - @param save_cached: 保存缓存 方便调试时不用每次都重新下载 - @return: + 处理参数 """ # 设置超时默认时间 - self.requests_kwargs.setdefault("timeout", 22) # connect=22 read=22 + self.requests_kwargs.setdefault( + "timeout", setting.REQUEST_TIMEOUT + ) # connect=22 read=22 # 设置stream - self.requests_kwargs.setdefault( - "stream", True - ) # 默认情况下，当你进行网络请求后，响应体会立即被下载。你可以通过 stream 参数覆盖这个行为，推迟下载响应体直到访问 Response.content 属性。此时仅有响应头被下载下来了。缺点： stream 设为 True，Requests 无法将连接释放回连接池，除非你 消耗了所有的数据，或者调用了 Response.close。 这样会带来连接效率低下的问题。 + # 默认情况下，当你进行网络请求后，响应体会立即被下载。 + # stream=True是，调用Response.content 才会下载响应体，默认只返回header。 + # 缺点： stream 设为 True，Requests 无法将连接释放回连接池，除非消耗了所有的数据，或者调用了 Response.close。 这样会带来连接效率低下的问题。 + self.requests_kwargs.setdefault("stream", True) # 关闭证书验证 self.requests_kwargs.setdefault("verify", False) @@ -233,54 +318,72 @@ def get_response(self, save_cached=False): # 设置请求方法 method = self.__dict__.get("method") if not method: - if "data" in self.requests_kwargs: + if "data" in self.requests_kwargs or "json" in self.requests_kwargs: method = "POST" else: method = "GET" + self.method = method - # 随机user—agent + # 设置user—agent headers = self.requests_kwargs.get("headers", {}) if "user-agent" not in headers and "User-Agent" not in headers: if self.random_user_agent and setting.RANDOM_HEADERS: - headers.update({"User-Agent": self.__class__.user_agent_pool.get()}) + # 随机user—agent + ua = self.__class__.user_agent_pool.get(setting.USER_AGENT_TYPE) + headers.update({"User-Agent": ua}) self.requests_kwargs.update(headers=headers) + else: + # 使用默认的user—agent + self.requests_kwargs.setdefault( + "headers", {"User-Agent": setting.DEFAULT_USERAGENT} + ) else: - self.requests_kwargs.setdefault( - "headers", - { - "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36" - }, - ) + self.custom_ua = True # 代理 proxies = self.requests_kwargs.get("proxies", -1) - if proxies == -1 and setting.PROXY_ENABLE and self.__class__.proxies_pool: + if proxies == -1 and setting.PROXY_ENABLE and setting.PROXY_EXTRACT_API: while True: - proxies = self.__class__.proxies_pool.get() + proxies = self._proxies_pool.get_proxy() if proxies: + self.requests_kwargs.update(proxies=proxies) break else: log.debug("暂无可用代理 ...") - proxies and self.requests_kwargs.update(proxies=proxies) + else: + self.custom_proxies = True + + def get_response(self, save_cached=False): + """ + 获取带有selector功能的response + @param save_cached: 保存缓存 方便调试时不用每次都重新下载 + @return: + """ + self.make_requests_kwargs() log.debug( """ - -------------- %s.%s request for ---------------- + -------------- %srequest for ---------------- url = %s method = %s - body = %s + args = %s """ % ( - self.parser_name, - ( - self.callback - and callable(self.callback) - and getattr(self.callback, "__name__") - or self.callback - ) - or "parser", + "" + if not self.parser_name + else "%s.%s " + % ( + self.parser_name, + ( + self.callback + and callable(self.callback) + and getattr(self.callback, "__name__") + or self.callback + ) + or "parse", + ), self.url, - method, + self.method, self.requests_kwargs, ) ) @@ -290,35 +393,90 @@ def get_response(self, save_cached=False): # # self.requests_kwargs.update(hooks={'response': hooks}) + # self.use_session 优先级高 use_session = ( setting.USE_SESSION if self.use_session is None else self.use_session - ) # self.use_session 优先级高 + ) - if use_session: - response = self._session.request(method, self.url, **self.requests_kwargs) + if self.render: + response = self._render_downloader.download(self) + elif use_session: + response = self._session_downloader.download(self) else: - response = requests.request(method, self.url, **self.requests_kwargs) + response = self._downloader.download(self) + + response.make_absolute_links = self.make_absolute_links - response = Response(response) if save_cached: self.save_cached(response, expire_time=self.__class__.cached_expire_time) return response + def get_params(self): + return self.requests_kwargs.get("params") + + def get_proxies(self) -> dict: + """ + + Returns: {"https": "https://ip:port", "http": "http://ip:port"} + + """ + return self.requests_kwargs.get("proxies") + + def get_proxy(self) -> str: + """ + + Returns: ip:port + + """ + proxies = self.get_proxies() + if proxies: + return re.sub( + "http.*?//", "", proxies.get("http", "") or proxies.get("https", "") + ) + + def del_proxy(self): + proxy = self.get_proxy() + if proxy: + self._proxies_pool.del_proxy(proxy) + del self.requests_kwargs["proxies"] + + def get_headers(self) -> dict: + return self.requests_kwargs.get("headers", {}) + + def get_user_agent(self) -> str: + return self.get_headers().get("user_agent") or self.get_headers().get( + "User-Agent" + ) + + def get_cookies(self) -> dict: + cookies = self.requests_kwargs.get("cookies") + if cookies and isinstance(cookies, RequestsCookieJar): + cookies = cookies.get_dict() + + if not cookies: + cookie_str = self.get_headers().get("Cookie") or self.get_headers().get( + "cookie" + ) + if cookie_str: + cookies = tools.get_cookies_from_str(cookie_str) + return cookies + @property def fingerprint(self): """ request唯一表识 @return: """ - args = [self.__dict__.get("url", "")] - params = self.requests_kwargs.get("params") - datas = self.requests_kwargs.get("data") - if params: - args.append(str(params)) - - if datas: - args.append(str(datas)) + url = self.__dict__.get("url", "") + # url 归一化 + url = tools.canonicalize_url(url) + args = [url] + + for arg in ["params", "data", "files", "auth", "cert", "json"]: + if self.requests_kwargs.get(arg): + args.append(self.requests_kwargs.get(arg)) + return tools.get_md5(*args) @property @@ -331,7 +489,9 @@ def _cache_db(self): @property def _cached_redis_key(self): if self.__class__.cached_redis_key: - return f"response_cached:{self.__class__.cached_redis_key}:{self.fingerprint}" + return ( + f"response_cached:{self.__class__.cached_redis_key}:{self.fingerprint}" + ) else: return f"response_cached:test:{self.fingerprint}" @@ -343,9 +503,7 @@ def save_cached(self, response, expire_time=1200): @return: """ - self._cache_db.strset( - self._cached_redis_key, response.to_dict, ex=expire_time - ) + self._cache_db.strset(self._cached_redis_key, response.to_dict, ex=expire_time) def get_response_from_cached(self, save_cached=True): """ @@ -382,4 +540,4 @@ def from_dict(cls, request_dict): return cls(**request_dict) def copy(self): - return self.__class__.from_dict(self.to_dict) + return self.__class__.from_dict(copy.deepcopy(self.to_dict)) diff --git a/feapder/network/response.py b/feapder/network/response.py index 987e8485..7f97861b 100644 --- a/feapder/network/response.py +++ b/feapder/network/response.py @@ -11,22 +11,25 @@ import datetime import os import re -import time +import tempfile +import webbrowser from urllib.parse import urlparse, urlunparse, urljoin -from bs4 import UnicodeDammit +from bs4 import UnicodeDammit, BeautifulSoup from requests.cookies import RequestsCookieJar from requests.models import Response as res +from w3lib.encoding import http_content_type_encoding, html_body_declared_encoding +from feapder import setting from feapder.network.selector import Selector from feapder.utils.log import log -from feapder.utils.tools import is_have_chinese FAIL_ENCODING = "ISO-8859-1" # html 源码中的特殊字符，需要删掉，否则会影响etree的构建 SPECIAL_CHARACTERS = [ - "[\x00-\x08\x0B\x0C\x0E-\x1F\x7F-\x9F]" # 移除控制字符 全部字符列表 https://zh.wikipedia.org/wiki/%E6%8E%A7%E5%88%B6%E5%AD%97%E7%AC%A6 + # 移除控制字符 全部字符列表 https://zh.wikipedia.org/wiki/%E6%8E%A7%E5%88%B6%E5%AD%97%E7%AC%A6 + "[\x00-\x08\x0B\x0C\x0E-\x1F\x7F-\x9F]" ] SPECIAL_CHARACTER_PATTERNS = [ @@ -35,16 +38,50 @@ class Response(res): - def __init__(self, response): + def __init__(self, response, make_absolute_links=None): + """ + + Args: + response: requests请求返回的response + make_absolute_links: 是否自动补全url + """ super(Response, self).__init__() self.__dict__.update(response.__dict__) + self.make_absolute_links = ( + make_absolute_links + if make_absolute_links is not None + else setting.MAKE_ABSOLUTE_LINKS + ) + self._cached_selector = None self._cached_text = None self._cached_json = None - self.code = None + self._encoding = None + self.encoding_errors = "strict" # strict / replace / ignore + self.browser = self.driver = None + + @classmethod + def from_text( + cls, + text: str, + url: str = "", + cookies: dict = None, + headers: dict = None, + encoding="utf-8", + ): + response_dict = { + "_content": text.encode(encoding=encoding), + "cookies": cookies or {}, + "encoding": encoding, + "headers": headers or {}, + "status_code": 200, + "elapsed": 0, + "url": url, + } + return cls.from_dict(response_dict) @classmethod def from_dict(cls, response_dict): @@ -81,48 +118,54 @@ def to_dict(self): return response_dict - def __setattr__(self, key, value): - if key == "code": - if not self.__dict__.get("custom_text"): - self.__dict__["_cached_selector"] = None - self.__dict__["_cached_text"] = None - self.__dict__["_cached_json"] = None - self.__dict__[key] = value + def __clear_cache(self): + self.__dict__["_cached_selector"] = None + self.__dict__["_cached_text"] = None + self.__dict__["_cached_json"] = None - elif key == "text": - self.__dict__["_cached_text"] = value - self.__dict__["_cached_selector"] = None - self.__dict__["custom_text"] = True - - elif key == "json": - self.__dict__["_cached_json"] = value - self.__dict__["custom_text"] = True - - else: - self.__dict__[key] = value - - def _get_html(self): - - if self.encoding != FAIL_ENCODING: - # return response as a unicode string - # html = super(Response, self).text - html = self.__text + @property + def encoding(self): + """ + 编码优先级：自定义编码 > header中编码 > 页面编码 > 根据content猜测的编码 + """ + self._encoding = ( + self._encoding + or self._headers_encoding() + or self._body_declared_encoding() + or self.apparent_encoding + ) + return self._encoding + + @encoding.setter + def encoding(self, val): + self.__clear_cache() + self._encoding = val + + code = encoding + + def _headers_encoding(self): + """ + 从headers获取头部charset编码 + """ + content_type = self.headers.get("Content-Type") or self.headers.get( + "content-type" + ) + if content_type: + return ( + http_content_type_encoding(content_type) or "utf-8" + if "application/json" in content_type + else None + ) - else: - # don't attempt decode, return response in bytes - html = self.content - html = self._get_unicode_html(html) - if not is_have_chinese(html): - self.encoding = "gb2312" - html = self.text + def _body_declared_encoding(self): + """ + 从html xml等获取 + """ - return html or "" + return html_body_declared_encoding(self.content) def _get_unicode_html(self, html): - if isinstance(html, str): - return html - - if not html: + if not html or not isinstance(html, bytes): return html converted = UnicodeDammit(html, is_html=True) @@ -168,10 +211,10 @@ def _make_absolute(self, link): def _absolute_links(self, text): regexs = [ - r'(<(?i)a.*?href\s*?=\s*?["\'])(.+?)(["\'])', # a - r'(<(?i)img.*?src\s*?=\s*?["\'])(.+?)(["\'])', # img - r'(<(?i)link.*?href\s*?=\s*?["\'])(.+?)(["\'])', # css - r'(<(?i)script.*?src\s*?=\s*?["\'])(.+?)(["\'])', # js + r'( 标签后插入一个标签 + repl = fr'\1' + body = re.sub(rb"(|\s.*?>))", repl.encode("utf-8"), body) + + fd, fname = tempfile.mkstemp(".html") + os.write(fd, body) + os.close(fd) + return webbrowser.open(f"file://{fname}") diff --git a/feapder/network/selector.py b/feapder/network/selector.py index 381c6b7c..901f4eb5 100644 --- a/feapder/network/selector.py +++ b/feapder/network/selector.py @@ -9,10 +9,13 @@ """ import re +import parsel import six from lxml import etree +from packaging import version from parsel import Selector as ParselSelector from parsel import SelectorList as ParselSelectorList +from parsel import selector from w3lib.html import replace_entities as w3lib_replace_entities @@ -54,8 +57,7 @@ def extract_regex(regex, text, replace_entities=True, flags=0): def create_root_node(text, parser_cls, base_url=None): - """Create root node for text using given parser class. - """ + """Create root node for text using given parser class.""" body = text.strip().replace("\x00", "").encode("utf8") or b"" parser = parser_cls(recover=True, encoding="utf8", huge_tree=True) root = etree.fromstring(body, parser=parser, base_url=base_url) @@ -64,6 +66,10 @@ def create_root_node(text, parser_cls, base_url=None): return root +if version.parse(parsel.__version__) < version.parse("1.7.0"): + selector.create_root_node = create_root_node + + class SelectorList(ParselSelectorList): """ The :class:`SelectorList` class is a subclass of the builtin ``list`` @@ -150,6 +156,3 @@ def re(self, regex, replace_entities=True, flags=re.S): return extract_regex( regex, self.get(), replace_entities=replace_entities, flags=flags ) - - def _get_root(self, text, base_url=None): - return create_root_node(text, self._parser, base_url=base_url) diff --git a/feapder/network/user_agent.py b/feapder/network/user_agent.py index 4bf26fcf..7f9024d4 100644 --- a/feapder/network/user_agent.py +++ b/feapder/network/user_agent.py @@ -5,64 +5,1062 @@ @summary: --------- @author: Boris -@email: boris@bzkj.tech +@email: boris_liu@foxmail.com """ import random -USER_AGENTS = [ - "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.1 Safari/537.36", - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36", - "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36", - "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2226.0 Safari/537.36", - "Mozilla/5.0 (Windows NT 6.4; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36", - "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36", - "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2224.3 Safari/537.36", - "Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.93 Safari/537.36", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.124 Safari/537.36", - "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36", - "Mozilla/5.0 (Windows NT 4.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36", - "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.67 Safari/537.36", - "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.67 Safari/537.36", - "Mozilla/5.0 (X11; OpenBSD i386) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.125 Safari/537.36", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1944.0 Safari/537.36", - "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.3319.102 Safari/537.36", - "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.2309.372 Safari/537.36", - "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.2117.157 Safari/537.36", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.47 Safari/537.36", - "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1866.237 Safari/537.36", - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.137 Safari/4E423F", - "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.116 Safari/537.36 Mozilla/5.0 (iPad; U; CPU OS 3_2 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Version/4.0.4 Mobile/7B334b Safari/531.21.10", - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.517 Safari/537.36", - "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1667.0 Safari/537.36", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1664.3 Safari/537.36", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1664.3 Safari/537.36", - "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.16 Safari/537.36", - "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1623.0 Safari/537.36", - "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.17 Safari/537.36", - "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.62 Safari/537.36", - "Mozilla/5.0 (X11; CrOS i686 4319.74.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.57 Safari/537.36", - "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.2 Safari/537.36", - "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1468.0 Safari/537.36", - "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1467.0 Safari/537.36", - "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1464.0 Safari/537.36", - "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1500.55 Safari/537.36", - "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36", - "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36", - "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36", - "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36", - "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.90 Safari/537.36", - "Mozilla/5.0 (X11; NetBSD) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.116 Safari/537.36", - "Mozilla/5.0 (X11; CrOS i686 3912.101.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.116 Safari/537.36", - "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.60 Safari/537.17", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1309.0 Safari/537.17", - "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.15 (KHTML, like Gecko) Chrome/24.0.1295.0 Safari/537.15", - "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.14 (KHTML, like Gecko) Chrome/24.0.1292.0 Safari/537.14", -] +USER_AGENTS = { + "chrome": [ + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.1 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2226.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.4; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2224.3 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.93 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.124 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 4.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.67 Safari/537.36", + "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.67 Safari/537.36", + "Mozilla/5.0 (X11; OpenBSD i386) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.125 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1944.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.3319.102 Safari/537.36", + "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.2309.372 Safari/537.36", + "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.2117.157 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.47 Safari/537.36", + "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1866.237 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.137 Safari/4E423F", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.517 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1667.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1664.3 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1664.3 Safari/537.36", + "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.16 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1623.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.17 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.62 Safari/537.36", + "Mozilla/5.0 (X11; CrOS i686 4319.74.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.57 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.2 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1468.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1467.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1464.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1500.55 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36", + "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.90 Safari/537.36", + "Mozilla/5.0 (X11; NetBSD) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.116 Safari/537.36", + "Mozilla/5.0 (X11; CrOS i686 3912.101.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.116 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.60 Safari/537.17", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1309.0 Safari/537.17", + "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.15 (KHTML, like Gecko) Chrome/24.0.1295.0 Safari/537.15", + "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.14 (KHTML, like Gecko) Chrome/24.0.1292.0 Safari/537.14", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3215.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.84 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.62 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3790.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.92 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.75 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.79 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.89 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.63 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.116 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.90 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.24 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.136 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.62 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.0.3016 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36 Kinza/6.1.5", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.48 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.125 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.135 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.83 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.2.0.1713 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.47 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.2 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.819 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.75 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.102 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.41 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.785 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.9 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3235.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3409.85 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4371.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.9 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.43 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36 CravingExplorer/2.4.1", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.122 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.106 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.84 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4121.813 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.121 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.107 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.9 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.129 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.158 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.190 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.58 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.113 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.106 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.140 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36", + "Mozilla/5.0 (Microsoft Windows NT 10.0.16299.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36 (FTM)", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4500.0 Iron Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.121 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4427.5 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3835.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.190 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; ) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/82.0.4085.4 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.75 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.135 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.82 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.116 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.116 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.91 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.109 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.4000.0 Iron Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.135 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.122 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.0.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.41 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; ) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.116 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.41 Safari/537.36", + "Mozilla/5.0 (Windows NT 5.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 ADG/11.0.2566 AOLBUILD/11.0.2566 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/78.0.3904.108 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/88.0.4324.152 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 ADG/11.0.2510 AOLBUILD/11.0.2510 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; ) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36 AOLShield/83.0.4103.0", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 AOL/11.0 AOLBUILD/11.0.1839 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.75 ADG/11.0.2414 AOLBUILD/11.0.2414 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 ADG/11.0.2566 AOLBUILD/11.0.2566 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36 AOLShield/83.0.4103.2", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/80.0.3987.87 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/84.0.4147.105 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/86.0.4240.183 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/88.0.4324.152 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/90.0.4430.72 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 ADG/11.0.2510 AOLBUILD/11.0.2510 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/86.0.4240.198 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 ADG/11.0.2566 AOLBUILD/11.0.2566 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/78.0.3904.97 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/84.0.4147.105 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/86.0.4240.198 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/88.0.4324.182 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/78.0.3904.108 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/87.0.4280.88 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/89.0.4389.114 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 ADG/11.0.2510 AOLBUILD/11.0.2510 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/87.0.4280.101 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 AOL/11.0 AOLBUILD/11.0.1839 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 ADG/11.0.2470 AOLBUILD/11.0.2470 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 ADG/11.0.2566 AOLBUILD/11.0.2566 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36 AOLShield/79.0.3945.5", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/77.0.3865.90 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/79.0.3945.88 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/80.0.3987.162 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/84.0.4147.89 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/86.0.4240.99 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/87.0.4280.141 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/89.0.4389.72 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.102 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.106 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.128 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.190 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.150 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.75 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.157 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.123 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4558.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.101 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.85 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.121 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; ) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.113 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.135 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.102 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.109 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4564.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.121 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.54 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.83 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.93 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.87 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.72 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.122 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.107 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.125 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.81 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.135 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.81 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.101 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.190 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.164 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.74 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.60 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3409.13 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.26 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.0.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.54 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.81 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.84 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.64 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4591.54 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.101.4951.54 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.83 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.107 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.75 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.7113.93 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.54 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.0.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.190 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.5005.49 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.54 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.1150.52 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4950.0 Iron Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4450.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36", + "Mozilla/5.0 (Windows NT 11.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4868.173 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.84 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.1483.27 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.66 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.3478.83 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.0.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.117 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.67 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.60 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.0.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.5005.115 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.82 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.5118.205 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Agency/97.8.8247.48", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.63 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.164 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4137.1 SputnikBrowser/5.6.6280.0 (GOST) Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.84 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.43 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.122 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.122 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.79 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.106 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.122 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/82.0.4078.2 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.122 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.122 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.87 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.3538.77 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.122 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.83 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.83 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.125 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.5 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.6 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_0_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.1 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3409.631 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.3 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_0_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.101 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.2 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.93 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.8 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.5 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_0_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3409.1 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.183 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.44 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.779 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.19 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.6 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36 FS", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36\tChrome 79.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36\tChrome Generic", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_16_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.83 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.113 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_16_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.182 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.192 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_2_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.69 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.146 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.186 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.192 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.150 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.170 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_2_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_2_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4450.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.182 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.150 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_2_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.192 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.192 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_2_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.67 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_3_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/524.34", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.102 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.146 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.192 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_2_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.82 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_2_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.192 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.105 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_2_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.146 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.193 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.51 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_2_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.152 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.152 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.3538.77 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.150 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.150 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/537.36 (KHTML, like Gecko, Mediapartners-Google) Chrome/77.0.3865.99 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/537.36 (KHTML, like Gecko, Mediapartners-Google) Chrome/81.0.4044.108 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/537.36 (KHTML, like Gecko, Mediapartners-Google) Chrome/83.0.4103.118 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/537.36 (KHTML, like Gecko, Mediapartners-Google) Chrome/84.0.4147.108 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/537.36 (KHTML, like Gecko, Mediapartners-Google) Chrome/84.0.4147.140 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/537.36 (KHTML, like Gecko, Mediapartners-Google) Chrome/85.0.4183.122 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/537.36 (KHTML, like Gecko, Mediapartners-Google) Chrome/87.0.4280.90 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/537.36 (KHTML, like Gecko, Mediapartners-Google) Chrome/88.0.4324.175 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/537.36 (KHTML, like Gecko, Mediapartners-Google) Chrome/89.0.4389.93 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/537.36 (KHTML, like Gecko, Mediapartners-Google) Chrome/89.0.4389.127 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/86.0.4240.75 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/79.0.3945.88 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/80.0.3987.116 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/81.0.4044.113 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/84.0.4147.135 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/86.0.4240.75 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/86.0.4240.198 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/87.0.4280.141 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/89.0.4389.72 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/78.0.3904.70 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/80.0.3987.116 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/80.0.3987.162 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/86.0.4240.75 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/87.0.4280.67 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/88.0.4324.152 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/89.0.4389.90 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/77.0.3865.90 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/78.0.3904.108 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/80.0.3987.87 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/80.0.3987.162 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/83.0.4103.116 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/85.0.4183.83 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/86.0.4240.99 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/86.0.4240.198 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/87.0.4280.141 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/88.0.4324.182 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/89.0.4389.90 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/90.0.4430.72 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/79.0.3945.88 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/79.0.3945.88 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/77.0.3865.90 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/78.0.3904.108 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/80.0.3987.122 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/81.0.4044.113 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/84.0.4147.89 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/85.0.4183.102 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/86.0.4240.183 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/87.0.4280.88 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/88.0.4324.146 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/89.0.4389.72 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/89.0.4389.114 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_0) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/78.0.3904.108 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_1) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/78.0.3904.70 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/78.0.3904.97 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/79.0.3945.130 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/78.0.3904.108 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/80.0.3987.87 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/80.0.3987.149 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/84.0.4147.89 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/86.0.4240.99 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/80.0.3987.149 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/81.0.4044.122 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/84.0.4147.89 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/87.0.4280.101 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/83.0.4103.97 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/84.0.4147.105 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/86.0.4240.75 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/78.0.3904.87 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/83.0.4103.106 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/84.0.4147.125 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/85.0.4183.121 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/86.0.4240.183 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/88.0.4324.152 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/83.0.4103.116 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/85.0.4183.102 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/86.0.4240.111 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/87.0.4280.60 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/87.0.4280.141 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/88.0.4324.182 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/89.0.4389.90 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_16_0) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/80.0.3987.116 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_0_0) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/86.0.4240.183 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_0_1) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/87.0.4280.67 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_0_1) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/88.0.4324.96 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_0_1) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/88.0.4324.192 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1_0) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/87.0.4280.67 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1_0) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/88.0.4324.96 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1_0) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/89.0.4389.72 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_2_0) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/87.0.4280.101 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_2_0) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/88.0.4324.152 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_2_1) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/87.0.4280.101 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_2_1) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/88.0.4324.182 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_2_1) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/89.0.4389.90 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_2_2) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/88.0.4324.146 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_2_2) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/89.0.4389.72 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_2_3) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/88.0.4324.96 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_2_3) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/89.0.4389.72 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_2_3) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/89.0.4389.114 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_3_0) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/89.0.4389.114 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/537.36 (KHTML, like Gecko, Mediapartners-Google) Chrome/89.0.4389.130 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.128 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_3_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.128 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.69 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.54 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.82 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.114 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.85 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.54 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4582.189 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/82.0.4083.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4612.206 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.80 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.80 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4702.147 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.80 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.93 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.63 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.80 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.82 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.93 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.80 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.80 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.107 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4691.94 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4889.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.71 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.79 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.79 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.84 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.84 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.84 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.54 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.54 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.54 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.54 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.9999.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.64 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.54 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.84 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.64 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.54 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.64 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.84 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.5005.40 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.60 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.63 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.55 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4880.146 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.55 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.80 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.80 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.54 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.147 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.109 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.75 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.71 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.54 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.75 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.109 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4886.93 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.150 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1_0) AppleWebKit/537.36 (KHTML, like Gecko) Brave Chrome/89.0.4389.105 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4886.148 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.75 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.80 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.102 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.75 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.64 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.5163.147 Safari/537.36" + ], + "opera": [ + "Opera/9.80 (X11; Linux i686; Ubuntu/14.10) Presto/2.12.388 Version/12.16", + "Opera/9.80 (Windows NT 6.0) Presto/2.12.388 Version/12.14", + "Mozilla/5.0 (Windows NT 6.0; rv:2.0) Gecko/20100101 Firefox/4.0 Opera 12.14", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0) Opera 12.14", + "Opera/12.80 (Windows NT 5.1; U; en) Presto/2.10.289 Version/12.02", + "Opera/9.80 (Windows NT 6.1; U; es-ES) Presto/2.9.181 Version/12.00", + "Opera/9.80 (Windows NT 5.1; U; zh-sg) Presto/2.9.181 Version/12.00", + "Opera/12.0(Windows NT 5.2;U;en)Presto/22.9.168 Version/12.00", + "Opera/12.0(Windows NT 5.1;U;en)Presto/22.9.168 Version/12.00", + "Mozilla/5.0 (Windows NT 5.1) Gecko/20100101 Firefox/14.0 Opera/12.0", + "Opera/9.80 (Windows NT 6.1; WOW64; U; pt) Presto/2.10.229 Version/11.62", + "Opera/9.80 (Windows NT 6.0; U; pl) Presto/2.10.229 Version/11.62", + "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52", + "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; de) Presto/2.9.168 Version/11.52", + "Opera/9.80 (Windows NT 5.1; U; en) Presto/2.9.168 Version/11.51", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; de) Opera 11.51", + "Opera/9.80 (X11; Linux x86_64; U; fr) Presto/2.9.168 Version/11.50", + "Opera/9.80 (X11; Linux i686; U; hu) Presto/2.9.168 Version/11.50", + "Opera/9.80 (X11; Linux i686; U; ru) Presto/2.8.131 Version/11.11", + "Opera/9.80 (X11; Linux i686; U; es-ES) Presto/2.8.131 Version/11.11", + "Mozilla/5.0 (Windows NT 5.1; U; en; rv:1.8.1) Gecko/20061208 Firefox/5.0 Opera 11.11", + "Opera/9.80 (X11; Linux x86_64; U; bg) Presto/2.8.131 Version/11.10", + "Opera/9.80 (Windows NT 6.0; U; en) Presto/2.8.99 Version/11.10", + "Opera/9.80 (Windows NT 5.1; U; zh-tw) Presto/2.8.131 Version/11.10", + "Opera/9.80 (Windows NT 6.1; Opera Tablet/15165; U; en) Presto/2.8.149 Version/11.1", + "Opera/9.80 (X11; Linux x86_64; U; Ubuntu/10.10 (maverick); pl) Presto/2.7.62 Version/11.01", + "Opera/9.80 (X11; Linux i686; U; ja) Presto/2.7.62 Version/11.01", + "Opera/9.80 (X11; Linux i686; U; fr) Presto/2.7.62 Version/11.01", + "Opera/9.80 (Windows NT 6.1; U; zh-tw) Presto/2.7.62 Version/11.01", + "Opera/9.80 (Windows NT 6.1; U; zh-cn) Presto/2.7.62 Version/11.01", + "Opera/9.80 (Windows NT 6.1; U; sv) Presto/2.7.62 Version/11.01", + "Opera/9.80 (Windows NT 6.1; U; en-US) Presto/2.7.62 Version/11.01", + "Opera/9.80 (Windows NT 6.1; U; cs) Presto/2.7.62 Version/11.01", + "Opera/9.80 (Windows NT 6.0; U; pl) Presto/2.7.62 Version/11.01", + "Opera/9.80 (Windows NT 5.2; U; ru) Presto/2.7.62 Version/11.01", + "Opera/9.80 (Windows NT 5.1; U;) Presto/2.7.62 Version/11.01", + "Opera/9.80 (Windows NT 5.1; U; cs) Presto/2.7.62 Version/11.01", + "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2.13) Gecko/20101213 Opera/9.80 (Windows NT 6.1; U; zh-tw) Presto/2.7.62 Version/11.01", + "Mozilla/5.0 (Windows NT 6.1; U; nl; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 11.01", + "Mozilla/5.0 (Windows NT 6.1; U; de; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 11.01", + "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; de) Opera 11.01", + "Opera/9.80 (X11; Linux x86_64; U; pl) Presto/2.7.62 Version/11.00", + "Opera/9.80 (X11; Linux i686; U; it) Presto/2.7.62 Version/11.00", + "Opera/9.80 (Windows NT 6.1; U; zh-cn) Presto/2.6.37 Version/11.00", + "Opera/9.80 (Windows NT 6.1; U; pl) Presto/2.7.62 Version/11.00", + "Opera/9.80 (Windows NT 6.1; U; ko) Presto/2.7.62 Version/11.00", + "Opera/9.80 (Windows NT 6.1; U; fi) Presto/2.7.62 Version/11.00", + "Opera/9.80 (Windows NT 6.1; U; en-GB) Presto/2.7.62 Version/11.00", + "Opera/9.80 (Windows NT 6.1 x64; U; en) Presto/2.7.62 Version/11.00", + "Opera/9.80 (Windows NT 6.0; U; en) Presto/2.7.39 Version/11.00", + ], + "firefox": [ + "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1", + "Mozilla/5.0 (Windows NT 6.3; rv:36.0) Gecko/20100101 Firefox/36.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10; rv:33.0) Gecko/20100101 Firefox/33.0", + "Mozilla/5.0 (X11; Linux i586; rv:31.0) Gecko/20100101 Firefox/31.0", + "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:31.0) Gecko/20130401 Firefox/31.0", + "Mozilla/5.0 (Windows NT 5.1; rv:31.0) Gecko/20100101 Firefox/31.0", + "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:29.0) Gecko/20120101 Firefox/29.0", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0) Gecko/20100101 Firefox/29.0", + "Mozilla/5.0 (X11; OpenBSD amd64; rv:28.0) Gecko/20100101 Firefox/28.0", + "Mozilla/5.0 (X11; Linux x86_64; rv:28.0) Gecko/20100101 Firefox/28.0", + "Mozilla/5.0 (Windows NT 6.1; rv:27.3) Gecko/20130101 Firefox/27.3", + "Mozilla/5.0 (Windows NT 6.2; Win64; x64; rv:27.0) Gecko/20121011 Firefox/27.0", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0) Gecko/20100101 Firefox/25.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:25.0) Gecko/20100101 Firefox/25.0", + "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:24.0) Gecko/20100101 Firefox/24.0", + "Mozilla/5.0 (Windows NT 6.0; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:24.0) Gecko/20100101 Firefox/24.0", + "Mozilla/5.0 (Windows NT 6.2; rv:22.0) Gecko/20130405 Firefox/23.0", + "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:23.0) Gecko/20130406 Firefox/23.0", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:23.0) Gecko/20131011 Firefox/23.0", + "Mozilla/5.0 (Windows NT 6.2; rv:22.0) Gecko/20130405 Firefox/22.0", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:22.0) Gecko/20130328 Firefox/22.0", + "Mozilla/5.0 (Windows NT 6.1; rv:22.0) Gecko/20130405 Firefox/22.0", + "Mozilla/5.0 (Microsoft Windows NT 6.2.9200.0); rv:22.0) Gecko/20130405 Firefox/22.0", + "Mozilla/5.0 (Windows NT 6.2; Win64; x64; rv:16.0.1) Gecko/20121011 Firefox/21.0.1", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:16.0.1) Gecko/20121011 Firefox/21.0.1", + "Mozilla/5.0 (Windows NT 6.2; Win64; x64; rv:21.0.0) Gecko/20121011 Firefox/21.0.0", + "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:21.0) Gecko/20130331 Firefox/21.0", + "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:21.0) Gecko/20100101 Firefox/21.0", + "Mozilla/5.0 (X11; Linux i686; rv:21.0) Gecko/20100101 Firefox/21.0", + "Mozilla/5.0 (Windows NT 6.2; WOW64; rv:21.0) Gecko/20130514 Firefox/21.0", + "Mozilla/5.0 (Windows NT 6.2; rv:21.0) Gecko/20130326 Firefox/21.0", + "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko/20130401 Firefox/21.0", + "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko/20130331 Firefox/21.0", + "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko/20130330 Firefox/21.0", + "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko/20100101 Firefox/21.0", + "Mozilla/5.0 (Windows NT 6.1; rv:21.0) Gecko/20130401 Firefox/21.0", + "Mozilla/5.0 (Windows NT 6.1; rv:21.0) Gecko/20130328 Firefox/21.0", + "Mozilla/5.0 (Windows NT 6.1; rv:21.0) Gecko/20100101 Firefox/21.0", + "Mozilla/5.0 (Windows NT 5.1; rv:21.0) Gecko/20130401 Firefox/21.0", + "Mozilla/5.0 (Windows NT 5.1; rv:21.0) Gecko/20130331 Firefox/21.0", + "Mozilla/5.0 (Windows NT 5.1; rv:21.0) Gecko/20100101 Firefox/21.0", + "Mozilla/5.0 (Windows NT 5.0; rv:21.0) Gecko/20100101 Firefox/21.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:21.0) Gecko/20100101 Firefox/21.0", + "Mozilla/5.0 (Windows NT 6.2; Win64; x64;) Gecko/20100101 Firefox/20.0", + "Mozilla/5.0 (Windows x86; rv:19.0) Gecko/20100101 Firefox/19.0", + "Mozilla/5.0 (Windows NT 6.1; rv:6.0) Gecko/20100101 Firefox/19.0", + "Mozilla/5.0 (Windows NT 6.1; rv:14.0) Gecko/20100101 Firefox/18.0.1", + "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:18.0) Gecko/20100101 Firefox/18.0", + "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:17.0) Gecko/20100101 Firefox/17.0.6", + ], + "internetexplorer": [ + "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; AS; rv:11.0) like Gecko", + "Mozilla/5.0 (compatible, MSIE 11, Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko", + "Mozilla/5.0 (compatible; MSIE 10.6; Windows NT 6.1; Trident/5.0; InfoPath.2; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 2.0.50727) 3gpp-gba UNTRUSTED/1.0", + "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 7.0; InfoPath.3; .NET CLR 3.1.40767; Trident/6.0; en-IN)", + "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)", + "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0)", + "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/5.0)", + "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/4.0; InfoPath.2; SV1; .NET CLR 2.0.50727; WOW64)", + "Mozilla/5.0 (compatible; MSIE 10.0; Macintosh; Intel Mac OS X 10_7_3; Trident/6.0)", + "Mozilla/4.0 (Compatible; MSIE 8.0; Windows NT 5.2; Trident/6.0)", + "Mozilla/4.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/5.0)", + "Mozilla/1.22 (compatible; MSIE 10.0; Windows 3.1)", + "Mozilla/5.0 (Windows; U; MSIE 9.0; WIndows NT 9.0; en-US))", + "Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 7.1; Trident/5.0)", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; Media Center PC 6.0; InfoPath.3; MS-RTC LM 8; Zune 4.7)", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; Media Center PC 6.0; InfoPath.3; MS-RTC LM 8; Zune 4.7", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; Zune 4.0; InfoPath.3; MS-RTC LM 8; .NET4.0C; .NET4.0E)", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; chromeframe/12.0.742.112)", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 2.0.50727; SLCC2; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; Zune 4.0; Tablet PC 2.0; InfoPath.3; .NET4.0C; .NET4.0E)", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; yie8)", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.2; .NET CLR 1.1.4322; .NET4.0C; Tablet PC 2.0)", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; FunWebProducts)", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; chromeframe/13.0.782.215)", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; chromeframe/11.0.696.57)", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0) chromeframe/10.0.648.205", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/4.0; GTB7.4; InfoPath.1; SV1; .NET CLR 2.8.52393; WOW64; en-US)", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0; Trident/5.0; chromeframe/11.0.696.57)", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0; Trident/4.0; GTB7.4; InfoPath.3; SV1; .NET CLR 3.1.76908; WOW64; en-US)", + "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; GTB7.4; InfoPath.2; SV1; .NET CLR 3.3.69573; WOW64; en-US)", + "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)", + "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; InfoPath.1; SV1; .NET CLR 3.8.36217; WOW64; en-US)", + "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; .NET CLR 2.7.58687; SLCC2; Media Center PC 5.0; Zune 3.4; Tablet PC 3.6; InfoPath.3)", + "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.2; Trident/4.0; Media Center PC 4.0; SLCC1; .NET CLR 3.0.04320)", + "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 1.1.4322)", + "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; InfoPath.2; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 2.0.50727)", + "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727)", + "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; SLCC1; .NET CLR 1.1.4322)", + "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.0; Trident/4.0; InfoPath.1; SV1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 3.0.04506.30)", + "Mozilla/5.0 (compatible; MSIE 7.0; Windows NT 5.0; Trident/4.0; FBSMTWB; .NET CLR 2.0.34861; .NET CLR 3.0.3746.3218; .NET CLR 3.5.33652; msn OptimizedIE8;ENUS)", + "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.2; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)", + "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; Media Center PC 6.0; InfoPath.2; MS-RTC LM 8)", + "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; Media Center PC 6.0; InfoPath.2; MS-RTC LM 8", + "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; Media Center PC 6.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET4.0C)", + "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; InfoPath.3; .NET4.0C; .NET4.0E; .NET CLR 3.5.30729; .NET CLR 3.0.30729; MS-RTC LM 8)", + "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; InfoPath.2)", + "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; Zune 3.0)", + ], + "safari": [ + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/7046A194A", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/537.13+ (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/534.55.3 (KHTML, like Gecko) Version/5.1.3 Safari/534.53.10", + "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; de-at) AppleWebKit/533.21.1 (KHTML, like Gecko) Version/5.0.5 Safari/533.21.1", + "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_7; da-dk) AppleWebKit/533.21.1 (KHTML, like Gecko) Version/5.0.5 Safari/533.21.1", + "Mozilla/5.0 (Windows; U; Windows NT 6.1; tr-TR) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", + "Mozilla/5.0 (Windows; U; Windows NT 6.1; ko-KR) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", + "Mozilla/5.0 (Windows; U; Windows NT 6.1; fr-FR) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", + "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", + "Mozilla/5.0 (Windows; U; Windows NT 6.1; cs-CZ) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", + "Mozilla/5.0 (Windows; U; Windows NT 6.0; ja-JP) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", + "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", + "Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10_5_8; zh-cn) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", + "Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10_5_8; ja-jp) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", + "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_7; ja-jp) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", + "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; zh-cn) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", + "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; sv-se) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", + "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; ko-kr) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", + "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; ja-jp) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", + "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; it-it) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", + "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; fr-fr) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", + "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; es-es) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", + "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; en-us) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", + "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; en-gb) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", + "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; de-de) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", + "Mozilla/5.0 (Windows; U; Windows NT 6.1; sv-SE) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", + "Mozilla/5.0 (Windows; U; Windows NT 6.1; ja-JP) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", + "Mozilla/5.0 (Windows; U; Windows NT 6.1; de-DE) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", + "Mozilla/5.0 (Windows; U; Windows NT 6.0; hu-HU) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", + "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", + "Mozilla/5.0 (Windows; U; Windows NT 6.0; de-DE) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", + "Mozilla/5.0 (Windows; U; Windows NT 5.1; ru-RU) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", + "Mozilla/5.0 (Windows; U; Windows NT 5.1; ja-JP) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", + "Mozilla/5.0 (Windows; U; Windows NT 5.1; it-IT) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", + "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", + "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_7; en-us) AppleWebKit/534.16+ (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", + "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; fr-ch) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", + "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_5; de-de) AppleWebKit/534.15+ (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", + "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_5; ar) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", + "Mozilla/5.0 (Android 2.2; Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", + "Mozilla/5.0 (Windows; U; Windows NT 6.1; zh-HK) AppleWebKit/533.18.1 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5", + "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5", + "Mozilla/5.0 (Windows; U; Windows NT 6.0; tr-TR) AppleWebKit/533.18.1 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5", + "Mozilla/5.0 (Windows; U; Windows NT 6.0; nb-NO) AppleWebKit/533.18.1 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5", + "Mozilla/5.0 (Windows; U; Windows NT 6.0; fr-FR) AppleWebKit/533.18.1 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5", + "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-TW) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5", + "Mozilla/5.0 (Windows; U; Windows NT 5.1; ru-RU) AppleWebKit/533.18.1 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5", + "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; zh-cn) AppleWebKit/533.18.1 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5", + ], + "mobile": [ + "Mozilla/5.0 (PlayBook; U; RIM Tablet OS 2.1.0; en-US) AppleWebKit/536.2+ (KHTML like Gecko) Version/14.2 Safari/536.2+", + "Mozilla/5.0 (PlayBook; U; RIM Tablet OS 2.1.0; en-US) AppleWebKit/536.2+ (KHTML like Gecko) Version/14.2 Safari/536.2+", + "Mozilla/5.0 (BB10; Touch) AppleWebKit/537.10+ (KHTML, like Gecko) Version/14.2 Mobile Safari/537.10+", + "Mozilla/5.0 (BB10; Touch) AppleWebKit/537.10+ (KHTML, like Gecko) Version/14.2 Mobile Safari/537.10+", + "Mozilla/5.0 (Linux; U; Android 4.3; en-us; SM-N900T Build/JSS15J) AppleWebKit/534.30 (KHTML, like Gecko) Version/14.2 Mobile Safari/534.30", + "Mozilla/5.0 (Linux; U; Android 4.3; en-us; SM-N900T Build/JSS15J) AppleWebKit/534.30 (KHTML, like Gecko) Version/14.2 Mobile Safari/534.30", + "Mozilla/5.0 (Linux; U; Android 4.1; en-us; GT-N7100 Build/JRO03C) AppleWebKit/534.30 (KHTML, like Gecko) Version/14.2 Mobile Safari/534.30", + "Mozilla/5.0 (Linux; U; Android 4.1; en-us; GT-N7100 Build/JRO03C) AppleWebKit/534.30 (KHTML, like Gecko) Version/14.2 Mobile Safari/534.30", + "Mozilla/5.0 (Linux; U; Android 4.0; en-us; GT-I9300 Build/IMM76D) AppleWebKit/534.30 (KHTML, like Gecko) Version/14.2 Mobile Safari/534.30", + "Mozilla/5.0 (Linux; U; Android 4.0; en-us; GT-I9300 Build/IMM76D) AppleWebKit/534.30 (KHTML, like Gecko) Version/14.2 Mobile Safari/534.30", + "Mozilla/5.0 (Linux; Android 5.0; SM-G900P Build/LRX21T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4576.0 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 5.0; SM-G900P Build/LRX21T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4576.0 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 7.0; SM-G950U Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4576.0 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 7.0; SM-G950U Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4576.0 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 8.0.0; SM-G965U Build/R16NW) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4576.0 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 8.0.0; SM-G965U Build/R16NW) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4576.0 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 8.1.0; SM-T837A) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4576.0 Safari/537.36", + "Mozilla/5.0 (Linux; Android 8.1.0; SM-T837A) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4576.0 Safari/537.36", + "Mozilla/5.0 (iPad; CPU OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.2 Mobile/15E148 Safari/604.1", + "Mozilla/5.0 (iPad; CPU OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.2 Mobile/15E148 Safari/604.1", + "Mozilla/5.0 (iPad; CPU OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.2 Mobile/15E148 Safari/604.1", + "Mozilla/5.0 (iPad; CPU OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.2 Mobile/15E148 Safari/604.1", + "Mozilla/5.0 (iPad; CPU OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.2 Mobile/15E148 Safari/604.1", + "Mozilla/5.0 (iPad; CPU OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.2 Mobile/15E148 Safari/604.1", + "Mozilla/5.0 (iPad; CPU OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.2 Mobile/15E148 Safari/604.1", + "Mozilla/5.0 (iPad; CPU OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.2 Mobile/15E148 Safari/604.1", + "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/14.2 Mobile/15A372 Safari/604.1", + "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/14.2 Mobile/15A372 Safari/604.1", + "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/14.2 Mobile/15A372 Safari/604.1", + "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/14.2 Mobile/15A372 Safari/604.1", + "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/14.2 Mobile/15A372 Safari/604.1", + "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/14.2 Mobile/15A372 Safari/604.1", + "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/14.2 Mobile/15A372 Safari/604.1", + "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/14.2 Mobile/15A372 Safari/604.1", + "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/14.2 Mobile/15A372 Safari/604.1", + "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/14.2 Mobile/15A372 Safari/604.1", + "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/14.2 Mobile/15A372 Safari/604.1", + "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/14.2 Mobile/15A372 Safari/604.1", + "Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_1 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) Version/14.2 Mobile/14E304 Safari/602.1", + "Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_1 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) Version/14.2 Mobile/14E304 Safari/602.1", + "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/14.2 Mobile/15A372 Safari/604.1", + "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/14.2 Mobile/15A372 Safari/604.1", + "Mozilla/5.0 (iPhone; CPU iPhone OS 12_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.2 Mobile/15E148 Safari/604.1", + "Mozilla/5.0 (iPhone; CPU iPhone OS 12_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.2 Mobile/15E148 Safari/604.1", + "Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.2 Mobile/15E148 Safari/604.1", + "Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.2 Mobile/15E148 Safari/604.1", + "Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.2 Mobile/15E148 Safari/604.1", + "Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.2 Mobile/15E148 Safari/604.1", + "Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.2 Mobile/15E148 Safari/604.1", + "Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.2 Mobile/15E148 Safari/604.1", + "Mozilla/5.0 (iPhone; CPU iPhone OS 14_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.2 Mobile/15E148 Safari/604.1", + "Mozilla/5.0 (iPhone; CPU iPhone OS 14_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.2 Mobile/15E148 Safari/604.1", + "Mozilla/5.0 (iPhone; CPU iPhone OS 14_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.2 Mobile/15E148 Safari/604.1", + "Mozilla/5.0 (iPhone; CPU iPhone OS 14_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.2 Mobile/15E148 Safari/604.1", + "Mozilla/5.0 (iPhone; CPU iPhone OS 14_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.2 Mobile/15E148 Safari/604.1", + "Mozilla/5.0 (iPhone; CPU iPhone OS 14_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.2 Mobile/15E148 Safari/604.1", + "Mozilla/5.0 (Mobile; LYF/F300B/LYF-F300B-001-01-15-130718-i;Android; rv:89.0 Gecko/48.0 Firefox/90.0 KAIOS/2.5", + "Mozilla/5.0 (Mobile; LYF/F300B/LYF-F300B-001-01-15-130718-i;Android; rv:89.0 Gecko/48.0 Firefox/90.0 KAIOS/2.5", + "Mozilla/5.0 (Linux; U; en-us; KFAPWI Build/JDQ39) AppleWebKit/535.19 (KHTML, like Gecko) Silk/3.13 Safari/535.19 Silk-Accelerated=true", + "Mozilla/5.0 (Linux; U; en-us; KFAPWI Build/JDQ39) AppleWebKit/535.19 (KHTML, like Gecko) Silk/3.13 Safari/535.19 Silk-Accelerated=true", + "Mozilla/5.0 (Linux; U; Android 4.4.2; en-us; LGMS323 Build/KOT49I.MS32310c) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/93.0.4576.0 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; U; Android 4.4.2; en-us; LGMS323 Build/KOT49I.MS32310c) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/93.0.4576.0 Mobile Safari/537.36", + "Mozilla/5.0 (Windows Phone 10.0; Android 4.2.1; Microsoft; Lumia 550) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4576.0 Mobile Safari/537.36 Edge/14.14263", + "Mozilla/5.0 (Windows Phone 10.0; Android 4.2.1; Microsoft; Lumia 550) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4576.0 Mobile Safari/537.36 Edge/14.14263", + "Mozilla/5.0 (Windows Phone 10.0; Android 4.2.1; Microsoft; Lumia 950) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4576.0 Mobile Safari/537.36 Edge/14.14263", + "Mozilla/5.0 (Windows Phone 10.0; Android 4.2.1; Microsoft; Lumia 950) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4576.0 Mobile Safari/537.36 Edge/14.14263", + "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 10 Build/MOB31T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4576.0 Safari/537.36", + "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 10 Build/MOB31T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4576.0 Safari/537.36", + "Mozilla/5.0 (Linux; Android 4.4.2; Nexus 4 Build/KOT49H) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4576.0 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 4.4.2; Nexus 4 Build/KOT49H) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4576.0 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4576.0 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4576.0 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 8.0.0; Nexus 5X Build/OPR4.170623.006) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4576.0 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 8.0.0; Nexus 5X Build/OPR4.170623.006) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4576.0 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 7.1.1; Nexus 6 Build/N6F26U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4576.0 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 7.1.1; Nexus 6 Build/N6F26U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4576.0 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 8.0.0; Nexus 6P Build/OPP3.170518.006) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4576.0 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 8.0.0; Nexus 6P Build/OPP3.170518.006) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4576.0 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 7 Build/MOB30X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4576.0 Safari/537.36", + "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 7 Build/MOB30X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4576.0 Safari/537.36", + "Mozilla/5.0 (compatible; MSIE 10.0; Windows Phone 8.0; Trident/6.0; IEMobile/10.0; ARM; Touch; NOKIA; Lumia 520)", + "Mozilla/5.0 (compatible; MSIE 10.0; Windows Phone 8.0; Trident/6.0; IEMobile/10.0; ARM; Touch; NOKIA; Lumia 520)", + "Mozilla/5.0 (MeeGo; NokiaN9) AppleWebKit/534.13 (KHTML, like Gecko) NokiaBrowser/8.5.0 Mobile Safari/534.13", + "Mozilla/5.0 (MeeGo; NokiaN9) AppleWebKit/534.13 (KHTML, like Gecko) NokiaBrowser/8.5.0 Mobile Safari/534.13", + "Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4576.0 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4576.0 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 8.0.0; Pixel 2 XL Build/OPD1.170816.004) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4576.0 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 8.0.0; Pixel 2 XL Build/OPD1.170816.004) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4576.0 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 9; Pixel 3 Build/PQ1A.181105.017.A1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4576.0 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 9; Pixel 3 Build/PQ1A.181105.017.A1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4576.0 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 10; Pixel 4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4576.0 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 10; Pixel 4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4576.0 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 11; Pixel 4a (5G)) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4576.0 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 11; Pixel 4a (5G)) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4576.0 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4576.0 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4576.0 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 7.0; Moto G (4)) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4576.0 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 7.0; Moto G (4)) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4576.0 Mobile Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4576.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4576.0 Safari/537.36 Edg/93.0.4576.0", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:90.0 Gecko/20100101 Firefox/90.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.2 Safari/605.1.15", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4576.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4576.0 Safari/537.36 Edg/93.0.4576.0", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:90.0 Gecko/20100101 Firefox/90.0", + ], +} -def get(): - return random.choice(USER_AGENTS) +def get(ua_type: str = None): + if not ua_type: + ua_type = random.choice(list(USER_AGENTS.keys())) + elif ua_type not in USER_AGENTS: + raise ValueError( + "ua_type error, expect one of {}".format(list(USER_AGENTS.keys())) + ) + + return random.choice(USER_AGENTS[ua_type]) diff --git a/feapder/network/user_pool/__init__.py b/feapder/network/user_pool/__init__.py new file mode 100644 index 00000000..9f14982b --- /dev/null +++ b/feapder/network/user_pool/__init__.py @@ -0,0 +1,13 @@ +__all__ = [ + "GuestUserPool", + "GuestUser", + "NormalUserPool", + "NormalUser", + "GoldUserPool", + "GoldUser", + "GoldUserStatus", +] + +from .gold_user_pool import GoldUserPool, GoldUser, GoldUserStatus +from .guest_user_pool import GuestUserPool, GuestUser +from .normal_user_pool import NormalUserPool, NormalUser diff --git a/feapder/network/user_pool/base_user_pool.py b/feapder/network/user_pool/base_user_pool.py new file mode 100644 index 00000000..631c3a63 --- /dev/null +++ b/feapder/network/user_pool/base_user_pool.py @@ -0,0 +1,233 @@ +import abc +import json +import random +import time +from datetime import datetime + +from feapder.db.redisdb import RedisDB +from feapder.utils.log import log +from feapder.utils.tools import get_md5, timestamp_to_date + + +class GuestUser: + def __init__(self, user_agent=None, proxies=None, cookies=None, **kwargs): + self.__dict__.update(kwargs) + self.user_agent = user_agent + self.proxies = proxies + self.cookies = cookies + self.user_id = kwargs.get("user_id") or get_md5(user_agent, proxies, cookies) + + def __str__(self): + return f"<{self.__class__.__name__}>: " + json.dumps( + self.to_dict(), indent=4, ensure_ascii=False + ) + + def __repr__(self): + return self.__str__() + + def to_dict(self): + data = {} + for key, value in self.__dict__.items(): + if value is not None: + data[key] = value + return data + + def from_dict(cls, data): + return cls.__init__(**data) + + +class NormalUser(GuestUser): + def __init__(self, username, password, **kwargs): + super().__init__(**kwargs) + self.username = username + self.password = password + self.user_id = kwargs.get("user_id") or self.username # 用户名作为user_id + + +class GoldUser(NormalUser): + """ + 昂贵的账号 + """ + + redisdb: RedisDB = None + redis_key: str = None + + def __init__( + self, + max_use_times, + use_interval=0, + work_time=(7, 23), + login_interval=30 * 60, + exclusive_time=None, + **kwargs, + ): + """ + @param max_use_times: + @param use_interval: 使用时间间隔。 支持元组 指定间隔的时间范围 如（5，10）即5到10秒；或直接传整数 + @param work_time: 工作时间，默认 7点到23点 + @param login_interval: 登录时间间隔 防止频繁登录 导致账号被封 + @param exclusive_time: 独占时长 + """ + super().__init__(**kwargs) + self.max_use_times = max_use_times + self.use_interval = use_interval + self.work_time = work_time + self.login_interval = login_interval + self.exclusive_time = exclusive_time or ( + use_interval[-1] * 5 + if isinstance(use_interval, (tuple, list)) + else use_interval * 5 + ) + + self._delay_use = kwargs.get("_delay_use", 0) # 延时使用，用于等待解封的用户 + self._login_time = kwargs.get("_login_time", 0) + self._use_times = kwargs.get("_use_times", 0) + self._last_use_time = kwargs.get("_last_use_time", 0) + self._used_for_spider_name = kwargs.get("_used_for_spider_name") + self._reset_use_times_date = kwargs.get("_reset_use_times_date") + + def __eq__(self, other): + return self.username == other.username + + def update(self, ohter): + self.__dict__.update(ohter.to_dict()) + + def sycn_to_redis(self): + self.redisdb.hset(self.redis_key, self.user_id, self.to_dict()) + + def set_delay_use(self, seconds): + self._delay_use = seconds + self.sycn_to_redis() + + def set_cookies(self, cookies): + self.cookies = cookies + self.sycn_to_redis() + + def set_login_time(self, _login_time=None): + self._login_time = _login_time or time.time() + self.sycn_to_redis() + + def get_login_time(self): + return self._login_time + + def get_last_use_time(self): + return self._last_use_time + + def get_used_for_spider_name(self): + return self._used_for_spider_name + + def set_used_for_spider_name(self, name): + self._used_for_spider_name = name + self._use_times += 1 + self._last_use_time = time.time() + self.sycn_to_redis() + + def is_time_to_login(self): + return time.time() - self.get_login_time() > self.login_interval + + def next_login_time(self): + return timestamp_to_date(int(self.login_interval + self.get_login_time())) + + def is_time_to_use(self): + if self._delay_use: + is_time = time.time() - self._last_use_time > self._delay_use + if is_time: + self._delay_use = 0 # 不用同步了，使用用户时会同步 + + else: + is_time = time.time() - self._last_use_time > ( + random.randint(*self.use_interval) + if isinstance(self.use_interval, (tuple, list)) + else self.use_interval + ) + + return is_time + + def reset_use_times(self): + self._use_times = 0 + self._reset_use_times_date = datetime.now().strftime("%Y-%m-%d") + self.sycn_to_redis() + + @property + def use_times(self): + current_date = datetime.now().strftime("%Y-%m-%d") + if current_date != self._reset_use_times_date: + self.reset_use_times() + + return self._use_times + + def is_overwork(self): + if self.use_times > self.max_use_times: + log.info("账号 {} 请求次数超限制".format(self.username)) + return True + + return False + + def is_at_work_time(self): + if datetime.now().hour in list(range(*self.work_time)): + return True + + log.info("账号 {} 不再工作时间内".format(self.username)) + return False + + +class UserPoolInterface(metaclass=abc.ABCMeta): + @abc.abstractmethod + def login(self, *args, **kwargs): + """ + 登录 生产cookie + Args: + *args: + **kwargs: + + Returns: + + """ + raise NotImplementedError + + @abc.abstractmethod + def add_user(self, *args, **kwargs): + """ + 将带有cookie的用户添加到用户池 + Args: + *args: + **kwargs: + + Returns: + + """ + raise NotImplementedError + + @abc.abstractmethod + def get_user(self, block=True): + """ + 获取用户使用 + Args: + block: 无用户时是否等待 + + Returns: + + """ + raise NotImplementedError + + @abc.abstractmethod + def del_user(self, *args, **kwargs): + """ + 删除用户 + Args: + *args: + **kwargs: + + Returns: + + """ + raise NotImplementedError + + @abc.abstractmethod + def run(self): + """ + 维护一定数量的用户 + Returns: + + """ + raise NotImplementedError diff --git a/feapder/network/user_pool/gold_user_pool.py b/feapder/network/user_pool/gold_user_pool.py new file mode 100644 index 00000000..8512a295 --- /dev/null +++ b/feapder/network/user_pool/gold_user_pool.py @@ -0,0 +1,291 @@ +# -*- coding: utf-8 -*- +""" +Created on 2018/12/27 11:32 AM +--------- +@summary: 账号昂贵、限制查询次数及使用时间的用户UserPool +--------- +@author: Boris +@email: boris_liu@foxmail.com +""" + +import os +import random +import time +from enum import Enum, unique +from typing import Optional, List + +from feapder import setting +from feapder.db.redisdb import RedisDB +from feapder.network.user_pool.base_user_pool import GoldUser, UserPoolInterface +from feapder.utils import metrics +from feapder.utils.log import log +from feapder.utils.redis_lock import RedisLock +from feapder.utils.tools import send_msg + + +@unique +class GoldUserStatus(Enum): + # 使用状态 + USED = "used" + SUCCESS = "success" + OVERDUE = "overdue" # cookie 过期 + SLEEP = "sleep" + EXCEPTION = "exception" + # 登陆状态 + LOGIN_SUCCESS = "login_success" + LOGIN_FALIED = "login_failed" + + +class GoldUserPool(UserPoolInterface): + """ + 账号昂贵、限制查询次数的用户的UserPool + """ + + def __init__( + self, + redis_key, + *, + users: List[GoldUser], + keep_alive=False, + ): + """ + @param redis_key: user存放在redis中的key前缀 + @param users: 账号信息 + @param keep_alive: 是否保持常驻，以便user不足时立即补充 + """ + self._tab_user_pool = setting.TAB_USER_POOL.format( + redis_key=redis_key, user_type="gold" + ) + + self.users = users + self._keep_alive = keep_alive + + self._redisdb = RedisDB() + self._users_id = [] + + if not users: + raise ValueError("not users") + + # 给user的类属性复制 + self.users[0].__class__.redisdb = self._redisdb + self.users[0].__class__.redis_key = self._tab_user_pool + + self.__init_metrics() + self.__sync_users_base_info() + self.__sycn_users_info() + + def __init_metrics(self): + metrics.init(**setting.METRICS_OTHER_ARGS) + + def __sync_users_base_info(self): + # 本地同步基本信息到redis, 注 只能在初始化函数内同步 + for user in self.users: + cache_user = self.get_user_by_id(user.user_id) + if cache_user: + for key, value in user.to_dict().items(): + if not key.startswith("_"): + setattr(cache_user, key, value) + cache_user.sycn_to_redis() + + def __sycn_users_info(self): + # redis同步登录信息到本地 + for index, user in enumerate(self.users): + cache_user = self.get_user_by_id(user.user_id) + if cache_user: + self.users[index] = cache_user + + def _load_users_id(self): + self._users_id = self._redisdb.hkeys(self._tab_user_pool) + if self._users_id: + random.shuffle(self._users_id) + + def _get_user_id(self): + if not self._users_id: + self._load_users_id() + + if self._users_id: + return self._users_id.pop() + + def login(self, user: GoldUser) -> GoldUser: + """ + 登录 生产cookie + """ + raise NotImplementedError + + def get_user_by_id(self, user_id: str) -> GoldUser: + user_str = self._redisdb.hget(self._tab_user_pool, user_id) + if user_str: + user = GoldUser(**eval(user_str)) + return user + + def get_user( + self, + block=True, + username=None, + used_for_spider_name=None, + not_limit_use_interval=False, + ) -> Optional[GoldUser]: + """ + @params username: 获取指定的用户 + @params used_for_spider_name: 独享式使用，独享爬虫的名字。其他爬虫不可抢占 + @params block: 无用户时是否等待 + @params not_limit_frequence: 不限制使用频率 + @return: GoldUser + """ + while True: + try: + user_id = username or self._get_user_id() + user_str = None + if user_id: + user_str = self._redisdb.hget(self._tab_user_pool, user_id) + + if (not user_id or not user_str) and block: + self._keep_alive = False + self.run(username) + continue + + # 取到用户 + user = GoldUser(**eval(user_str)) + + # 独占式使用，若为其他爬虫，检查等待使用时间是否超过独占时间，若超过则可以使用 + if ( + user.get_used_for_spider_name() + and user.get_used_for_spider_name() != used_for_spider_name + ): + wait_time = time.time() - user.get_last_use_time() + if wait_time < user.exclusive_time: + log.info( + "用户{} 被 {} 爬虫独占，需等待 {} 秒后才可使用".format( + user.username, + user.get_used_for_spider_name(), + user.exclusive_time - wait_time, + ) + ) + time.sleep(1) + continue + + if not user.is_overwork() and user.is_at_work_time(): + if not user.cookies: + log.debug(f"用户 {user.username} 未登录，尝试登录") + self._keep_alive = False + self.run(username) + continue + + if not_limit_use_interval or user.is_time_to_use(): + user.set_used_for_spider_name(used_for_spider_name) + log.debug("使用用户 {}".format(user.username)) + self.record_user_status(user.user_id, GoldUserStatus.USED) + return user + else: + log.debug("{} 用户使用间隔过短 查看下一个用户".format(user.username)) + time.sleep(1) + continue + else: + if not user.is_at_work_time(): + log.info("用户 {} 不在工作时间 sleep 60s".format(user.username)) + if block: + time.sleep(60) + continue + else: + return None + + except Exception as e: + log.exception(e) + time.sleep(1) + + def del_user(self, user_id: str): + user = self.get_user_by_id(user_id) + if user: + user.set_cookies(None) + self.record_user_status(user.user_id, GoldUserStatus.OVERDUE) + + def add_user(self, user: GoldUser): + user.sycn_to_redis() + + def delay_use(self, user_id: str, delay_seconds: int): + user = self.get_user_by_id(user_id) + if user: + user.set_delay_use(delay_seconds) + + self.record_user_status(user_id, GoldUserStatus.SLEEP) + + def record_success_user(self, user_id: str): + self.record_user_status(user_id, GoldUserStatus.SUCCESS) + + def record_exception_user(self, user_id: str): + self.record_user_status(user_id, GoldUserStatus.EXCEPTION) + + def run(self, username=None): + while True: + try: + with RedisLock( + key=self._tab_user_pool, lock_timeout=3600, wait_timeout=0 + ) as _lock: + if _lock.locked: + self.__sycn_users_info() + online_user = 0 + for user in self.users: + if username and username != user.username: + continue + + try: + if user.cookies: + online_user += 1 + continue + + # 预检查 + if not user.is_time_to_login(): + log.info( + "账号{}与上次登录时间间隔过短，暂不登录: 将在{}登录使用".format( + user.username, user.next_login_time() + ) + ) + continue + + user = self.login(user) + if user.cookies: + # 保存cookie + user.set_login_time() + self.add_user(user) + self.record_user_status( + user.user_id, GoldUserStatus.LOGIN_SUCCESS + ) + log.debug("登录成功 {}".format(user.username)) + online_user += 1 + else: + log.info("登录失败 {}".format(user.username)) + self.record_user_status( + user.user_id, GoldUserStatus.LOGIN_FALIED + ) + except NotImplementedError: + log.error( + f"{self.__class__.__name__} must be implementation login method！" + ) + os._exit(0) + except Exception as e: + log.exception(e) + msg = f"{user.username} 账号登陆失败 exception: {str(e)}" + log.info(msg) + self.record_user_status( + user.user_id, GoldUserStatus.LOGIN_FALIED + ) + + send_msg( + msg=msg, + level="error", + message_prefix=f"{user.username} 账号登陆失败", + ) + + log.info("当前在线user数为 {}".format(online_user)) + + if self._keep_alive: + time.sleep(10) + else: + break + + except Exception as e: + log.exception(e) + time.sleep(1) + + def record_user_status(self, user_id: str, status: GoldUserStatus): + metrics.emit_counter(user_id, 1, classify=f"users_{status.value}") diff --git a/feapder/network/user_pool/guest_user_pool.py b/feapder/network/user_pool/guest_user_pool.py new file mode 100644 index 00000000..9d34aad3 --- /dev/null +++ b/feapder/network/user_pool/guest_user_pool.py @@ -0,0 +1,168 @@ +# -*- coding: utf-8 -*- +""" +Created on 2018/12/27 11:32 AM +--------- +@summary: 访客用户池 不需要登陆 +--------- +@author: Boris +@email: boris_liu@foxmail.com +""" + +import random +from typing import Optional + +import feapder.utils.tools as tools +from feapder import setting +from feapder.db.redisdb import RedisDB +from feapder.network.user_pool.base_user_pool import UserPoolInterface, GuestUser +from feapder.utils.log import log +from feapder.utils.webdriver import WebDriver + + +class GuestUserPool(UserPoolInterface): + """ + 访客用户池 不需要登陆 + """ + + def __init__( + self, + redis_key, + page_url=None, + min_users=1, + must_contained_keys=(), + keep_alive=False, + **kwargs, + ): + """ + @param redis_key: user存放在redis中的key前缀 + @param page_url: 生产user的url + @param min_users: 最小user数 + @param must_contained_keys: cookie中必须包含的key，用于校验cookie是否正确 + @param keep_alive: 是否保持常驻，以便user不足时立即补充 + --- + @param kwargs: WebDriver的一些参数 + load_images: 是否加载图片 + user_agent: 字符串 或 无参函数，返回值为user_agent + proxy: xxx.xxx.xxx.xxx:xxxx 或 无参函数，返回值为代理地址 + headless: 是否启用无头模式 + driver_type: CHROME,EDGE 或 PHANTOMJS,FIREFOX + timeout: 请求超时时间 + window_size: # 窗口大小 + executable_path: 浏览器路径，默认为默认路径 + """ + + self._redisdb = RedisDB() + + self._tab_user_pool = setting.TAB_USER_POOL.format( + redis_key=redis_key, user_type="guest" + ) + self._page_url = page_url + self._min_users = min_users + self._must_contained_keys = must_contained_keys + self._keep_alive = keep_alive + + self._kwargs = kwargs + self._kwargs.setdefault("load_images", False) + self._kwargs.setdefault("headless", True) + + self._users_id = [] + + def _load_users_id(self): + self._users_id = self._redisdb.hkeys(self._tab_user_pool) + if self._users_id: + random.shuffle(self._users_id) + + def _get_user_id(self): + if not self._users_id: + self._load_users_id() + + if self._users_id: + return self._users_id.pop() + + def login(self) -> Optional[GuestUser]: + """ + 默认使用webdirver去登录，生产cookie，可以重写 + """ + with WebDriver(**self._kwargs) as driver: + driver.get(self._page_url) + + cookies = driver.cookies + + for key in self._must_contained_keys: + if key not in cookies: + break + else: + user = GuestUser(user_agent=driver.user_agent, cookies=cookies) + return user + + log.error("获取cookie失败 cookies = {}".format(cookies)) + return None + + def add_user(self, user: GuestUser): + log.debug("add {}".format(user)) + self._redisdb.hset(self._tab_user_pool, user.user_id, user.to_dict()) + + def get_user(self, block=True) -> Optional[GuestUser]: + """ + + Args: + block: 无用户时是否等待 + + Returns: + + """ + while True: + try: + user_id = self._get_user_id() + user_str = None + if user_id: + user_str = self._redisdb.hget(self._tab_user_pool, user_id) + # 如果没取到user，可能是其他爬虫将此用户删除了，需要重刷新本地缓存的用户id + if not user_str: + self._load_users_id() + continue + + if not user_id and block: + self._keep_alive = False + self._min_users = 1 + self.run() + continue + + return user_str and GuestUser(**eval(user_str)) + except Exception as e: + log.exception(e) + tools.delay_time(1) + + def del_user(self, user_id: str): + self._redisdb.hdel(self._tab_user_pool, user_id) + self._load_users_id() + + def run(self): + while True: + try: + now_user_count = self._redisdb.hget_count(self._tab_user_pool) + need_user_count = self._min_users - now_user_count + + if need_user_count > 0: + log.info( + "当前在线user数为 {} 小于 {}, 生产user".format( + now_user_count, self._min_users + ) + ) + try: + user = self.login() + if user: + self.add_user(user) + except Exception as e: + log.exception(e) + else: + log.debug("当前user数为 {} 数量足够 暂不生产".format(now_user_count)) + + if self._keep_alive: + tools.delay_time(10) + else: + break + + except Exception as e: + log.exception(e) + tools.delay_time(1) diff --git a/feapder/network/user_pool/normal_user_pool.py b/feapder/network/user_pool/normal_user_pool.py new file mode 100644 index 00000000..63c99726 --- /dev/null +++ b/feapder/network/user_pool/normal_user_pool.py @@ -0,0 +1,247 @@ +# -*- coding: utf-8 -*- +""" +Created on 2018/12/27 11:32 AM +--------- +@summary: 普通用户池，适用于账号成本低且大量的场景 +--------- +@author: Boris +@email: boris_liu@foxmail.com +""" + +import os +import random +from typing import Iterable, Optional + +import feapder.utils.tools as tools +from feapder import setting +from feapder.db.mysqldb import MysqlDB +from feapder.db.redisdb import RedisDB +from feapder.network.user_pool.base_user_pool import UserPoolInterface, NormalUser +from feapder.utils.log import log +from feapder.utils.redis_lock import RedisLock + + +class NormalUserPool(UserPoolInterface): + """ + 普通用户池，适用于账号成本低且大量的场景 + """ + + def __init__( + self, + redis_key, + *, + table_userbase, + login_state_key="login_state", + lock_state_key="lock_state", + username_key="username", + password_key="password", + login_retry_times=1, + keep_alive=False, + ): + """ + @param redis_key: 项目名 + @param table_userbase: 用户表名 + @param login_state_key: 登录状态列名 + @param lock_state_key: 封锁状态列名 + @param username_key: 登陆名列名 + @param password_key: 密码列名 + @param login_retry_times: 登陆失败重试次数 + @param keep_alive: 是否保持常驻，以便user不足时立即补充 + """ + + self._tab_user_pool = setting.TAB_USER_POOL.format( + redis_key=redis_key, user_type="normal" + ) + + self._login_retry_times = login_retry_times + self._table_userbase = table_userbase + self._login_state_key = login_state_key + self._lock_state_key = lock_state_key + self._username_key = username_key + self._password_key = password_key + self._keep_alive = keep_alive + + self._users_id = [] + + self._redisdb = RedisDB() + self._mysqldb = MysqlDB() + + self._create_userbase() + + def _load_users_id(self): + self._users_id = self._redisdb.hkeys(self._tab_user_pool) + if self._users_id: + random.shuffle(self._users_id) + + def _get_user_id(self): + if not self._users_id: + self._load_users_id() + + if self._users_id: + return self._users_id.pop() + + def _create_userbase(self): + sql = f""" + CREATE TABLE IF NOT EXISTS `{self._table_userbase}` ( + `id` int(10) unsigned NOT NULL AUTO_INCREMENT, + `{self._username_key}` varchar(50) DEFAULT NULL COMMENT '用户名', + `{self._password_key}` varchar(255) DEFAULT NULL COMMENT '密码', + `{self._login_state_key}` int(11) DEFAULT '0' COMMENT '登录状态（0未登录 1已登录）', + `{self._lock_state_key}` int(11) DEFAULT '0' COMMENT '账号是否被封（0 未封 1 被封）', + PRIMARY KEY (`id`), + UNIQUE KEY `username` (`username`) USING BTREE + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + """ + self._mysqldb.execute(sql) + + def _load_user(self) -> Iterable[NormalUser]: + """ + 返回用户信息 + @return: yield username, password + """ + + sql = "select id, {username_key}, {password_key} from {table_userbase} where {lock_state_key} != 1 and {login_state_key} != 1".format( + username_key=self._username_key, + password_key=self._password_key, + table_userbase=self._table_userbase, + lock_state_key=self._lock_state_key, + login_state_key=self._login_state_key, + ) + + for id, username, password in self._mysqldb.find(sql): + yield NormalUser(user_id=id, username=username, password=password) + + def handle_login_failed_user(self, user: NormalUser): + """ + 处理登录失败的user + @return: + """ + + pass + + def handel_exception(self, e: Exception): + """ + 处理异常 + @param e: + @return: + """ + log.exception(e) + + def login(self, user: NormalUser) -> NormalUser: + """ + 登录 生产cookie + """ + raise NotImplementedError + + def add_user(self, user: NormalUser): + log.debug("add {}".format(user)) + self._redisdb.hset(self._tab_user_pool, user.user_id, user.to_dict()) + + sql = "update {table_userbase} set {login_state_key} = 1 where id = {user_id}".format( + table_userbase=self._table_userbase, + login_state_key=self._login_state_key, + username_key=self._username_key, + user_id=user.user_id, + ) + self._mysqldb.update(sql) + + def get_user(self, block=True) -> Optional[NormalUser]: + while True: + try: + user_id = self._get_user_id() + user_str = None + if user_id: + user_str = self._redisdb.hget(self._tab_user_pool, user_id) + # 如果没取到user，可能是其他爬虫将此用户删除了，需要重刷新本地缓存的用户id + if not user_str: + self._load_users_id() + continue + + if not user_id and block: + self._keep_alive = False + self.run() + continue + + return user_str and NormalUser(**eval(user_str)) + except Exception as e: + log.exception(e) + tools.delay_time(1) + + def del_user(self, user_id: int): + """ + 删除失效的user + @return: + """ + self._redisdb.hdel(self._tab_user_pool, user_id) + self._load_users_id() + + sql = "update {table_userbase} set {login_state_key} = 0 where id = {user_id}".format( + table_userbase=self._table_userbase, + login_state_key=self._login_state_key, + username_key=self._username_key, + user_id=user_id, + ) + + self._mysqldb.update(sql) + + def tag_user_locked(self, user_id: int): + """ + 标记用户被封堵 + """ + sql = "update {table_userbase} set {lock_state_key} = 1 where id = {user_id}".format( + table_userbase=self._table_userbase, + lock_state_key=self._lock_state_key, + username_key=self._username_key, + user_id=user_id, + ) + + self._mysqldb.update(sql) + + def run(self): + while True: + try: + try: + with RedisLock( + key=self._tab_user_pool, lock_timeout=3600, wait_timeout=0 + ) as _lock: + if _lock.locked: + for user in self._load_user(): + retry_times = 0 + while retry_times <= self._login_retry_times: + try: + login_user = self.login(user) + if login_user: + self.add_user(login_user) + else: + self.handle_login_failed_user(user) + break + except NotImplementedError: + log.error( + f"{self.__class__.__name__} must be implementation login method！" + ) + os._exit(0) + except Exception as e: + self.handel_exception(e) + log.debug( + f"login failed, user: {user} retry_times: {retry_times}" + ) + retry_times += 1 + else: + self.handle_login_failed_user(user) + + now_user_count = self._redisdb.hget_count( + self._tab_user_pool + ) + log.info("当前在线user数为 {}".format(now_user_count)) + + except Exception as e: + log.exception(e) + + if self._keep_alive: + tools.delay_time(10) + else: + break + + except Exception as e: + log.exception(e) + tools.delay_time(1) diff --git a/feapder/pipelines/__init__.py b/feapder/pipelines/__init__.py new file mode 100644 index 00000000..dcee959e --- /dev/null +++ b/feapder/pipelines/__init__.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- +""" +Created on 2021/3/17 10:57 下午 +--------- +@summary: +--------- +@author: Boris +@email: boris_liu@foxmail.com +""" + +import abc +from typing import Dict, List, Tuple + + +class BasePipeline(metaclass=abc.ABCMeta): + """ + pipeline 是单线程的，批量保存数据的操作，不建议在这里写网络请求代码，如下载图片等 + """ + + @abc.abstractmethod + def save_items(self, table, items: List[Dict]) -> bool: + """ + 保存数据 + Args: + table: 表名 + items: 数据，[{},{},...] + + Returns: 是否保存成功 True / False + 若False，不会将本批数据入到去重库，以便再次入库 + + """ + + return True + + def update_items(self, table, items: List[Dict], update_keys=Tuple) -> bool: + """ + 更新数据, 与UpdateItem配合使用，若爬虫中没使用UpdateItem，则可不实现此接口 + Args: + table: 表名 + items: 数据，[{},{},...] + update_keys: 更新的字段, 如 ("title", "publish_time") + + Returns: 是否更新成功 True / False + 若False，不会将本批数据入到去重库，以便再次入库 + + """ + + return True + + def close(self): + """ + 关闭，爬虫结束时调用 + Returns: + + """ + pass diff --git a/feapder/pipelines/console_pipeline.py b/feapder/pipelines/console_pipeline.py new file mode 100644 index 00000000..1ebb532e --- /dev/null +++ b/feapder/pipelines/console_pipeline.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- +""" +Created on 2021/3/18 12:39 上午 +--------- +@summary: +--------- +@author: Boris +@email: boris_liu@foxmail.com +""" + +from feapder.pipelines import BasePipeline +from typing import Dict, List, Tuple +from feapder.utils.log import log + + +class ConsolePipeline(BasePipeline): + """ + pipeline 是单线程的，批量保存数据的操作，不建议在这里写网络请求代码，如下载图片等 + """ + + def save_items(self, table, items: List[Dict]) -> bool: + """ + 保存数据 + Args: + table: 表名 + items: 数据，[{},{},...] + + Returns: 是否保存成功 True / False + 若False，不会将本批数据入到去重库，以便再次入库 + + """ + log.info("【调试输出】共导出 %s 条数据 到 %s" % (len(items), table)) + return True + + def update_items(self, table, items: List[Dict], update_keys=Tuple) -> bool: + """ + 更新数据 + Args: + table: 表名 + items: 数据，[{},{},...] + update_keys: 更新的字段, 如 ("title", "publish_time") + + Returns: 是否更新成功 True / False + 若False，不会将本批数据入到去重库，以便再次入库 + + """ + log.info("【调试输出】共导出 %s 条数据 到 %s" % (len(items), table)) + return True diff --git a/feapder/pipelines/csv_pipeline.py b/feapder/pipelines/csv_pipeline.py new file mode 100644 index 00000000..922a77d3 --- /dev/null +++ b/feapder/pipelines/csv_pipeline.py @@ -0,0 +1,254 @@ +# -*- coding: utf-8 -*- +""" +Created on 2025-10-16 +--------- +@summary: CSV 数据导出Pipeline +--------- +@author: 道长 +@email: ctrlf4@yeah.net +""" + +import csv +import os +import threading +from typing import Dict, List, Tuple + +from feapder.pipelines import BasePipeline +from feapder.utils.log import log + + +class CsvPipeline(BasePipeline): + """ + CSV 数据导出Pipeline + + 将爬虫数据保存为CSV文件。支持批量保存、并发写入控制、断点续爬等功能。 + + 特点： + - 单表单锁设计，避免全局锁带来的性能问题 + - 自动创建导出目录 + - 支持追加模式，便于断点续爬 + - 通过fsync确保数据落盘 + - 表级别的字段名缓存，确保跨批字段顺序一致 + """ + + # 用于保护每个表的文件写入操作（Per-Table Lock） + _file_locks = {} + + # 用于缓存每个表的字段名顺序（Per-Table Fieldnames Cache） + # 确保跨批次、跨线程的字段顺序一致 + _table_fieldnames = {} + + def __init__(self, csv_dir=None): + """ + 初始化CSV Pipeline + + Args: + csv_dir: CSV文件保存目录 + - 如果不传，从 setting.CSV_EXPORT_PATH 读取 + - 支持相对路径（如 "data/csv"） + - 支持绝对路径（如 "/Users/xxx/exports/csv"） + """ + super().__init__() + + # 如果未传入参数，从配置文件读取 + if csv_dir is None: + import feapder.setting as setting + csv_dir = setting.CSV_EXPORT_PATH + + # 支持绝对路径和相对路径，统一转换为绝对路径 + self.csv_dir = os.path.abspath(csv_dir) + self._ensure_csv_dir_exists() + + def _ensure_csv_dir_exists(self): + """确保CSV保存目录存在""" + if not os.path.exists(self.csv_dir): + try: + os.makedirs(self.csv_dir, exist_ok=True) + log.info(f"创建CSV保存目录: {self.csv_dir}") + except Exception as e: + log.error(f"创建CSV目录失败: {e}") + raise + + @staticmethod + def _get_lock(table): + """ + 获取表对应的文件锁 + + 采用Per-Table Lock设计，每个表都有独立的锁，避免锁竞争。 + 这样设计既能保证单表的文件写入安全，又能充分利用多表并行写入的优势。 + + Args: + table: 表名 + + Returns: + threading.Lock: 该表对应的锁对象 + """ + if table not in CsvPipeline._file_locks: + CsvPipeline._file_locks[table] = threading.Lock() + return CsvPipeline._file_locks[table] + + @staticmethod + def _get_and_cache_fieldnames(table, items): + """ + 获取并缓存表对应的字段名顺序 + + 第一次调用时从items[0]提取字段名并缓存，后续调用直接返回缓存的字段名。 + 这样设计确保： + 1. 跨批次的字段顺序保持一致（解决数据列错位问题） + 2. 多线程并发时字段顺序不被污染 + 3. 避免重复提取，性能更优 + + Args: + table: 表名 + items: 数据列表 [{}，{}，...] + + Returns: + list: 字段名列表 + """ + # 如果该表已经缓存了字段名，直接返回缓存的 + if table in CsvPipeline._table_fieldnames: + return CsvPipeline._table_fieldnames[table] + + # 第一次调用，从items提取字段名并缓存 + if not items: + return [] + + first_item = items[0] + fieldnames = list(first_item.keys()) if isinstance(first_item, dict) else [] + + if fieldnames: + # 缓存字段名（使用静态变量，跨实例共享） + CsvPipeline._table_fieldnames[table] = fieldnames + log.info(f"表 {table} 的字段名已缓存: {fieldnames}") + + return fieldnames + + def _get_csv_file_path(self, table): + """ + 获取表对应的CSV文件路径 + + Args: + table: 表名 + + Returns: + str: CSV文件的完整路径 + """ + return os.path.join(self.csv_dir, f"{table}.csv") + + + def _file_exists_and_has_content(self, csv_file): + """ + 检查CSV文件是否存在且有内容 + + Args: + csv_file: CSV文件路径 + + Returns: + bool: 文件存在且有内容返回True + """ + return os.path.exists(csv_file) and os.path.getsize(csv_file) > 0 + + def save_items(self, table, items: List[Dict]) -> bool: + """ + 保存数据到CSV文件 + + 采用追加模式打开文件，支持断点续爬。第一次写入时会自动添加表头。 + 使用Per-Table Lock确保多线程写入时的数据一致性。 + 使用缓存的字段名确保跨批次字段顺序一致，避免数据列错位。 + + Args: + table: 表名（对应CSV文件名） + items: 数据列表，[{}, {}, ...] + + Returns: + bool: 保存成功返回True，失败返回False + 失败时ItemBuffer会自动重试（最多10次） + """ + if not items: + return True + + csv_file = self._get_csv_file_path(table) + + # 使用缓存机制获取字段名（关键！确保跨批字段顺序一致） + fieldnames = self._get_and_cache_fieldnames(table, items) + + if not fieldnames: + log.warning(f"无法提取字段名，items: {items}") + return False + + try: + # 获取表级别的锁（关键！保证文件写入安全） + lock = self._get_lock(table) + with lock: + # 检查文件是否已存在且有内容 + file_exists = self._file_exists_and_has_content(csv_file) + + # 以追加模式打开文件 + with open( + csv_file, + "a", + encoding="utf-8", + newline="" + ) as f: + writer = csv.DictWriter(f, fieldnames=fieldnames) + + # 如果文件不存在或为空，写入表头 + if not file_exists: + writer.writeheader() + + # 批量写入数据行 + # 使用缓存的fieldnames确保列顺序一致，避免跨批数据错位 + writer.writerows(items) + + # 刷新缓冲区到磁盘，确保数据不丢失 + f.flush() + os.fsync(f.fileno()) + + # 记录导出日志 + log.info( + f"共导出 {len(items)} 条数据 到 {table}.csv (文件路径: {csv_file})" + ) + return True + + except Exception as e: + log.error( + f"CSV写入失败. table: {table}, csv_file: {csv_file}, error: {e}" + ) + return False + + def update_items(self, table, items: List[Dict], update_keys=Tuple) -> bool: + """ + 更新数据 + + 注意：CSV文件本身不支持真正的"更新"操作（需要查询后替换）。 + 目前的实现是直接追加写入，相当于INSERT操作。 + + 如果需要真正的UPDATE操作，建议： + 1. 定期重新生成CSV文件 + 2. 使用数据库（MySQL/MongoDB）来处理UPDATE + 3. 或在应用层进行去重和更新 + + Args: + table: 表名 + items: 数据列表，[{}, {}, ...] + update_keys: 更新的字段（此实现中未使用） + + Returns: + bool: 操作成功返回True + """ + # 对于CSV，update操作实现为追加写入 + # 若需要真正的UPDATE操作，建议在应用层处理 + return self.save_items(table, items) + + def close(self): + """ + 关闭Pipeline，释放资源 + + 在爬虫结束时由ItemBuffer自动调用。 + """ + try: + # 清理文件锁字典（可选，用于释放内存） + # 在长期运行的场景下，可能需要定期清理 + pass + except Exception as e: + log.error(f"关闭CSV Pipeline时出错: {e}") diff --git a/feapder/pipelines/mongo_pipeline.py b/feapder/pipelines/mongo_pipeline.py new file mode 100644 index 00000000..253ebabc --- /dev/null +++ b/feapder/pipelines/mongo_pipeline.py @@ -0,0 +1,84 @@ +# -*- coding: utf-8 -*- +""" +Created on 2021-04-18 14:12:21 +--------- +@summary: 导出数据 +--------- +@author: Mkdir700 +@email: mkdir700@gmail.com +""" +from typing import Dict, List, Tuple + +from feapder.db.mongodb import MongoDB +from feapder.pipelines import BasePipeline +from feapder.utils.log import log + + +class MongoPipeline(BasePipeline): + def __init__(self): + self._to_db = None + + @property + def to_db(self): + if not self._to_db: + self._to_db = MongoDB() + + return self._to_db + + def save_items(self, table, items: List[Dict]) -> bool: + """ + 保存数据 + Args: + table: 表名 + items: 数据，[{},{},...] + + Returns: 是否保存成功 True / False + 若False，不会将本批数据入到去重库，以便再次入库 + + """ + try: + add_count = self.to_db.add_batch(coll_name=table, datas=items) + datas_size = len(items) + log.info( + "共导出 %s 条数据到 %s, 新增 %s条, 重复 %s 条" + % (datas_size, table, add_count, datas_size - add_count) + ) + return True + except Exception as e: + log.exception(e) + return False + + def update_items(self, table, items: List[Dict], update_keys=Tuple) -> bool: + """ + 更新数据 + Args: + table: 表名 + items: 数据，[{},{},...] + update_keys: 更新的字段, 如 ("title", "publish_time") + + Returns: 是否更新成功 True / False + 若False，不会将本批数据入到去重库，以便再次入库 + + """ + try: + add_count = self.to_db.add_batch( + coll_name=table, + datas=items, + update_columns=update_keys or list(items[0].keys()), + ) + datas_size = len(items) + update_count = datas_size - add_count + msg = "共导出 %s 条数据到 %s, 新增 %s 条, 更新 %s 条" % ( + datas_size, + table, + add_count, + update_count, + ) + if update_keys: + msg += " 更新字段为 {}".format(update_keys) + log.info(msg) + + return True + except Exception as e: + log.exception(e) + return False diff --git a/feapder/pipelines/mysql_pipeline.py b/feapder/pipelines/mysql_pipeline.py new file mode 100644 index 00000000..3ffb3fc1 --- /dev/null +++ b/feapder/pipelines/mysql_pipeline.py @@ -0,0 +1,76 @@ +# -*- coding: utf-8 -*- +""" +Created on 2018-07-29 22:48:30 +--------- +@summary: 导出数据 +--------- +@author: Boris +@email: boris_liu@foxmail.com +""" +from typing import Dict, List, Tuple + +import feapder.utils.tools as tools +from feapder.db.mysqldb import MysqlDB +from feapder.pipelines import BasePipeline +from feapder.utils.log import log + + +class MysqlPipeline(BasePipeline): + def __init__(self): + self._to_db = None + + @property + def to_db(self): + if not self._to_db: + self._to_db = MysqlDB() + + return self._to_db + + def save_items(self, table, items: List[Dict]) -> bool: + """ + 保存数据 + Args: + table: 表名 + items: 数据，[{},{},...] + + Returns: 是否保存成功 True / False + 若False，不会将本批数据入到去重库，以便再次入库 + + """ + + sql, datas = tools.make_batch_sql(table, items) + add_count = self.to_db.add_batch(sql, datas) + datas_size = len(datas) + if add_count: + log.info( + "共导出 %s 条数据 到 %s, 重复 %s 条" % (datas_size, table, datas_size - add_count) + ) + else: + log.debug("没有插入数据，可能全部重复") + + return add_count != None + + def update_items(self, table, items: List[Dict], update_keys=Tuple) -> bool: + """ + 更新数据 + Args: + table: 表名 + items: 数据，[{},{},...] + update_keys: 更新的字段, 如 ("title", "publish_time") + + Returns: 是否更新成功 True / False + 若False，不会将本批数据入到去重库，以便再次入库 + + """ + + sql, datas = tools.make_batch_sql( + table, items, update_columns=update_keys or list(items[0].keys()) + ) + update_count = self.to_db.add_batch(sql, datas) + if update_count: + msg = "共更新 %s 条数据 到 %s" % (update_count // 2, table) + if update_keys: + msg += " 更新字段为 {}".format(update_keys) + log.info(msg) + + return update_count != None diff --git a/feapder/requirements.txt b/feapder/requirements.txt index da1a43bf..888d27f2 100644 --- a/feapder/requirements.txt +++ b/feapder/requirements.txt @@ -1,12 +1,20 @@ better-exceptions>=0.2.2 -DBUtils>=2.0 -parsel>=1.5.2 +DBUtils>=3.0 +parsel>=1.8.1 PyExecJS>=1.5.1 -PyMySQL>=0.9.3 -redis>=2.10.6 -requests>=2.22.0 -bs4>=0.0.1 -ipython>=7.14.0 -bitarray>=1.5.3 -redis-py-cluster>=1.3.4 -cryptography>=3.3.2 \ No newline at end of file +pymongo>=4.0.0 +PyMySQL>=1.1.0 +redis>=5.0.0,<9.0.0 +requests>=2.31.0 +selenium>=4.10.0 +beautifulsoup4>=4.12.0 +ipython>=8.0.0 +bitarray>=2.8.0 +cryptography>=41.0.0 +urllib3>=2.0.0,<3.0.0 +loguru>=0.5.3 +influxdb>=5.3.1 +pyperclip>=1.8.2 +webdriver-manager>=4.0.0 +terminal-layout>=2.1.3 +playwright>=1.40.0 diff --git a/feapder/setting.py b/feapder/setting.py index 641f3c78..c52b318c 100644 --- a/feapder/setting.py +++ b/feapder/setting.py @@ -4,15 +4,15 @@ # redis 表名 # 任务表模版 -TAB_REQUSETS = "{redis_key}:z_requsets" +TAB_REQUESTS = "{redis_key}:z_requests" # 任务失败模板 -TAB_FAILED_REQUSETS = "{redis_key}:z_failed_requsets" +TAB_FAILED_REQUESTS = "{redis_key}:z_failed_requests" +# 数据保存失败模板 +TAB_FAILED_ITEMS = "{redis_key}:s_failed_items" # 爬虫状态表模版 -TAB_SPIDER_STATUS = "{redis_key}:z_spider_status" -# item 表模版 -TAB_ITEM = "{redis_key}:s_{item_name}" -# 爬虫时间记录表 -TAB_SPIDER_TIME = "{redis_key}:h_spider_time" +TAB_SPIDER_STATUS = "{redis_key}:h_spider_status" +# 用户池 +TAB_USER_POOL = "{redis_key}:h_{user_type}_pool" # MYSQL MYSQL_IP = os.getenv("MYSQL_IP") @@ -21,83 +21,221 @@ MYSQL_USER_NAME = os.getenv("MYSQL_USER_NAME") MYSQL_USER_PASS = os.getenv("MYSQL_USER_PASS") +# MONGODB +MONGO_IP = os.getenv("MONGO_IP", "localhost") +MONGO_PORT = int(os.getenv("MONGO_PORT", 27017)) +MONGO_DB = os.getenv("MONGO_DB") +MONGO_USER_NAME = os.getenv("MONGO_USER_NAME") +MONGO_USER_PASS = os.getenv("MONGO_USER_PASS") +MONGO_URL = os.getenv("MONGO_URL") + # REDIS # ip:port 多个可写为列表或者逗号隔开 如 ip1:port1,ip2:port2 或 ["ip1:port1", "ip2:port2"] REDISDB_IP_PORTS = os.getenv("REDISDB_IP_PORTS") REDISDB_USER_PASS = os.getenv("REDISDB_USER_PASS") -# 默认 0 到 15 共16个数据库 REDISDB_DB = int(os.getenv("REDISDB_DB", 0)) +# 连接redis时携带的其他参数，如ssl=True +REDISDB_KWARGS = dict() # 适用于redis哨兵模式 -REDISDB_SERVICE_NAME = os.getenv("REDISDB_SERVICE_NAME") +REDISDB_SERVICE_NAME = os.getenv("REDISDB_SERVICE_NAME") + +# 数据入库的pipeline，可自定义，默认MysqlPipeline +ITEM_PIPELINES = [ + "feapder.pipelines.mysql_pipeline.MysqlPipeline", + # "feapder.pipelines.mongo_pipeline.MongoPipeline", + # "feapder.pipelines.csv_pipeline.CsvPipeline", + # "feapder.pipelines.console_pipeline.ConsolePipeline", +] +CSV_EXPORT_PATH = "data/csv" # CSV文件保存路径，支持相对路径和绝对路径 +EXPORT_DATA_MAX_FAILED_TIMES = 10 # 导出数据时最大的失败次数，包括保存和更新，超过这个次数报警 +EXPORT_DATA_MAX_RETRY_TIMES = 10 # 导出数据时最大的重试次数，包括保存和更新，超过这个次数则放弃重试 # 爬虫相关 # COLLECTOR -COLLECTOR_SLEEP_TIME = 1 # 从任务队列中获取任务到内存队列的间隔 -COLLECTOR_TASK_COUNT = 10 # 每次获取任务数量 +COLLECTOR_TASK_COUNT = 32 # 每次获取任务数量，追求速度推荐32 # SPIDER -SPIDER_THREAD_COUNT = 1 # 爬虫并发数 -SPIDER_SLEEP_TIME = 0 # 下载时间间隔（解析完一个response后休眠时间） -SPIDER_TASK_COUNT = 1 # 每个parser从内存队列中获取任务的数量 -SPIDER_MAX_RETRY_TIMES = 100 # 每个请求最大重试次数 +SPIDER_THREAD_COUNT = 1 # 爬虫并发数，追求速度推荐32 +# 下载时间间隔 单位秒。 支持随机 如 SPIDER_SLEEP_TIME = [2, 5] 则间隔为 2~5秒之间的随机数，包含2和5 +SPIDER_SLEEP_TIME = 0 +SPIDER_MAX_RETRY_TIMES = 10 # 每个请求最大重试次数 # 是否主动执行添加 设置为False 需要手动调用start_monitor_task，适用于多进程情况下 SPIDER_AUTO_START_REQUESTS = True - -# 重新尝试失败的requests 当requests重试次数超过允许的最大重试次数算失败 +KEEP_ALIVE = False # 爬虫是否常驻 + +# 浏览器渲染 +WEBDRIVER = dict( + pool_size=1, # 浏览器的数量 + load_images=True, # 是否加载图片 + user_agent=None, # 字符串 或 无参函数，返回值为user_agent + proxy=None, # xxx.xxx.xxx.xxx:xxxx 或 无参函数，返回值为代理地址 + headless=False, # 是否为无头浏览器 + driver_type="CHROME", # CHROME、EDGE、PHANTOMJS、FIREFOX + timeout=30, # 请求超时时间 + window_size=(1024, 800), # 窗口大小 + executable_path=None, # 浏览器路径，默认为默认路径 + render_time=0, # 渲染时长，即打开网页等待指定时间后再获取源码 + custom_argument=[ + "--ignore-certificate-errors", + "--disable-blink-features=AutomationControlled", + ], # 自定义浏览器渲染参数 + xhr_url_regexes=None, # 拦截xhr接口，支持正则，数组类型 + auto_install_driver=True, # 自动下载浏览器驱动 支持chrome 和 firefox + download_path=None, # 下载文件的路径 + use_stealth_js=False, # 使用stealth.min.js隐藏浏览器特征 +) + +PLAYWRIGHT = dict( + user_agent=None, # 字符串 或 无参函数，返回值为user_agent + proxy=None, # xxx.xxx.xxx.xxx:xxxx 或 无参函数，返回值为代理地址 + headless=False, # 是否为无头浏览器 + driver_type="chromium", # chromium、firefox、webkit + timeout=30, # 请求超时时间 + window_size=(1024, 800), # 窗口大小 + executable_path=None, # 浏览器路径，默认为默认路径 + download_path=None, # 下载文件的路径 + render_time=0, # 渲染时长，即打开网页等待指定时间后再获取源码 + wait_until="networkidle", # 等待页面加载完成的事件,可选值："commit", "domcontentloaded", "load", "networkidle" + use_stealth_js=False, # 使用stealth.min.js隐藏浏览器特征 + page_on_event_callback=None, # page.on() 事件的回调 如 page_on_event_callback={"dialog": lambda dialog: dialog.accept()} + storage_state_path=None, # 保存浏览器状态的路径 + url_regexes=None, # 拦截接口，支持正则，数组类型 + save_all=False, # 是否保存所有拦截的接口, 配合url_regexes使用，为False时只保存最后一次拦截的接口 +) + +# 爬虫启动时，重新抓取失败的requests RETRY_FAILED_REQUESTS = False -# request 超时时间，超过这个时间重新做（不是网络请求的超时时间）单位秒 -REQUEST_TIME_OUT = 600 # 10分钟 +# 爬虫启动时，重新入库失败的item +RETRY_FAILED_ITEMS = False # 保存失败的request SAVE_FAILED_REQUEST = True - -# 下载缓存 利用redis缓存，由于内存小，所以仅供测试时使用 +# request防丢机制。（指定的REQUEST_LOST_TIMEOUT时间内request还没做完，会重新下发 重做） +REQUEST_LOST_TIMEOUT = 600 # 10分钟 +# request网络请求超时时间 +REQUEST_TIMEOUT = 22 # 等待服务器响应的超时时间，浮点数，或(connect timeout, read timeout)元组 +# item在内存队列中最大缓存数量 +ITEM_MAX_CACHED_COUNT = 5000 +# item每批入库的最大数量 +ITEM_UPLOAD_BATCH_MAX_SIZE = 1000 +# item入库时间间隔 +ITEM_UPLOAD_INTERVAL = 1 +# 内存任务队列最大缓存的任务数，默认不限制；仅对AirSpider有效。 +TASK_MAX_CACHED_SIZE = 0 + +# 下载缓存 利用redis缓存，但由于内存大小限制，所以建议仅供开发调试代码时使用，防止每次debug都需要网络请求 RESPONSE_CACHED_ENABLE = False # 是否启用下载缓存 成本高的数据或容易变需求的数据，建议设置为True RESPONSE_CACHED_EXPIRE_TIME = 3600 # 缓存时间 秒 -RESPONSE_CACHED_USED = False # 是否使用缓存 补才数据时可设置为True - -WARNING_FAILED_COUNT = 1000 # 任务失败数 超过WARNING_FAILED_COUNT则报警 +RESPONSE_CACHED_USED = False # 是否使用缓存 补采数据时可设置为True -# 爬虫初始化工作 # redis 存放item与request的根目录 REDIS_KEY = "" -# 每次启动时需要删除的表 -DELETE_TABS = [] -# 爬虫做完request后是否自动结束或者等待任务 -AUTO_STOP_WHEN_SPIDER_DONE = True -# 是否将item添加到 mysql 支持列表 指定添加的item 可模糊指定 -ADD_ITEM_TO_MYSQL = True -# 是否将item添加到 redis 支持列表 指定添加的item 可模糊指定 -ADD_ITEM_TO_REDIS = False - -# PROCESS 进程数 未用 -PROCESS_COUNT = 1 +# 爬虫启动时删除的key，类型: 元组/bool/string。 支持正则; 常用于清空任务队列，否则重启时会断点续爬 +DELETE_KEYS = [] # 设置代理 PROXY_EXTRACT_API = None # 代理提取API ，返回的代理分割符为\r\n PROXY_ENABLE = True +PROXY_MAX_FAILED_TIMES = 5 # 代理最大失败次数，超过则不使用，自动删除 +PROXY_POOL = "feapder.network.proxy_pool.ProxyPool" # 代理池 # 随机headers RANDOM_HEADERS = True +# UserAgent类型 支持 'chrome', 'opera', 'firefox', 'internetexplorer', 'safari'，'mobile' 若不指定则随机类型 +USER_AGENT_TYPE = "chrome" +# 默认使用的浏览器头 +DEFAULT_USERAGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36" # requests 使用session USE_SESSION = False -# 去重 -ITEM_FILTER_ENABLE = False # item 去重 -REQUEST_FILTER_ENABLE = False # request 去重 +# 下载 +DOWNLOADER = "feapder.network.downloader.RequestsDownloader" # 请求下载器 +SESSION_DOWNLOADER = "feapder.network.downloader.RequestsSessionDownloader" +RENDER_DOWNLOADER = "feapder.network.downloader.SeleniumDownloader" # 渲染下载器 +# RENDER_DOWNLOADER="feapder.network.downloader.PlaywrightDownloader" +MAKE_ABSOLUTE_LINKS = True # 自动转成绝对连接 -# 报警 -DINGDING_WARNING_URL = "" # 钉钉机器人api -DINGDING_WARNING_PHONE = "" # 报警人 -LINGXI_TOKEN = "" # 灵犀报警token +# 去重 +ITEM_FILTER_ENABLE = False # item 去重 +ITEM_FILTER_SETTING = dict( + filter_type=1 # 永久去重（BloomFilter） = 1 、内存去重（MemoryFilter） = 2、 临时去重（ExpireFilter）= 3、轻量去重（LiteFilter）= 4 +) +REQUEST_FILTER_ENABLE = False # request 去重 +REQUEST_FILTER_SETTING = dict( + filter_type=3, # 永久去重（BloomFilter） = 1 、内存去重（MemoryFilter） = 2、 临时去重（ExpireFilter）= 3、 轻量去重（LiteFilter）= 4 + expire_time=2592000, # 过期时间1个月 +) + +# 报警 支持钉钉、飞书、企业微信、邮件 +# 钉钉报警 +DINGDING_WARNING_URL = "" # 钉钉机器人api +DINGDING_WARNING_PHONE = "" # 被@的群成员手机号，支持列表，可指定多个。 +DINGDING_WARNING_USER_ID = "" # 被@的群成员userId，支持列表，可指定多个 +DINGDING_WARNING_ALL = False # 是否提示所有人， 默认为False +DINGDING_WARNING_SECRET = None # 加签密钥 +# 飞书报警 +# https://open.feishu.cn/document/ukTMukTMukTM/ucTM5YjL3ETO24yNxkjN#e1cdee9f +FEISHU_WARNING_URL = "" # 飞书机器人api +FEISHU_WARNING_USER = None # 报警人 {"open_id":"ou_xxxxx", "name":"xxxx"} 或 [{"open_id":"ou_xxxxx", "name":"xxxx"}] +FEISHU_WARNING_ALL = False # 是否提示所有人， 默认为False +# 邮件报警 +EMAIL_SENDER = "" # 发件人 +EMAIL_PASSWORD = "" # 授权码 +EMAIL_RECEIVER = "" # 收件人 支持列表，可指定多个 +EMAIL_SMTPSERVER = "smtp.163.com" # 邮件服务器 默认为163邮箱 +# 企业微信报警 +WECHAT_WARNING_URL = "" # 企业微信机器人api +WECHAT_WARNING_PHONE = "" # 报警人 将会在群内@此人, 支持列表，可指定多人 +WECHAT_WARNING_ALL = False # 是否提示所有人， 默认为False +# QMSG报警 +QMSG_WARNING_URL = "" # qmsg机器人api +QMSG_WARNING_QQ = "" # 指定要接收消息的QQ号或者QQ群。多个以英文逗号分割，例如：12345,12346,支持列表，可指定多人 +QMSG_WARNING_BOT = "" # 机器人的QQ号 +# 时间间隔 +WARNING_INTERVAL = 3600 # 相同报警的报警时间间隔，防止刷屏; 0表示不去重 +WARNING_LEVEL = "DEBUG" # 报警级别， DEBUG / INFO / ERROR +WARNING_FAILED_COUNT = 1000 # 任务失败数 超过WARNING_FAILED_COUNT则报警 +WARNING_CHECK_TASK_COUNT_INTERVAL = 1200 # 检查已做任务数量的时间间隔，若两次时间间隔之间，任务数无变化则报警 +# 日志 LOG_NAME = os.path.basename(os.getcwd()) LOG_PATH = "log/%s.log" % LOG_NAME # log存储路径 -LOG_LEVEL = "DEBUG" -LOG_IS_WRITE_TO_FILE = False +LOG_LEVEL = os.getenv("LOG_LEVEL", "DEBUG") # 日志级别 +LOG_COLOR = True # 是否带有颜色 +LOG_IS_WRITE_TO_CONSOLE = True # 是否打印到控制台 +LOG_IS_WRITE_TO_FILE = False # 是否写文件 +LOG_MODE = "w" # 写文件的模式 +LOG_MAX_BYTES = 10 * 1024 * 1024 # 每个日志文件的最大字节数 +LOG_BACKUP_COUNT = 20 # 日志文件保留数量 +LOG_ENCODING = "utf8" # 日志文件编码 +# 是否详细的打印异常 +PRINT_EXCEPTION_DETAILS = True +# 设置不带颜色的日志格式 +LOG_FORMAT = "%(threadName)s|%(asctime)s|%(filename)s|%(funcName)s|line:%(lineno)d|%(levelname)s| %(message)s" +# 设置带有颜色的日志格式 +os.environ["LOGURU_FORMAT"] = ( + "{time:YYYY-MM-DD HH:mm:ss.SSS} | " + "{level: <8} | " + "{name}:{function}:line:{line} | {message}" +) OTHERS_LOG_LEVAL = "ERROR" # 第三方库的log等级 -############## 导入用户自定义的setting ############# +# 打点监控 influxdb 配置 +INFLUXDB_HOST = os.getenv("INFLUXDB_HOST", "localhost") +INFLUXDB_PORT = int(os.getenv("INFLUXDB_PORT", 8086)) +INFLUXDB_UDP_PORT = int(os.getenv("INFLUXDB_UDP_PORT", 8089)) +INFLUXDB_USER = os.getenv("INFLUXDB_USER") +INFLUXDB_PASSWORD = os.getenv("INFLUXDB_PASSWORD") +INFLUXDB_DATABASE = os.getenv("INFLUXDB_DB") +# 监控数据存储的表名，爬虫管理系统上会以task_id命名 +INFLUXDB_MEASUREMENT = "task_" + os.getenv("TASK_ID") if os.getenv("TASK_ID") else None +# 打点监控其他参数，若这里也配置了influxdb的参数, 则会覆盖外面的配置 +METRICS_OTHER_ARGS = dict(retention_policy_duration="180d", emit_interval=60) + +############# 导入用户自定义的setting ############# try: from setting import * + + # 兼容老版本的配置 + KEEP_ALIVE = not AUTO_STOP_WHEN_SPIDER_DONE except: pass diff --git a/feapder/templates/air_spider_template.tmpl b/feapder/templates/air_spider_template.tmpl index 373514e9..4cd1f505 100644 --- a/feapder/templates/air_spider_template.tmpl +++ b/feapder/templates/air_spider_template.tmpl @@ -12,10 +12,14 @@ import feapder class ${spider_name}(feapder.AirSpider): def start_requests(self): - yield feapder.Request("https://www.baidu.com") + yield feapder.Request("https://spidertools.cn") def parse(self, request, response): - print(response) + # 提取网站title + print(response.xpath("//title/text()").extract_first()) + # 提取网站描述 + print(response.xpath("//meta[@name='description']/@content").extract_first()) + print("网站地址: ", response.url) if __name__ == "__main__": diff --git a/feapder/templates/batch_spider_template.tmpl b/feapder/templates/batch_spider_template.tmpl index 86d016b8..9802e994 100644 --- a/feapder/templates/batch_spider_template.tmpl +++ b/feapder/templates/batch_spider_template.tmpl @@ -8,6 +8,7 @@ Created on {DATE} """ import feapder +from feapder import ArgumentParser class ${spider_name}(feapder.BatchSpider): @@ -18,21 +19,25 @@ class ${spider_name}(feapder.BatchSpider): REDISDB_DB=0, MYSQL_IP="localhost", MYSQL_PORT=3306, - MYSQL_DB="feapder", - MYSQL_USER_NAME="feapder", - MYSQL_USER_PASS="feapder123", + MYSQL_DB="", + MYSQL_USER_NAME="", + MYSQL_USER_PASS="", ) def start_requests(self, task): - yield feapder.Request("https://www.baidu.com") + yield feapder.Request("https://spidertools.cn") def parse(self, request, response): - print(response) + # 提取网站title + print(response.xpath("//title/text()").extract_first()) + # 提取网站描述 + print(response.xpath("//meta[@name='description']/@content").extract_first()) + print("网站地址: ", response.url) if __name__ == "__main__": spider = ${spider_name}( - redis_key="xxx:xxxx", # redis中存放任务等信息的根key + redis_key="xxx:xxxx", # 分布式爬虫调度信息存储位置 task_table="", # mysql中的任务表 task_keys=["id", "xxx"], # 需要获取任务表里的字段名，可添加多个 task_state="state", # mysql中任务状态字段 @@ -41,5 +46,24 @@ if __name__ == "__main__": batch_interval=7, # 批次周期 天为单位 若为小时 可写 1 / 24 ) - # spider.start_monitor_task() # 下发及监控任务 - spider.start() # 采集 + parser = ArgumentParser(description="${spider_name}爬虫") + + parser.add_argument( + "--start_master", + action="store_true", + help="添加任务", + function=spider.start_monitor_task, + ) + parser.add_argument( + "--start_worker", action="store_true", help="启动爬虫", function=spider.start + ) + + parser.start() + + # 直接启动 + # spider.start() # 启动爬虫 + # spider.start_monitor_task() # 添加任务 + + # 通过命令行启动 + # python ${file_name} --start_master # 添加任务 + # python ${file_name} --start_worker # 启动爬虫 diff --git a/feapder/templates/item_template.tmpl b/feapder/templates/item_template.tmpl index 290002b2..0499ffb8 100644 --- a/feapder/templates/item_template.tmpl +++ b/feapder/templates/item_template.tmpl @@ -12,9 +12,11 @@ from feapder import Item class ${item_name}Item(Item): """ - This class was generated by feapder. - command: feapder create -i ${table_name}. + This class was generated by feapder + command: feapder create -i ${command} """ + __table_name__ = "${table_name}" + def __init__(self, *args, **kwargs): ${propertys} diff --git a/feapder/templates/project_template/CHECK_DATA.md b/feapder/templates/project_template/CHECK_DATA.md new file mode 100644 index 00000000..9e34b6aa --- /dev/null +++ b/feapder/templates/project_template/CHECK_DATA.md @@ -0,0 +1,49 @@ +# 数据审核 +## 表说明： + +> 表名 含义（更新策略） + +## 一、准确性 + +**字段设计是否满足需求？ 表之间的关联字段是否满足要求？ （需要人工检查）** + +> 注意：是否设计了自增 id，id 的类型是否设置为 bigint？ +> 注意：unique index 是否需要设计？ +> 注意：各张表之间是否需要设计关联字段； + +* [ ] 是 +* [ ] 否 + +**各字段采集内容及存储格式是否满足要求？是否与网页一致？是否有信息缺失？** + +> 备注：可尝试对每个字段进行升降序排列，然后抽样检查； + +**是否考虑了网站同一类数据可能出现的数据格式不一致情况？** + +> 建议：代码对各个字段不做兼容性处理、数据不一致则抛出异常并记录 + +* [ ] 是 +* [ ] 否 + +## 二、全量性 + +**如果是增量采集，是否最早信息和最晚信息都采集了，同时条目总数是否正确；** +**如果是批次采集，是否每个批次都有？** + +>备注：需要去网页端评估单个批次的总量； +>参考sql语句：SELECT count(1), batch_date from [table_name] GROUP BY batch_date; + +**如果与另外一张表有关联关系，是否信息关联完整？** + +## 三、稳定性 + +* [ ] 是否能够长期稳定采集？ +* [ ] 是否加IP代理？ +* [ ] 是否支持断点续跑? +* [ ] 是否能确保按时启动，定期采集? +* [ ] 是否已开启报警？ + +## 四、采集频次、类型、存储方式 + +* [ ] 采集频次是否满足要求？ +* [ ] 采集类型是否满足要求：增量采集 or 批次采集? diff --git a/feapder/templates/project_template/README.md b/feapder/templates/project_template/README.md new file mode 100644 index 00000000..c160ae2c --- /dev/null +++ b/feapder/templates/project_template/README.md @@ -0,0 +1,8 @@ +# xxx爬虫文档 +## 调研 + +## 数据库设计 + +## 爬虫逻辑 + +## 项目架构 \ No newline at end of file diff --git a/feapder/templates/project_template/main.py b/feapder/templates/project_template/main.py index ce6ab97d..2dfc45ec 100644 --- a/feapder/templates/project_template/main.py +++ b/feapder/templates/project_template/main.py @@ -11,10 +11,16 @@ from spiders import * +def crawl_xxx(): + """ + AirSpider爬虫 + """ + spider = xxx.XXXSpider() + spider.start() def crawl_xxx(): """ - 普通爬虫 + Spider爬虫 """ spider = xxx.XXXSpider(redis_key="xxx:xxx") spider.start() @@ -22,8 +28,7 @@ def crawl_xxx(): def crawl_xxx(args): """ - 批次爬虫 - @param args: 1 / 2 / init + BatchSpider爬虫 """ spider = xxx_spider.XXXSpider( task_table="", # mysql中的任务表 @@ -39,7 +44,7 @@ def crawl_xxx(args): spider.start_monitor_task() elif args == 2: spider.start() - elif args == "init": + elif args == 3: spider.init_task() @@ -47,10 +52,28 @@ def crawl_xxx(args): parser = ArgumentParser(description="xxx爬虫") parser.add_argument( - "--crawl_xxx", action="store_true", help="xxx", function=crawl_xxx + "--crawl_xxx", action="store_true", help="xxx爬虫", function=crawl_xxx ) parser.add_argument( - "--crawl_xxx", type=int, nargs=1, help="xxx(1|2）", function=crawl_xxx + "--crawl_xxx", action="store_true", help="xxx爬虫", function=crawl_xxx + ) + parser.add_argument( + "--crawl_xxx", + type=int, + nargs=1, + help="xxx爬虫", + choices=[1, 2, 3], + function=crawl_xxx, ) parser.start() + + # main.py作为爬虫启动的统一入口，提供命令行的方式启动多个爬虫，若只有一个爬虫，可不编写main.py + # 将上面的xxx修改为自己实际的爬虫名 + # 查看运行命令 python main.py --help + # AirSpider与Spider爬虫运行方式 python main.py --crawl_xxx + # BatchSpider运行方式 + # 1. 下发任务：python main.py --crawl_xxx 1 + # 2. 采集：python main.py --crawl_xxx 2 + # 3. 重置任务：python main.py --crawl_xxx 3 + diff --git a/feapder/templates/project_template/setting.py b/feapder/templates/project_template/setting.py index a1c3afd2..140aaa07 100644 --- a/feapder/templates/project_template/setting.py +++ b/feapder/templates/project_template/setting.py @@ -1,75 +1,196 @@ # -*- coding: utf-8 -*- """爬虫配置文件""" -import os - - -# MYSQL -MYSQL_IP = "" -MYSQL_PORT = 3306 -MYSQL_DB = "" -MYSQL_USER_NAME = "" -MYSQL_USER_PASS = "" - -# REDIS -# IP:PORT -REDISDB_IP_PORTS = "xxx:6379" -REDISDB_USER_PASS = "" -# 默认 0 到 15 共16个数据库 -REDISDB_DB = 0 - +# import os +# import sys +# +# # MYSQL +# MYSQL_IP = "localhost" +# MYSQL_PORT = 3306 +# MYSQL_DB = "" +# MYSQL_USER_NAME = "" +# MYSQL_USER_PASS = "" +# +# # MONGODB +# MONGO_IP = "localhost" +# MONGO_PORT = 27017 +# MONGO_DB = "" +# MONGO_USER_NAME = "" +# MONGO_USER_PASS = "" +# MONGO_URL = " +# +# # REDIS +# # ip:port 多个可写为列表或者逗号隔开 如 ip1:port1,ip2:port2 或 ["ip1:port1", "ip2:port2"] +# REDISDB_IP_PORTS = "localhost:6379" +# REDISDB_USER_PASS = "" +# REDISDB_DB = 0 +# # 连接redis时携带的其他参数，如ssl=True +# REDISDB_KWARGS = dict() +# # 适用于redis哨兵模式 +# REDISDB_SERVICE_NAME = "" +# +# # 数据入库的pipeline，可自定义，默认MysqlPipeline +# ITEM_PIPELINES = [ +# "feapder.pipelines.mysql_pipeline.MysqlPipeline", +# # "feapder.pipelines.mongo_pipeline.MongoPipeline", +# # "feapder.pipelines.csv_pipeline.CsvPipeline", +# # "feapder.pipelines.console_pipeline.ConsolePipeline", +# ] +# CSV_EXPORT_PATH = "data/csv" # CSV文件保存路径，支持相对路径和绝对路径 +# EXPORT_DATA_MAX_FAILED_TIMES = 10 # 导出数据时最大的失败次数，包括保存和更新，超过这个次数报警 +# EXPORT_DATA_MAX_RETRY_TIMES = 10 # 导出数据时最大的重试次数，包括保存和更新，超过这个次数则放弃重试 +# # # 爬虫相关 # # COLLECTOR -# COLLECTOR_SLEEP_TIME = 1 # 从任务队列中获取任务到内存队列的间隔 -# COLLECTOR_TASK_COUNT = 100 # 每次获取任务数量 +# COLLECTOR_TASK_COUNT = 32 # 每次获取任务数量，追求速度推荐32 # # # SPIDER -# SPIDER_THREAD_COUNT = 10 # 爬虫并发数 -# SPIDER_SLEEP_TIME = 0 # 下载时间间隔（解析完一个response后休眠时间） -# SPIDER_MAX_RETRY_TIMES = 100 # 每个请求最大重试次数 +# SPIDER_THREAD_COUNT = 1 # 爬虫并发数，追求速度推荐32 +# # 下载时间间隔 单位秒。 支持随机 如 SPIDER_SLEEP_TIME = [2, 5] 则间隔为 2~5秒之间的随机数，包含2和5 +# SPIDER_SLEEP_TIME = 0 +# SPIDER_MAX_RETRY_TIMES = 10 # 每个请求最大重试次数 +# KEEP_ALIVE = False # 爬虫是否常驻 -# # 重新尝试失败的requests 当requests重试次数超过允许的最大重试次数算失败 +# 下载 +# DOWNLOADER = "feapder.network.downloader.RequestsDownloader" # 请求下载器 +# SESSION_DOWNLOADER = "feapder.network.downloader.RequestsSessionDownloader" +# RENDER_DOWNLOADER = "feapder.network.downloader.SeleniumDownloader" # 渲染下载器 +# # RENDER_DOWNLOADER="feapder.network.downloader.PlaywrightDownloader" +# MAKE_ABSOLUTE_LINKS = True # 自动转成绝对连接 + +# # 浏览器渲染 +# WEBDRIVER = dict( +# pool_size=1, # 浏览器的数量 +# load_images=True, # 是否加载图片 +# user_agent=None, # 字符串 或 无参函数，返回值为user_agent +# proxy=None, # xxx.xxx.xxx.xxx:xxxx 或 无参函数，返回值为代理地址 +# headless=False, # 是否为无头浏览器 +# driver_type="CHROME", # CHROME、EDGE、PHANTOMJS、FIREFOX +# timeout=30, # 请求超时时间 +# window_size=(1024, 800), # 窗口大小 +# executable_path=None, # 浏览器路径，默认为默认路径 +# render_time=0, # 渲染时长，即打开网页等待指定时间后再获取源码 +# custom_argument=[ +# "--ignore-certificate-errors", +# "--disable-blink-features=AutomationControlled", +# ], # 自定义浏览器渲染参数 +# xhr_url_regexes=None, # 拦截xhr接口，支持正则，数组类型 +# auto_install_driver=True, # 自动下载浏览器驱动 支持chrome 和 firefox +# download_path=None, # 下载文件的路径 +# use_stealth_js=False, # 使用stealth.min.js隐藏浏览器特征 +# ) +# +# PLAYWRIGHT = dict( +# user_agent=None, # 字符串 或 无参函数，返回值为user_agent +# proxy=None, # xxx.xxx.xxx.xxx:xxxx 或 无参函数，返回值为代理地址 +# headless=False, # 是否为无头浏览器 +# driver_type="chromium", # chromium、firefox、webkit +# timeout=30, # 请求超时时间 +# window_size=(1024, 800), # 窗口大小 +# executable_path=None, # 浏览器路径，默认为默认路径 +# download_path=None, # 下载文件的路径 +# render_time=0, # 渲染时长，即打开网页等待指定时间后再获取源码 +# wait_until="networkidle", # 等待页面加载完成的事件,可选值："commit", "domcontentloaded", "load", "networkidle" +# use_stealth_js=False, # 使用stealth.min.js隐藏浏览器特征 +# page_on_event_callback=None, # page.on() 事件的回调 如 page_on_event_callback={"dialog": lambda dialog: dialog.accept()} +# storage_state_path=None, # 保存浏览器状态的路径 +# url_regexes=None, # 拦截接口，支持正则，数组类型 +# save_all=False, # 是否保存所有拦截的接口, 配合url_regexes使用，为False时只保存最后一次拦截的接口 +# ) +# +# # 爬虫启动时，重新抓取失败的requests # RETRY_FAILED_REQUESTS = False -# # request 超时时间，超过这个时间重新做（不是网络请求的超时时间）单位秒 -# REQUEST_TIME_OUT = 600 # 10分钟 +# # 爬虫启动时，重新入库失败的item +# RETRY_FAILED_ITEMS = False # # 保存失败的request # SAVE_FAILED_REQUEST = True +# # request防丢机制。（指定的REQUEST_LOST_TIMEOUT时间内request还没做完，会重新下发 重做） +# REQUEST_LOST_TIMEOUT = 600 # 10分钟 +# # request网络请求超时时间 +# REQUEST_TIMEOUT = 22 # 等待服务器响应的超时时间，浮点数，或(connect timeout, read timeout)元组 +# # item在内存队列中最大缓存数量 +# ITEM_MAX_CACHED_COUNT = 5000 +# # item每批入库的最大数量 +# ITEM_UPLOAD_BATCH_MAX_SIZE = 1000 +# # item入库时间间隔 +# ITEM_UPLOAD_INTERVAL = 1 +# # 内存任务队列最大缓存的任务数，默认不限制；仅对AirSpider有效。 +# TASK_MAX_CACHED_SIZE = 0 # -# # 下载缓存 利用redis缓存，由于内存小，所以仅供测试时使用 +# # 下载缓存 利用redis缓存，但由于内存大小限制，所以建议仅供开发调试代码时使用，防止每次debug都需要网络请求 # RESPONSE_CACHED_ENABLE = False # 是否启用下载缓存 成本高的数据或容易变需求的数据，建议设置为True # RESPONSE_CACHED_EXPIRE_TIME = 3600 # 缓存时间 秒 -# RESPONSE_CACHED_USED = False # 是否使用缓存 补才数据时可设置为True -# -# WARNING_FAILED_COUNT = 1000 # 任务失败数 超过WARNING_FAILED_COUNT则报警 -# -# # 爬虫初始化工作 -# # 爬虫做完request后是否自动结束或者等待任务 -# AUTO_STOP_WHEN_SPIDER_DONE = True -# # 是否将item添加到 mysql 支持列表 指定添加的item 可模糊指定 -# ADD_ITEM_TO_MYSQL = True -# # 是否将item添加到 redis 支持列表 指定添加的item 可模糊指定 -# ADD_ITEM_TO_REDIS = False -# +# RESPONSE_CACHED_USED = False # 是否使用缓存 补采数据时可设置为True # # # 设置代理 # PROXY_EXTRACT_API = None # 代理提取API ，返回的代理分割符为\r\n # PROXY_ENABLE = True +# PROXY_MAX_FAILED_TIMES = 5 # 代理最大失败次数，超过则不使用，自动删除 +# PROXY_POOL = "feapder.network.proxy_pool.ProxyPool" # 代理池 # # # 随机headers # RANDOM_HEADERS = True +# # UserAgent类型 支持 'chrome', 'opera', 'firefox', 'internetexplorer', 'safari'，'mobile' 若不指定则随机类型 +# USER_AGENT_TYPE = "chrome" +# # 默认使用的浏览器头 +# DEFAULT_USERAGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36" # # requests 使用session # USE_SESSION = False # # # 去重 -# ITEM_FILTER_ENABLE = False # item 去重 -# REQUEST_FILTER_ENABLE = False # request 去重 +# ITEM_FILTER_ENABLE = False # item 去重 +# REQUEST_FILTER_ENABLE = False # request 去重 +# ITEM_FILTER_SETTING = dict( +# filter_type=1 # 永久去重（BloomFilter） = 1 、内存去重（MemoryFilter） = 2、 临时去重（ExpireFilter）= 3、轻量去重（LiteFilter）= 4 +# ) +# REQUEST_FILTER_SETTING = dict( +# filter_type=3, # 永久去重（BloomFilter） = 1 、内存去重（MemoryFilter） = 2、 临时去重（ExpireFilter）= 3、 轻量去重（LiteFilter）= 4 +# expire_time=2592000, # 过期时间1个月 +# ) # -# # 报警 -# DINGDING_WARNING_URL = "" # 钉钉机器人api -# DINGDING_WARNING_PHONE = "" # 报警人 -# LINGXI_TOKEN = "" # 灵犀报警token +# # 报警 支持钉钉、飞书、企业微信、邮件 +# # 钉钉报警 +# DINGDING_WARNING_URL = "" # 钉钉机器人api +# DINGDING_WARNING_PHONE = "" # 被@的群成员手机号，支持列表，可指定多个。 +# DINGDING_WARNING_USER_ID = "" # 被@的群成员userId，支持列表，可指定多个 +# DINGDING_WARNING_ALL = False # 是否提示所有人， 默认为False +# DINGDING_WARNING_SECRET = None # 加签密钥 +# # 飞书报警 +# # https://open.feishu.cn/document/ukTMukTMukTM/ucTM5YjL3ETO24yNxkjN#e1cdee9f +# FEISHU_WARNING_URL = "" # 飞书机器人api +# FEISHU_WARNING_USER = None # 报警人 {"open_id":"ou_xxxxx", "name":"xxxx"} 或 [{"open_id":"ou_xxxxx", "name":"xxxx"}] +# FEISHU_WARNING_ALL = False # 是否提示所有人， 默认为False +# # 邮件报警 +# EMAIL_SENDER = "" # 发件人 +# EMAIL_PASSWORD = "" # 授权码 +# EMAIL_RECEIVER = "" # 收件人 支持列表，可指定多个 +# EMAIL_SMTPSERVER = "smtp.163.com" # 邮件服务器 默认为163邮箱 +# # 企业微信报警 +# WECHAT_WARNING_URL = "" # 企业微信机器人api +# WECHAT_WARNING_PHONE = "" # 报警人 将会在群内@此人, 支持列表，可指定多人 +# WECHAT_WARNING_ALL = False # 是否提示所有人， 默认为False +# # QMSG报警 +# QMSG_WARNING_URL = "" # qmsg机器人api +# QMSG_WARNING_QQ = "" # 指定要接收消息的QQ号或者QQ群。多个以英文逗号分割，例如：12345,12346,支持列表，可指定多人 +# QMSG_WARNING_BOT = "" # 机器人的QQ号 +# # 时间间隔 +# WARNING_INTERVAL = 3600 # 相同报警的报警时间间隔，防止刷屏; 0表示不去重 +# WARNING_LEVEL = "DEBUG" # 报警级别， DEBUG / INFO / ERROR +# WARNING_FAILED_COUNT = 1000 # 任务失败数 超过WARNING_FAILED_COUNT则报警 # # LOG_NAME = os.path.basename(os.getcwd()) # LOG_PATH = "log/%s.log" % LOG_NAME # log存储路径 # LOG_LEVEL = "DEBUG" -# LOG_IS_WRITE_TO_FILE = False +# LOG_COLOR = True # 是否带有颜色 +# LOG_IS_WRITE_TO_CONSOLE = True # 是否打印到控制台 +# LOG_IS_WRITE_TO_FILE = False # 是否写文件 +# LOG_MODE = "w" # 写文件的模式 +# LOG_MAX_BYTES = 10 * 1024 * 1024 # 每个日志文件的最大字节数 +# LOG_BACKUP_COUNT = 20 # 日志文件保留数量 +# LOG_ENCODING = "utf8" # 日志文件编码 # OTHERS_LOG_LEVAL = "ERROR" # 第三方库的log等级 +# +# # 切换工作路径为当前项目路径 +# project_path = os.path.abspath(os.path.dirname(__file__)) +# os.chdir(project_path) # 切换工作路经 +# sys.path.insert(0, project_path) +# print("当前工作路径为 " + os.getcwd()) diff --git a/feapder/templates/spider_template.tmpl b/feapder/templates/spider_template.tmpl index be60a002..bb209527 100644 --- a/feapder/templates/spider_template.tmpl +++ b/feapder/templates/spider_template.tmpl @@ -17,10 +17,14 @@ class ${spider_name}(feapder.Spider): ) def start_requests(self): - yield feapder.Request("https://www.baidu.com") + yield feapder.Request("https://spidertools.cn") def parse(self, request, response): - print(response) + # 提取网站title + print(response.xpath("//title/text()").extract_first()) + # 提取网站描述 + print(response.xpath("//meta[@name='description']/@content").extract_first()) + print("网站地址: ", response.url) if __name__ == "__main__": diff --git a/feapder/templates/task_spider_template.tmpl b/feapder/templates/task_spider_template.tmpl new file mode 100644 index 00000000..66bbbba1 --- /dev/null +++ b/feapder/templates/task_spider_template.tmpl @@ -0,0 +1,79 @@ +# -*- coding: utf-8 -*- +""" +Created on {DATE} +--------- +@summary: +--------- +@author: {USER} +""" + +import feapder +from feapder import ArgumentParser + + +class ${spider_name}(feapder.TaskSpider): + # 自定义数据库，若项目中有setting.py文件，此自定义可删除 + __custom_setting__ = dict( + REDISDB_IP_PORTS="localhost:6379", + REDISDB_USER_PASS="", + REDISDB_DB=0, + MYSQL_IP="localhost", + MYSQL_PORT=3306, + MYSQL_DB="", + MYSQL_USER_NAME="", + MYSQL_USER_PASS="", + ) + + def start_requests(self, task): + task_id = task.id + url = task.url + yield feapder.Request(url, task_id=task_id) + + def parse(self, request, response): + # 提取网站title + print(response.xpath("//title/text()").extract_first()) + # 提取网站描述 + print(response.xpath("//meta[@name='description']/@content").extract_first()) + print("网站地址: ", response.url) + + # mysql 需要更新任务状态为做完 即 state=1 + yield self.update_task_batch(request.task_id) + + +if __name__ == "__main__": + # 用mysql做任务表，需要先建好任务任务表 + spider = ${spider_name}( + redis_key="xxx:xxx", # 分布式爬虫调度信息存储位置 + task_table="", # mysql中的任务表 + task_keys=["id", "url"], # 需要获取任务表里的字段名，可添加多个 + task_state="state", # mysql中任务状态字段 + ) + + # 用redis做任务表 + # spider = ${spider_name}( + # redis_key="xxx:xxxx", # 分布式爬虫调度信息存储位置 + # task_table="", # 任务表名 + # task_table_type="redis", # 任务表类型为redis + # ) + + parser = ArgumentParser(description="${spider_name}爬虫") + + parser.add_argument( + "--start_master", + action="store_true", + help="添加任务", + function=spider.start_monitor_task, + ) + parser.add_argument( + "--start_worker", action="store_true", help="启动爬虫", function=spider.start + ) + + parser.start() + + # 直接启动 + # spider.start() # 启动爬虫 + # spider.start_monitor_task() # 添加任务 + + # 通过命令行启动 + # python ${file_name} --start_master # 添加任务 + # python ${file_name} --start_worker # 启动爬虫 \ No newline at end of file diff --git a/feapder/templates/update_item_template.tmpl b/feapder/templates/update_item_template.tmpl new file mode 100644 index 00000000..a65f478d --- /dev/null +++ b/feapder/templates/update_item_template.tmpl @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +""" +Created on {DATE} +--------- +@summary: +--------- +@author: {USER} +""" + +from feapder import UpdateItem + + +class ${item_name}Item(UpdateItem): + """ + This class was generated by feapder + command: feapder create -i ${command} + """ + + __table_name__ = "${table_name}" + + def __init__(self, *args, **kwargs): + ${propertys} diff --git a/feapder/utils/__init__.py b/feapder/utils/__init__.py index 37f64b71..6f9647eb 100644 --- a/feapder/utils/__init__.py +++ b/feapder/utils/__init__.py @@ -5,5 +5,5 @@ @summary: --------- @author: Boris -@email: boris@bzkj.tech +@email: boris_liu@foxmail.com ''' \ No newline at end of file diff --git a/feapder/utils/email_sender.py b/feapder/utils/email_sender.py new file mode 100644 index 00000000..ec284eb4 --- /dev/null +++ b/feapder/utils/email_sender.py @@ -0,0 +1,95 @@ +# -*- coding: utf-8 -*- +""" +Created on 2020/2/19 12:57 PM +--------- +@summary: +--------- +@author: Boris +@email: boris_liu@foxmail.com +""" + +import os +import smtplib +from email.header import Header +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText +from email.utils import formataddr + +from feapder.utils.log import log + + +class EmailSender(object): + SENDER = "feapder报警系统" + + def __init__(self, username, password, smtpserver="smtp.163.com"): + self.username = username + self.password = password + self.smtpserver = smtpserver + self.smtp_client = smtplib.SMTP_SSL(smtpserver) + self.sender = EmailSender.SENDER + + def __enter__(self): + self.login() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.quit() + + def quit(self): + self.smtp_client.quit() + + def login(self): + self.smtp_client.connect(self.smtpserver) + self.smtp_client.login(self.username, self.password) + + def send( + self, + receivers: list, + title: str, + content: str, + content_type: str = "plain", + filepath: str = None, + ): + """ + + Args: + receivers: + title: + content: + content_type: html / plain + filepath: + + Returns: + + """ + # 创建一个带附件的实例 + message = MIMEMultipart() + message["From"] = formataddr( + (self.sender, self.username) + ) # 括号里的对应发件人邮箱昵称、发件人邮箱账号 + message["To"] = ",".join( + [formataddr((receiver, receiver)) for receiver in receivers] + ) + + message["Subject"] = Header(title, "utf-8") + + content = MIMEText(content, content_type, "utf-8") + message.attach(content) + + # 构造附件 + if filepath: + attach = MIMEText(open(filepath, "rb").read(), "base64", "utf-8") + attach.add_header( + "content-disposition", + "attachment", + filename=("utf-8", "", os.path.basename(filepath)), + ) + message.attach(attach) + + msg = message.as_string() + # 此处直接发送多个邮箱有问题，改成一个个发送 + for receiver in receivers: + log.debug("发送邮件到 {}".format(receiver)) + self.smtp_client.sendmail(self.username, receiver, msg) + log.debug("邮件发送成功！！！") + return True diff --git a/feapder/utils/export_data.py b/feapder/utils/export_data.py deleted file mode 100644 index c83cb155..00000000 --- a/feapder/utils/export_data.py +++ /dev/null @@ -1,167 +0,0 @@ -# -*- coding: utf-8 -*- -""" -Created on 2018-07-29 22:48:30 ---------- -@summary: 导出数据 ---------- -@author: Boris -@email: boris_liu@foxmail.com -""" -import feapder.utils.tools as tools -from feapder.db.mysqldb import MysqlDB -from feapder.db.redisdb import RedisDB -from feapder.utils.log import log - - -class ExportData(object): - def __init__(self): - self._redisdb = None - self._to_db = None - - @property - def redisdb(self): - if not self._redisdb: - self._redisdb = RedisDB() - - return self._redisdb - - @property - def to_db(self): - if not self._to_db: - self._to_db = MysqlDB() - - return self._to_db - - def export(self, from_table, to_table, auto_update=False, batch_count=100): - """ - @summary: - 用于从redis的item中导出数据到关系型数据库，如mysql/oracle - from_table与to_table表结构必须一致 - --------- - @param from_table: - @param to_table: - @param auto_update: 当数据存在时是否自动更新 默认否 - --------- - @result: - """ - total_count = 0 - - while True: - datas = [] - try: - datas = self.redisdb.sget(from_table, count=batch_count, is_pop=False) - if not datas: - log.info( - """ - \r%s -> %s 共导出 %s 条数据""" - % (from_table, to_table, total_count) - ) - break - - json_datas = [eval(data) for data in datas] - sql, json_datas = tools.make_batch_sql( - to_table, json_datas, auto_update - ) - if self.to_db.add_batch(sql, json_datas): - total_count += len(json_datas) - self.redisdb.srem(from_table, datas) - - except Exception as e: - log.exception(e) - log.error(datas) - - def export_all( - self, - tables, - auto_update=False, - batch_count=100, - every_table_per_export_callback=None, - ): - """ - @summary: 导出所有item - --------- - @param tables: 如qidian 则导出起点下面所有的items - 数据库中的表格式必须有规律 如导出 qidian:comment:s_qidian_book_comment_dynamic_item 对应导入 qidian_book_comment_dynamic - @param auto_update: 是否自动更新 - @param batch_count: 每批次导出的数量 - @every_table_per_export_callback: 导出前的回调函数, 用来修改特定表的参数 to_table, auto_update, batch_count - 如： - def every_table_per_export_callback(to_table, auto_update, batch_count): - if to_table == 'xxx': - auto_update = True - return to_table, auto_update, batch_count - --------- - @result: - """ - tables = ( - self.redisdb.getkeys(tables + "*_item") - if not isinstance(tables, list) - else tables - ) - if not tables: - log.info("无表数据") - for table in tables: - from_table = table - to_table = tools.get_info(str(from_table), ":s_(.*?)_item$", fetch_one=True) - if callable(every_table_per_export_callback): - to_table, auto_update, batch_count = every_table_per_export_callback( - to_table, auto_update, batch_count - ) - - log.info( - """ - \r正在导出 %s -> %s""" - % (from_table, to_table) - ) - self.export(from_table, to_table, auto_update, batch_count) - - def export_items(self, tab_item, items_data): - """ - @summary: - --------- - @param tab_item: redis中items的表名 - @param items_data: [item.to_dict] 数据 - --------- - @result: - """ - - to_table = tools.get_info(tab_item, ":s_(.*?)_item$", fetch_one=True) - sql, datas = tools.make_batch_sql(to_table, items_data) - add_count = self.to_db.add_batch(sql, datas) - datas_size = len(datas) - if add_count is None: - log.error("导出数据到表 %s 失败" % (to_table)) - else: - log.info( - "共导出 %s 条数据 到 %s, 重复 %s 条" - % (datas_size, to_table, datas_size - add_count) - ) - - return add_count != None - - def update_items(self, tab_item, items_data, update_keys=()): - """ - @summary: - --------- - @param tab_item: redis中items的表名 - @param items_data: [item.to_dict] 数据 - @param update_keys: 更新的字段 - --------- - @result: - """ - to_table = tools.get_info(tab_item, ":s_(.*?)_item$", fetch_one=True) - sql, datas = tools.make_batch_sql( - to_table, - items_data, - update_columns=update_keys or list(items_data[0].keys()), - ) - update_count = self.to_db.add_batch(sql, datas) - if update_count is None: - log.error("更新表 %s 数据失败" % (to_table)) - else: - msg = "共更新 %s 条数据 到 %s" % (update_count // 2, to_table) - if update_keys: - msg += " 更新字段为 {}".format(update_keys) - log.info(msg) - - return update_count != None diff --git a/feapder/utils/js/intercept.js b/feapder/utils/js/intercept.js new file mode 100644 index 00000000..6685e674 --- /dev/null +++ b/feapder/utils/js/intercept.js @@ -0,0 +1,41 @@ +!function(t,e){for(var n in e)t[n]=e[n]}(window,function(t){function e(r){if(n[r])return n[r].exports;var o=n[r]={i:r,l:!1,exports:{}};return t[r].call(o.exports,o,o.exports,e),o.l=!0,o.exports}var n={};return e.m=t,e.c=n,e.i=function(t){return t},e.d=function(t,n,r){e.o(t,n)||Object.defineProperty(t,n,{configurable:!1,enumerable:!0,get:r})},e.n=function(t){var n=t&&t.__esModule?function(){return t.default}:function(){return t};return e.d(n,"a",n),n},e.o=function(t,e){return Object.prototype.hasOwnProperty.call(t,e)},e.p="",e(e.s=3)}([function(t,e,n){"use strict";function r(t,e){var n={};for(var r in t)n[r]=t[r];return n.target=n.currentTarget=e,n}function o(t){function e(e){return function(){var n=this.hasOwnProperty(e+"_")?this[e+"_"]:this.xhr[e],r=(t[e]||{}).getter;return r&&r(n,this)||n}}function n(e){return function(n){var o=this.xhr,i=this,u=t[e];if("on"===e.substring(0,2))i[e+"_"]=n,o[e]=function(u){u=r(u,i),t[e]&&t[e].call(i,o,u)||n.call(i,u)};else{var s=(u||{}).setter;n=s&&s(n,i)||n,this[e+"_"]=n;try{o[e]=n}catch(t){}}}}function o(e){return function(){var n=[].slice.call(arguments);if(t[e]){var r=t[e].call(this,n,this.xhr);if(r)return r}return this.xhr[e].apply(this.xhr,n)}}return window[s]=window[s]||XMLHttpRequest,XMLHttpRequest=function(){var t=new window[s];for(var r in t){var i="";try{i=u(t[r])}catch(t){}"function"===i?this[r]=o(r):Object.defineProperty(this,r,{get:e(r),set:n(r),enumerable:!0})}var a=this;t.getProxy=function(){return a},this.xhr=t},window[s]}function i(){window[s]&&(XMLHttpRequest=window[s]),window[s]=void 0}Object.defineProperty(e,"__esModule",{value:!0});var u="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol&&t!==Symbol.prototype?"symbol":typeof t};e.configEvent=r,e.hook=o,e.unHook=i;var s="_rxhr"},function(t,e,n){"use strict";function r(t){if(h)throw"Proxy already exists";return h=new f(t)}function o(){h=null,(0,d.unHook)()}function i(t){return t.replace(/^\s+|\s+$/g,"")}function u(t){return t.watcher||(t.watcher=document.createElement("a"))}function s(t,e){var n=t.getProxy(),r="on"+e+"_",o=(0,d.configEvent)({type:e},n);n[r]&&n[r](o);var i;"function"==typeof Event?i=new Event(e,{bubbles:!1}):(i=document.createEvent("Event"),i.initEvent(e,!1,!0)),u(t).dispatchEvent(i)}function a(t){this.xhr=t,this.xhrProxy=t.getProxy()}function c(t){function e(t){a.call(this,t)}return e[b]=Object.create(a[b]),e[b].next=t,e}function f(t){function e(t,e){var n=new P(t);if(!f)return n.resolve();var r={response:e.response,status:e.status,statusText:e.statusText,config:t.config,headers:t.resHeader||t.getAllResponseHeaders().split("\r\n").reduce(function(t,e){if(""===e)return t;var n=e.split(":");return t[n.shift()]=i(n.join(":")),t},{})};f(r,n)}function n(t,e,n){var r=new H(t),o={config:t.config,error:n};h?h(o,r):r.next(o)}function r(){return!0}function o(t,e){return n(t,this,e),!0}function a(t,n){return 4===t.readyState&&0!==t.status?e(t,n):4!==t.readyState&&s(t,w),!0}var c=t.onRequest,f=t.onResponse,h=t.onError;return(0,d.hook)({onload:r,onloadend:r,onerror:o,ontimeout:o,onabort:o,onreadystatechange:function(t){return a(t,this)},open:function(t,e){var r=this,o=e.config={headers:{}};o.method=t[0],o.url=t[1],o.async=t[2],o.user=t[3],o.password=t[4],o.xhr=e;var i="on"+w;e[i]||(e[i]=function(){return a(e,r)});var u=function(t){n(e,r,(0,d.configEvent)(t,r))};if([x,y,g].forEach(function(t){var n="on"+t;e[n]||(e[n]=u)}),c)return!0},send:function(t,e){var n=e.config;if(n.withCredentials=e.withCredentials,n.body=t[0],c){var r=function(){c(n,new m(e))};return!1===n.async?r():setTimeout(r),!0}},setRequestHeader:function(t,e){return e.config.headers[t[0].toLowerCase()]=t[1],!0},addEventListener:function(t,e){var n=this;if(-1!==l.indexOf(t[0])){var r=t[1];return u(e).addEventListener(t[0],function(e){var o=(0,d.configEvent)(e,n);o.type=t[0],o.isTrusted=!0,r.call(n,o)}),!0}},getAllResponseHeaders:function(t,e){var n=e.resHeader;if(n){var r="";for(var o in n)r+=o+": "+n[o]+"\r\n";return r}},getResponseHeader:function(t,e){var n=e.resHeader;if(n)return n[(t[0]||"").toLowerCase()]}})}Object.defineProperty(e,"__esModule",{value:!0}),e.proxy=r,e.unProxy=o;var h,d=n(0),l=["load","loadend","timeout","error","readystatechange","abort"],v=l[0],p=l[1],y=l[2],x=l[3],w=l[4],g=l[5],b="prototype";a[b]=Object.create({resolve:function(t){var e=this.xhrProxy,n=this.xhr;e.readyState=4,n.resHeader=t.headers,e.response=e.responseText=t.response,e.statusText=t.statusText,e.status=t.status,s(n,w),s(n,v),s(n,p)},reject:function(t){this.xhrProxy.status=0,s(this.xhr,t.type),s(this.xhr,p)}});var m=c(function(t){var e=this.xhr;t=t||e.config,e.withCredentials=t.withCredentials,e.open(t.method,t.url,!1!==t.async,t.user,t.password);for(var n in t.headers)e.setRequestHeader(n,t.headers[n]);e.send(t.body)}),P=c(function(t){this.resolve(t)}),H=c(function(t){this.reject(t)})},,function(t,e,n){"use strict";Object.defineProperty(e,"__esModule",{value:!0}),e.ah=void 0;var r=n(0),o=n(1);e.ah={proxy:o.proxy,unProxy:o.unProxy,hook:r.hook,unHook:r.unHook}}])); + +// 存储数据 +window.__ajaxData = {} +// 拦截规则 +window.__urlRegexes = [] + +ah.proxy({ + onRequest: (config,handler)=>{ + handler.next(config); + } + , + onError: (err,handler)=>{ + handler.next(err) + } + , + onResponse: (response,handler)=>{ + var url = response.config.url; + if (window.__urlRegexes.length > 0) { + for (const regex of window.__urlRegexes) { + var re = new RegExp(regex, "g"); + if (re.exec(url)) { + window.__ajaxData[regex] = { + request: { + 'url': response.config.xhr.config.url, + 'data': response.config.xhr.config.body, + 'headers': response.config.xhr.config.headers + }, + response: { + 'url': response.config.url, + 'headers': response.headers, + 'content': response.response, + 'status_code': response.status + } + }; + } + } + } + handler.next(response) + } +}) diff --git a/feapder/utils/js/stealth.min.js b/feapder/utils/js/stealth.min.js new file mode 100644 index 00000000..91784572 --- /dev/null +++ b/feapder/utils/js/stealth.min.js @@ -0,0 +1,7 @@ +/*! + * Note: Auto-generated, do not update manually. + * Generated by: https://github.com/berstend/puppeteer-extra/tree/master/packages/extract-stealth-evasions + * Generated on: Sun, 24 Apr 2022 12:07:11 GMT + * License: MIT + */ +(({_utilsFns:_utilsFns,_mainFunction:_mainFunction,_args:_args})=>{const utils=Object.fromEntries(Object.entries(_utilsFns).map((([key,value])=>[key,eval(value)])));utils.init(),eval(_mainFunction)(utils,..._args)})({_utilsFns:{init:"() => {\n utils.preloadCache()\n}",stripProxyFromErrors:"(handler = {}) => {\n const newHandler = {\n setPrototypeOf: function (target, proto) {\n if (proto === null)\n throw new TypeError('Cannot convert object to primitive value')\n if (Object.getPrototypeOf(target) === Object.getPrototypeOf(proto)) {\n throw new TypeError('Cyclic __proto__ value')\n }\n return Reflect.setPrototypeOf(target, proto)\n }\n }\n // We wrap each trap in the handler in a try/catch and modify the error stack if they throw\n const traps = Object.getOwnPropertyNames(handler)\n traps.forEach(trap => {\n newHandler[trap] = function () {\n try {\n // Forward the call to the defined proxy handler\n return handler[trap].apply(this, arguments || [])\n } catch (err) {\n // Stack traces differ per browser, we only support chromium based ones currently\n if (!err || !err.stack || !err.stack.includes(`at `)) {\n throw err\n }\n\n // When something throws within one of our traps the Proxy will show up in error stacks\n // An earlier implementation of this code would simply strip lines with a blacklist,\n // but it makes sense to be more surgical here and only remove lines related to our Proxy.\n // We try to use a known \"anchor\" line for that and strip it with everything above it.\n // If the anchor line cannot be found for some reason we fall back to our blacklist approach.\n\n const stripWithBlacklist = (stack, stripFirstLine = true) => {\n const blacklist = [\n `at Reflect.${trap} `, // e.g. Reflect.get or Reflect.apply\n `at Object.${trap} `, // e.g. Object.get or Object.apply\n `at Object.newHandler. [as ${trap}] ` // caused by this very wrapper :-)\n ]\n return (\n err.stack\n .split('\\n')\n // Always remove the first (file) line in the stack (guaranteed to be our proxy)\n .filter((line, index) => !(index === 1 && stripFirstLine))\n // Check if the line starts with one of our blacklisted strings\n .filter(line => !blacklist.some(bl => line.trim().startsWith(bl)))\n .join('\\n')\n )\n }\n\n const stripWithAnchor = (stack, anchor) => {\n const stackArr = stack.split('\\n')\n anchor = anchor || `at Object.newHandler. [as ${trap}] ` // Known first Proxy line in chromium\n const anchorIndex = stackArr.findIndex(line =>\n line.trim().startsWith(anchor)\n )\n if (anchorIndex === -1) {\n return false // 404, anchor not found\n }\n // Strip everything from the top until we reach the anchor line\n // Note: We're keeping the 1st line (zero index) as it's unrelated (e.g. `TypeError`)\n stackArr.splice(1, anchorIndex)\n return stackArr.join('\\n')\n }\n\n // Special cases due to our nested toString proxies\n err.stack = err.stack.replace(\n 'at Object.toString (',\n 'at Function.toString ('\n )\n if ((err.stack || '').includes('at Function.toString (')) {\n err.stack = stripWithBlacklist(err.stack, false)\n throw err\n }\n\n // Try using the anchor method, fallback to blacklist if necessary\n err.stack = stripWithAnchor(err.stack) || stripWithBlacklist(err.stack)\n\n throw err // Re-throw our now sanitized error\n }\n }\n })\n return newHandler\n}",stripErrorWithAnchor:"(err, anchor) => {\n const stackArr = err.stack.split('\\n')\n const anchorIndex = stackArr.findIndex(line => line.trim().startsWith(anchor))\n if (anchorIndex === -1) {\n return err // 404, anchor not found\n }\n // Strip everything from the top until we reach the anchor line (remove anchor line as well)\n // Note: We're keeping the 1st line (zero index) as it's unrelated (e.g. `TypeError`)\n stackArr.splice(1, anchorIndex)\n err.stack = stackArr.join('\\n')\n return err\n}",replaceProperty:"(obj, propName, descriptorOverrides = {}) => {\n return Object.defineProperty(obj, propName, {\n // Copy over the existing descriptors (writable, enumerable, configurable, etc)\n ...(Object.getOwnPropertyDescriptor(obj, propName) || {}),\n // Add our overrides (e.g. value, get())\n ...descriptorOverrides\n })\n}",preloadCache:"() => {\n if (utils.cache) {\n return\n }\n utils.cache = {\n // Used in our proxies\n Reflect: {\n get: Reflect.get.bind(Reflect),\n apply: Reflect.apply.bind(Reflect)\n },\n // Used in `makeNativeString`\n nativeToStringStr: Function.toString + '' // => `function toString() { [native code] }`\n }\n}",makeNativeString:"(name = '') => {\n return utils.cache.nativeToStringStr.replace('toString', name || '')\n}",patchToString:"(obj, str = '') => {\n const handler = {\n apply: function (target, ctx) {\n // This fixes e.g. `HTMLMediaElement.prototype.canPlayType.toString + \"\"`\n if (ctx === Function.prototype.toString) {\n return utils.makeNativeString('toString')\n }\n // `toString` targeted at our proxied Object detected\n if (ctx === obj) {\n // We either return the optional string verbatim or derive the most desired result automatically\n return str || utils.makeNativeString(obj.name)\n }\n // Check if the toString protype of the context is the same as the global prototype,\n // if not indicates that we are doing a check across different windows., e.g. the iframeWithdirect` test case\n const hasSameProto = Object.getPrototypeOf(\n Function.prototype.toString\n ).isPrototypeOf(ctx.toString) // eslint-disable-line no-prototype-builtins\n if (!hasSameProto) {\n // Pass the call on to the local Function.prototype.toString instead\n return ctx.toString()\n }\n return target.call(ctx)\n }\n }\n\n const toStringProxy = new Proxy(\n Function.prototype.toString,\n utils.stripProxyFromErrors(handler)\n )\n utils.replaceProperty(Function.prototype, 'toString', {\n value: toStringProxy\n })\n}",patchToStringNested:"(obj = {}) => {\n return utils.execRecursively(obj, ['function'], utils.patchToString)\n}",redirectToString:"(proxyObj, originalObj) => {\n const handler = {\n apply: function (target, ctx) {\n // This fixes e.g. `HTMLMediaElement.prototype.canPlayType.toString + \"\"`\n if (ctx === Function.prototype.toString) {\n return utils.makeNativeString('toString')\n }\n\n // `toString` targeted at our proxied Object detected\n if (ctx === proxyObj) {\n const fallback = () =>\n originalObj && originalObj.name\n ? utils.makeNativeString(originalObj.name)\n : utils.makeNativeString(proxyObj.name)\n\n // Return the toString representation of our original object if possible\n return originalObj + '' || fallback()\n }\n\n if (typeof ctx === 'undefined' || ctx === null) {\n return target.call(ctx)\n }\n\n // Check if the toString protype of the context is the same as the global prototype,\n // if not indicates that we are doing a check across different windows., e.g. the iframeWithdirect` test case\n const hasSameProto = Object.getPrototypeOf(\n Function.prototype.toString\n ).isPrototypeOf(ctx.toString) // eslint-disable-line no-prototype-builtins\n if (!hasSameProto) {\n // Pass the call on to the local Function.prototype.toString instead\n return ctx.toString()\n }\n\n return target.call(ctx)\n }\n }\n\n const toStringProxy = new Proxy(\n Function.prototype.toString,\n utils.stripProxyFromErrors(handler)\n )\n utils.replaceProperty(Function.prototype, 'toString', {\n value: toStringProxy\n })\n}",replaceWithProxy:"(obj, propName, handler) => {\n const originalObj = obj[propName]\n const proxyObj = new Proxy(obj[propName], utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { value: proxyObj })\n utils.redirectToString(proxyObj, originalObj)\n\n return true\n}",replaceGetterWithProxy:"(obj, propName, handler) => {\n const fn = Object.getOwnPropertyDescriptor(obj, propName).get\n const fnStr = fn.toString() // special getter function string\n const proxyObj = new Proxy(fn, utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { get: proxyObj })\n utils.patchToString(proxyObj, fnStr)\n\n return true\n}",mockWithProxy:"(obj, propName, pseudoTarget, handler) => {\n const proxyObj = new Proxy(pseudoTarget, utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { value: proxyObj })\n utils.patchToString(proxyObj)\n\n return true\n}",createProxy:"(pseudoTarget, handler) => {\n const proxyObj = new Proxy(pseudoTarget, utils.stripProxyFromErrors(handler))\n utils.patchToString(proxyObj)\n\n return proxyObj\n}",splitObjPath:"objPath => ({\n // Remove last dot entry (property) ==> `HTMLMediaElement.prototype`\n objName: objPath.split('.').slice(0, -1).join('.'),\n // Extract last dot entry ==> `canPlayType`\n propName: objPath.split('.').slice(-1)[0]\n})",replaceObjPathWithProxy:"(objPath, handler) => {\n const { objName, propName } = utils.splitObjPath(objPath)\n const obj = eval(objName) // eslint-disable-line no-eval\n return utils.replaceWithProxy(obj, propName, handler)\n}",execRecursively:"(obj = {}, typeFilter = [], fn) => {\n function recurse(obj) {\n for (const key in obj) {\n if (obj[key] === undefined) {\n continue\n }\n if (obj[key] && typeof obj[key] === 'object') {\n recurse(obj[key])\n } else {\n if (obj[key] && typeFilter.includes(typeof obj[key])) {\n fn.call(this, obj[key])\n }\n }\n }\n }\n recurse(obj)\n return obj\n}",stringifyFns:"(fnObj = { hello: () => 'world' }) => {\n // Object.fromEntries() ponyfill (in 6 lines) - supported only in Node v12+, modern browsers are fine\n // https://github.com/feross/fromentries\n function fromEntries(iterable) {\n return [...iterable].reduce((obj, [key, val]) => {\n obj[key] = val\n return obj\n }, {})\n }\n return (Object.fromEntries || fromEntries)(\n Object.entries(fnObj)\n .filter(([key, value]) => typeof value === 'function')\n .map(([key, value]) => [key, value.toString()]) // eslint-disable-line no-eval\n )\n}",materializeFns:"(fnStrObj = { hello: \"() => 'world'\" }) => {\n return Object.fromEntries(\n Object.entries(fnStrObj).map(([key, value]) => {\n if (value.startsWith('function')) {\n // some trickery is needed to make oldschool functions work :-)\n return [key, eval(`() => ${value}`)()] // eslint-disable-line no-eval\n } else {\n // arrow functions just work\n return [key, eval(value)] // eslint-disable-line no-eval\n }\n })\n )\n}",makeHandler:"() => ({\n // Used by simple `navigator` getter evasions\n getterValue: value => ({\n apply(target, ctx, args) {\n // Let's fetch the value first, to trigger and escalate potential errors\n // Illegal invocations like `navigator.__proto__.vendor` will throw here\n utils.cache.Reflect.apply(...arguments)\n return value\n }\n })\n})"},_mainFunction:'utils => {\n if (!window.chrome) {\n // Use the exact property descriptor found in headful Chrome\n // fetch it via `Object.getOwnPropertyDescriptor(window, \'chrome\')`\n Object.defineProperty(window, \'chrome\', {\n writable: true,\n enumerable: true,\n configurable: false, // note!\n value: {} // We\'ll extend that later\n })\n }\n\n // That means we\'re running headful and don\'t need to mock anything\n if (\'app\' in window.chrome) {\n return // Nothing to do here\n }\n\n const makeError = {\n ErrorInInvocation: fn => {\n const err = new TypeError(`Error in invocation of app.${fn}()`)\n return utils.stripErrorWithAnchor(\n err,\n `at ${fn} (eval at `\n )\n }\n }\n\n // There\'s a some static data in that property which doesn\'t seem to change,\n // we should periodically check for updates: `JSON.stringify(window.app, null, 2)`\n const STATIC_DATA = JSON.parse(\n `\n{\n "isInstalled": false,\n "InstallState": {\n "DISABLED": "disabled",\n "INSTALLED": "installed",\n "NOT_INSTALLED": "not_installed"\n },\n "RunningState": {\n "CANNOT_RUN": "cannot_run",\n "READY_TO_RUN": "ready_to_run",\n "RUNNING": "running"\n }\n}\n `.trim()\n )\n\n window.chrome.app = {\n ...STATIC_DATA,\n\n get isInstalled() {\n return false\n },\n\n getDetails: function getDetails() {\n if (arguments.length) {\n throw makeError.ErrorInInvocation(`getDetails`)\n }\n return null\n },\n getIsInstalled: function getDetails() {\n if (arguments.length) {\n throw makeError.ErrorInInvocation(`getIsInstalled`)\n }\n return false\n },\n runningState: function getDetails() {\n if (arguments.length) {\n throw makeError.ErrorInInvocation(`runningState`)\n }\n return \'cannot_run\'\n }\n }\n utils.patchToStringNested(window.chrome.app)\n }',_args:[]}),(({_utilsFns:_utilsFns,_mainFunction:_mainFunction,_args:_args})=>{const utils=Object.fromEntries(Object.entries(_utilsFns).map((([key,value])=>[key,eval(value)])));utils.init(),eval(_mainFunction)(utils,..._args)})({_utilsFns:{init:"() => {\n utils.preloadCache()\n}",stripProxyFromErrors:"(handler = {}) => {\n const newHandler = {\n setPrototypeOf: function (target, proto) {\n if (proto === null)\n throw new TypeError('Cannot convert object to primitive value')\n if (Object.getPrototypeOf(target) === Object.getPrototypeOf(proto)) {\n throw new TypeError('Cyclic __proto__ value')\n }\n return Reflect.setPrototypeOf(target, proto)\n }\n }\n // We wrap each trap in the handler in a try/catch and modify the error stack if they throw\n const traps = Object.getOwnPropertyNames(handler)\n traps.forEach(trap => {\n newHandler[trap] = function () {\n try {\n // Forward the call to the defined proxy handler\n return handler[trap].apply(this, arguments || [])\n } catch (err) {\n // Stack traces differ per browser, we only support chromium based ones currently\n if (!err || !err.stack || !err.stack.includes(`at `)) {\n throw err\n }\n\n // When something throws within one of our traps the Proxy will show up in error stacks\n // An earlier implementation of this code would simply strip lines with a blacklist,\n // but it makes sense to be more surgical here and only remove lines related to our Proxy.\n // We try to use a known \"anchor\" line for that and strip it with everything above it.\n // If the anchor line cannot be found for some reason we fall back to our blacklist approach.\n\n const stripWithBlacklist = (stack, stripFirstLine = true) => {\n const blacklist = [\n `at Reflect.${trap} `, // e.g. Reflect.get or Reflect.apply\n `at Object.${trap} `, // e.g. Object.get or Object.apply\n `at Object.newHandler. [as ${trap}] ` // caused by this very wrapper :-)\n ]\n return (\n err.stack\n .split('\\n')\n // Always remove the first (file) line in the stack (guaranteed to be our proxy)\n .filter((line, index) => !(index === 1 && stripFirstLine))\n // Check if the line starts with one of our blacklisted strings\n .filter(line => !blacklist.some(bl => line.trim().startsWith(bl)))\n .join('\\n')\n )\n }\n\n const stripWithAnchor = (stack, anchor) => {\n const stackArr = stack.split('\\n')\n anchor = anchor || `at Object.newHandler. [as ${trap}] ` // Known first Proxy line in chromium\n const anchorIndex = stackArr.findIndex(line =>\n line.trim().startsWith(anchor)\n )\n if (anchorIndex === -1) {\n return false // 404, anchor not found\n }\n // Strip everything from the top until we reach the anchor line\n // Note: We're keeping the 1st line (zero index) as it's unrelated (e.g. `TypeError`)\n stackArr.splice(1, anchorIndex)\n return stackArr.join('\\n')\n }\n\n // Special cases due to our nested toString proxies\n err.stack = err.stack.replace(\n 'at Object.toString (',\n 'at Function.toString ('\n )\n if ((err.stack || '').includes('at Function.toString (')) {\n err.stack = stripWithBlacklist(err.stack, false)\n throw err\n }\n\n // Try using the anchor method, fallback to blacklist if necessary\n err.stack = stripWithAnchor(err.stack) || stripWithBlacklist(err.stack)\n\n throw err // Re-throw our now sanitized error\n }\n }\n })\n return newHandler\n}",stripErrorWithAnchor:"(err, anchor) => {\n const stackArr = err.stack.split('\\n')\n const anchorIndex = stackArr.findIndex(line => line.trim().startsWith(anchor))\n if (anchorIndex === -1) {\n return err // 404, anchor not found\n }\n // Strip everything from the top until we reach the anchor line (remove anchor line as well)\n // Note: We're keeping the 1st line (zero index) as it's unrelated (e.g. `TypeError`)\n stackArr.splice(1, anchorIndex)\n err.stack = stackArr.join('\\n')\n return err\n}",replaceProperty:"(obj, propName, descriptorOverrides = {}) => {\n return Object.defineProperty(obj, propName, {\n // Copy over the existing descriptors (writable, enumerable, configurable, etc)\n ...(Object.getOwnPropertyDescriptor(obj, propName) || {}),\n // Add our overrides (e.g. value, get())\n ...descriptorOverrides\n })\n}",preloadCache:"() => {\n if (utils.cache) {\n return\n }\n utils.cache = {\n // Used in our proxies\n Reflect: {\n get: Reflect.get.bind(Reflect),\n apply: Reflect.apply.bind(Reflect)\n },\n // Used in `makeNativeString`\n nativeToStringStr: Function.toString + '' // => `function toString() { [native code] }`\n }\n}",makeNativeString:"(name = '') => {\n return utils.cache.nativeToStringStr.replace('toString', name || '')\n}",patchToString:"(obj, str = '') => {\n const handler = {\n apply: function (target, ctx) {\n // This fixes e.g. `HTMLMediaElement.prototype.canPlayType.toString + \"\"`\n if (ctx === Function.prototype.toString) {\n return utils.makeNativeString('toString')\n }\n // `toString` targeted at our proxied Object detected\n if (ctx === obj) {\n // We either return the optional string verbatim or derive the most desired result automatically\n return str || utils.makeNativeString(obj.name)\n }\n // Check if the toString protype of the context is the same as the global prototype,\n // if not indicates that we are doing a check across different windows., e.g. the iframeWithdirect` test case\n const hasSameProto = Object.getPrototypeOf(\n Function.prototype.toString\n ).isPrototypeOf(ctx.toString) // eslint-disable-line no-prototype-builtins\n if (!hasSameProto) {\n // Pass the call on to the local Function.prototype.toString instead\n return ctx.toString()\n }\n return target.call(ctx)\n }\n }\n\n const toStringProxy = new Proxy(\n Function.prototype.toString,\n utils.stripProxyFromErrors(handler)\n )\n utils.replaceProperty(Function.prototype, 'toString', {\n value: toStringProxy\n })\n}",patchToStringNested:"(obj = {}) => {\n return utils.execRecursively(obj, ['function'], utils.patchToString)\n}",redirectToString:"(proxyObj, originalObj) => {\n const handler = {\n apply: function (target, ctx) {\n // This fixes e.g. `HTMLMediaElement.prototype.canPlayType.toString + \"\"`\n if (ctx === Function.prototype.toString) {\n return utils.makeNativeString('toString')\n }\n\n // `toString` targeted at our proxied Object detected\n if (ctx === proxyObj) {\n const fallback = () =>\n originalObj && originalObj.name\n ? utils.makeNativeString(originalObj.name)\n : utils.makeNativeString(proxyObj.name)\n\n // Return the toString representation of our original object if possible\n return originalObj + '' || fallback()\n }\n\n if (typeof ctx === 'undefined' || ctx === null) {\n return target.call(ctx)\n }\n\n // Check if the toString protype of the context is the same as the global prototype,\n // if not indicates that we are doing a check across different windows., e.g. the iframeWithdirect` test case\n const hasSameProto = Object.getPrototypeOf(\n Function.prototype.toString\n ).isPrototypeOf(ctx.toString) // eslint-disable-line no-prototype-builtins\n if (!hasSameProto) {\n // Pass the call on to the local Function.prototype.toString instead\n return ctx.toString()\n }\n\n return target.call(ctx)\n }\n }\n\n const toStringProxy = new Proxy(\n Function.prototype.toString,\n utils.stripProxyFromErrors(handler)\n )\n utils.replaceProperty(Function.prototype, 'toString', {\n value: toStringProxy\n })\n}",replaceWithProxy:"(obj, propName, handler) => {\n const originalObj = obj[propName]\n const proxyObj = new Proxy(obj[propName], utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { value: proxyObj })\n utils.redirectToString(proxyObj, originalObj)\n\n return true\n}",replaceGetterWithProxy:"(obj, propName, handler) => {\n const fn = Object.getOwnPropertyDescriptor(obj, propName).get\n const fnStr = fn.toString() // special getter function string\n const proxyObj = new Proxy(fn, utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { get: proxyObj })\n utils.patchToString(proxyObj, fnStr)\n\n return true\n}",mockWithProxy:"(obj, propName, pseudoTarget, handler) => {\n const proxyObj = new Proxy(pseudoTarget, utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { value: proxyObj })\n utils.patchToString(proxyObj)\n\n return true\n}",createProxy:"(pseudoTarget, handler) => {\n const proxyObj = new Proxy(pseudoTarget, utils.stripProxyFromErrors(handler))\n utils.patchToString(proxyObj)\n\n return proxyObj\n}",splitObjPath:"objPath => ({\n // Remove last dot entry (property) ==> `HTMLMediaElement.prototype`\n objName: objPath.split('.').slice(0, -1).join('.'),\n // Extract last dot entry ==> `canPlayType`\n propName: objPath.split('.').slice(-1)[0]\n})",replaceObjPathWithProxy:"(objPath, handler) => {\n const { objName, propName } = utils.splitObjPath(objPath)\n const obj = eval(objName) // eslint-disable-line no-eval\n return utils.replaceWithProxy(obj, propName, handler)\n}",execRecursively:"(obj = {}, typeFilter = [], fn) => {\n function recurse(obj) {\n for (const key in obj) {\n if (obj[key] === undefined) {\n continue\n }\n if (obj[key] && typeof obj[key] === 'object') {\n recurse(obj[key])\n } else {\n if (obj[key] && typeFilter.includes(typeof obj[key])) {\n fn.call(this, obj[key])\n }\n }\n }\n }\n recurse(obj)\n return obj\n}",stringifyFns:"(fnObj = { hello: () => 'world' }) => {\n // Object.fromEntries() ponyfill (in 6 lines) - supported only in Node v12+, modern browsers are fine\n // https://github.com/feross/fromentries\n function fromEntries(iterable) {\n return [...iterable].reduce((obj, [key, val]) => {\n obj[key] = val\n return obj\n }, {})\n }\n return (Object.fromEntries || fromEntries)(\n Object.entries(fnObj)\n .filter(([key, value]) => typeof value === 'function')\n .map(([key, value]) => [key, value.toString()]) // eslint-disable-line no-eval\n )\n}",materializeFns:"(fnStrObj = { hello: \"() => 'world'\" }) => {\n return Object.fromEntries(\n Object.entries(fnStrObj).map(([key, value]) => {\n if (value.startsWith('function')) {\n // some trickery is needed to make oldschool functions work :-)\n return [key, eval(`() => ${value}`)()] // eslint-disable-line no-eval\n } else {\n // arrow functions just work\n return [key, eval(value)] // eslint-disable-line no-eval\n }\n })\n )\n}",makeHandler:"() => ({\n // Used by simple `navigator` getter evasions\n getterValue: value => ({\n apply(target, ctx, args) {\n // Let's fetch the value first, to trigger and escalate potential errors\n // Illegal invocations like `navigator.__proto__.vendor` will throw here\n utils.cache.Reflect.apply(...arguments)\n return value\n }\n })\n})"},_mainFunction:"utils => {\n if (!window.chrome) {\n // Use the exact property descriptor found in headful Chrome\n // fetch it via `Object.getOwnPropertyDescriptor(window, 'chrome')`\n Object.defineProperty(window, 'chrome', {\n writable: true,\n enumerable: true,\n configurable: false, // note!\n value: {} // We'll extend that later\n })\n }\n\n // That means we're running headful and don't need to mock anything\n if ('csi' in window.chrome) {\n return // Nothing to do here\n }\n\n // Check that the Navigation Timing API v1 is available, we need that\n if (!window.performance || !window.performance.timing) {\n return\n }\n\n const { timing } = window.performance\n\n window.chrome.csi = function() {\n return {\n onloadT: timing.domContentLoadedEventEnd,\n startE: timing.navigationStart,\n pageT: Date.now() - timing.navigationStart,\n tran: 15 // Transition type or something\n }\n }\n utils.patchToString(window.chrome.csi)\n }",_args:[]}),(({_utilsFns:_utilsFns,_mainFunction:_mainFunction,_args:_args})=>{const utils=Object.fromEntries(Object.entries(_utilsFns).map((([key,value])=>[key,eval(value)])));utils.init(),eval(_mainFunction)(utils,..._args)})({_utilsFns:{init:"() => {\n utils.preloadCache()\n}",stripProxyFromErrors:"(handler = {}) => {\n const newHandler = {\n setPrototypeOf: function (target, proto) {\n if (proto === null)\n throw new TypeError('Cannot convert object to primitive value')\n if (Object.getPrototypeOf(target) === Object.getPrototypeOf(proto)) {\n throw new TypeError('Cyclic __proto__ value')\n }\n return Reflect.setPrototypeOf(target, proto)\n }\n }\n // We wrap each trap in the handler in a try/catch and modify the error stack if they throw\n const traps = Object.getOwnPropertyNames(handler)\n traps.forEach(trap => {\n newHandler[trap] = function () {\n try {\n // Forward the call to the defined proxy handler\n return handler[trap].apply(this, arguments || [])\n } catch (err) {\n // Stack traces differ per browser, we only support chromium based ones currently\n if (!err || !err.stack || !err.stack.includes(`at `)) {\n throw err\n }\n\n // When something throws within one of our traps the Proxy will show up in error stacks\n // An earlier implementation of this code would simply strip lines with a blacklist,\n // but it makes sense to be more surgical here and only remove lines related to our Proxy.\n // We try to use a known \"anchor\" line for that and strip it with everything above it.\n // If the anchor line cannot be found for some reason we fall back to our blacklist approach.\n\n const stripWithBlacklist = (stack, stripFirstLine = true) => {\n const blacklist = [\n `at Reflect.${trap} `, // e.g. Reflect.get or Reflect.apply\n `at Object.${trap} `, // e.g. Object.get or Object.apply\n `at Object.newHandler. [as ${trap}] ` // caused by this very wrapper :-)\n ]\n return (\n err.stack\n .split('\\n')\n // Always remove the first (file) line in the stack (guaranteed to be our proxy)\n .filter((line, index) => !(index === 1 && stripFirstLine))\n // Check if the line starts with one of our blacklisted strings\n .filter(line => !blacklist.some(bl => line.trim().startsWith(bl)))\n .join('\\n')\n )\n }\n\n const stripWithAnchor = (stack, anchor) => {\n const stackArr = stack.split('\\n')\n anchor = anchor || `at Object.newHandler. [as ${trap}] ` // Known first Proxy line in chromium\n const anchorIndex = stackArr.findIndex(line =>\n line.trim().startsWith(anchor)\n )\n if (anchorIndex === -1) {\n return false // 404, anchor not found\n }\n // Strip everything from the top until we reach the anchor line\n // Note: We're keeping the 1st line (zero index) as it's unrelated (e.g. `TypeError`)\n stackArr.splice(1, anchorIndex)\n return stackArr.join('\\n')\n }\n\n // Special cases due to our nested toString proxies\n err.stack = err.stack.replace(\n 'at Object.toString (',\n 'at Function.toString ('\n )\n if ((err.stack || '').includes('at Function.toString (')) {\n err.stack = stripWithBlacklist(err.stack, false)\n throw err\n }\n\n // Try using the anchor method, fallback to blacklist if necessary\n err.stack = stripWithAnchor(err.stack) || stripWithBlacklist(err.stack)\n\n throw err // Re-throw our now sanitized error\n }\n }\n })\n return newHandler\n}",stripErrorWithAnchor:"(err, anchor) => {\n const stackArr = err.stack.split('\\n')\n const anchorIndex = stackArr.findIndex(line => line.trim().startsWith(anchor))\n if (anchorIndex === -1) {\n return err // 404, anchor not found\n }\n // Strip everything from the top until we reach the anchor line (remove anchor line as well)\n // Note: We're keeping the 1st line (zero index) as it's unrelated (e.g. `TypeError`)\n stackArr.splice(1, anchorIndex)\n err.stack = stackArr.join('\\n')\n return err\n}",replaceProperty:"(obj, propName, descriptorOverrides = {}) => {\n return Object.defineProperty(obj, propName, {\n // Copy over the existing descriptors (writable, enumerable, configurable, etc)\n ...(Object.getOwnPropertyDescriptor(obj, propName) || {}),\n // Add our overrides (e.g. value, get())\n ...descriptorOverrides\n })\n}",preloadCache:"() => {\n if (utils.cache) {\n return\n }\n utils.cache = {\n // Used in our proxies\n Reflect: {\n get: Reflect.get.bind(Reflect),\n apply: Reflect.apply.bind(Reflect)\n },\n // Used in `makeNativeString`\n nativeToStringStr: Function.toString + '' // => `function toString() { [native code] }`\n }\n}",makeNativeString:"(name = '') => {\n return utils.cache.nativeToStringStr.replace('toString', name || '')\n}",patchToString:"(obj, str = '') => {\n const handler = {\n apply: function (target, ctx) {\n // This fixes e.g. `HTMLMediaElement.prototype.canPlayType.toString + \"\"`\n if (ctx === Function.prototype.toString) {\n return utils.makeNativeString('toString')\n }\n // `toString` targeted at our proxied Object detected\n if (ctx === obj) {\n // We either return the optional string verbatim or derive the most desired result automatically\n return str || utils.makeNativeString(obj.name)\n }\n // Check if the toString protype of the context is the same as the global prototype,\n // if not indicates that we are doing a check across different windows., e.g. the iframeWithdirect` test case\n const hasSameProto = Object.getPrototypeOf(\n Function.prototype.toString\n ).isPrototypeOf(ctx.toString) // eslint-disable-line no-prototype-builtins\n if (!hasSameProto) {\n // Pass the call on to the local Function.prototype.toString instead\n return ctx.toString()\n }\n return target.call(ctx)\n }\n }\n\n const toStringProxy = new Proxy(\n Function.prototype.toString,\n utils.stripProxyFromErrors(handler)\n )\n utils.replaceProperty(Function.prototype, 'toString', {\n value: toStringProxy\n })\n}",patchToStringNested:"(obj = {}) => {\n return utils.execRecursively(obj, ['function'], utils.patchToString)\n}",redirectToString:"(proxyObj, originalObj) => {\n const handler = {\n apply: function (target, ctx) {\n // This fixes e.g. `HTMLMediaElement.prototype.canPlayType.toString + \"\"`\n if (ctx === Function.prototype.toString) {\n return utils.makeNativeString('toString')\n }\n\n // `toString` targeted at our proxied Object detected\n if (ctx === proxyObj) {\n const fallback = () =>\n originalObj && originalObj.name\n ? utils.makeNativeString(originalObj.name)\n : utils.makeNativeString(proxyObj.name)\n\n // Return the toString representation of our original object if possible\n return originalObj + '' || fallback()\n }\n\n if (typeof ctx === 'undefined' || ctx === null) {\n return target.call(ctx)\n }\n\n // Check if the toString protype of the context is the same as the global prototype,\n // if not indicates that we are doing a check across different windows., e.g. the iframeWithdirect` test case\n const hasSameProto = Object.getPrototypeOf(\n Function.prototype.toString\n ).isPrototypeOf(ctx.toString) // eslint-disable-line no-prototype-builtins\n if (!hasSameProto) {\n // Pass the call on to the local Function.prototype.toString instead\n return ctx.toString()\n }\n\n return target.call(ctx)\n }\n }\n\n const toStringProxy = new Proxy(\n Function.prototype.toString,\n utils.stripProxyFromErrors(handler)\n )\n utils.replaceProperty(Function.prototype, 'toString', {\n value: toStringProxy\n })\n}",replaceWithProxy:"(obj, propName, handler) => {\n const originalObj = obj[propName]\n const proxyObj = new Proxy(obj[propName], utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { value: proxyObj })\n utils.redirectToString(proxyObj, originalObj)\n\n return true\n}",replaceGetterWithProxy:"(obj, propName, handler) => {\n const fn = Object.getOwnPropertyDescriptor(obj, propName).get\n const fnStr = fn.toString() // special getter function string\n const proxyObj = new Proxy(fn, utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { get: proxyObj })\n utils.patchToString(proxyObj, fnStr)\n\n return true\n}",mockWithProxy:"(obj, propName, pseudoTarget, handler) => {\n const proxyObj = new Proxy(pseudoTarget, utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { value: proxyObj })\n utils.patchToString(proxyObj)\n\n return true\n}",createProxy:"(pseudoTarget, handler) => {\n const proxyObj = new Proxy(pseudoTarget, utils.stripProxyFromErrors(handler))\n utils.patchToString(proxyObj)\n\n return proxyObj\n}",splitObjPath:"objPath => ({\n // Remove last dot entry (property) ==> `HTMLMediaElement.prototype`\n objName: objPath.split('.').slice(0, -1).join('.'),\n // Extract last dot entry ==> `canPlayType`\n propName: objPath.split('.').slice(-1)[0]\n})",replaceObjPathWithProxy:"(objPath, handler) => {\n const { objName, propName } = utils.splitObjPath(objPath)\n const obj = eval(objName) // eslint-disable-line no-eval\n return utils.replaceWithProxy(obj, propName, handler)\n}",execRecursively:"(obj = {}, typeFilter = [], fn) => {\n function recurse(obj) {\n for (const key in obj) {\n if (obj[key] === undefined) {\n continue\n }\n if (obj[key] && typeof obj[key] === 'object') {\n recurse(obj[key])\n } else {\n if (obj[key] && typeFilter.includes(typeof obj[key])) {\n fn.call(this, obj[key])\n }\n }\n }\n }\n recurse(obj)\n return obj\n}",stringifyFns:"(fnObj = { hello: () => 'world' }) => {\n // Object.fromEntries() ponyfill (in 6 lines) - supported only in Node v12+, modern browsers are fine\n // https://github.com/feross/fromentries\n function fromEntries(iterable) {\n return [...iterable].reduce((obj, [key, val]) => {\n obj[key] = val\n return obj\n }, {})\n }\n return (Object.fromEntries || fromEntries)(\n Object.entries(fnObj)\n .filter(([key, value]) => typeof value === 'function')\n .map(([key, value]) => [key, value.toString()]) // eslint-disable-line no-eval\n )\n}",materializeFns:"(fnStrObj = { hello: \"() => 'world'\" }) => {\n return Object.fromEntries(\n Object.entries(fnStrObj).map(([key, value]) => {\n if (value.startsWith('function')) {\n // some trickery is needed to make oldschool functions work :-)\n return [key, eval(`() => ${value}`)()] // eslint-disable-line no-eval\n } else {\n // arrow functions just work\n return [key, eval(value)] // eslint-disable-line no-eval\n }\n })\n )\n}",makeHandler:"() => ({\n // Used by simple `navigator` getter evasions\n getterValue: value => ({\n apply(target, ctx, args) {\n // Let's fetch the value first, to trigger and escalate potential errors\n // Illegal invocations like `navigator.__proto__.vendor` will throw here\n utils.cache.Reflect.apply(...arguments)\n return value\n }\n })\n})"},_mainFunction:"(utils, { opts }) => {\n if (!window.chrome) {\n // Use the exact property descriptor found in headful Chrome\n // fetch it via `Object.getOwnPropertyDescriptor(window, 'chrome')`\n Object.defineProperty(window, 'chrome', {\n writable: true,\n enumerable: true,\n configurable: false, // note!\n value: {} // We'll extend that later\n })\n }\n\n // That means we're running headful and don't need to mock anything\n if ('loadTimes' in window.chrome) {\n return // Nothing to do here\n }\n\n // Check that the Navigation Timing API v1 + v2 is available, we need that\n if (\n !window.performance ||\n !window.performance.timing ||\n !window.PerformancePaintTiming\n ) {\n return\n }\n\n const { performance } = window\n\n // Some stuff is not available on about:blank as it requires a navigation to occur,\n // let's harden the code to not fail then:\n const ntEntryFallback = {\n nextHopProtocol: 'h2',\n type: 'other'\n }\n\n // The API exposes some funky info regarding the connection\n const protocolInfo = {\n get connectionInfo() {\n const ntEntry =\n performance.getEntriesByType('navigation')[0] || ntEntryFallback\n return ntEntry.nextHopProtocol\n },\n get npnNegotiatedProtocol() {\n // NPN is deprecated in favor of ALPN, but this implementation returns the\n // HTTP/2 or HTTP2+QUIC/39 requests negotiated via ALPN.\n const ntEntry =\n performance.getEntriesByType('navigation')[0] || ntEntryFallback\n return ['h2', 'hq'].includes(ntEntry.nextHopProtocol)\n ? ntEntry.nextHopProtocol\n : 'unknown'\n },\n get navigationType() {\n const ntEntry =\n performance.getEntriesByType('navigation')[0] || ntEntryFallback\n return ntEntry.type\n },\n get wasAlternateProtocolAvailable() {\n // The Alternate-Protocol header is deprecated in favor of Alt-Svc\n // (https://www.mnot.net/blog/2016/03/09/alt-svc), so technically this\n // should always return false.\n return false\n },\n get wasFetchedViaSpdy() {\n // SPDY is deprecated in favor of HTTP/2, but this implementation returns\n // true for HTTP/2 or HTTP2+QUIC/39 as well.\n const ntEntry =\n performance.getEntriesByType('navigation')[0] || ntEntryFallback\n return ['h2', 'hq'].includes(ntEntry.nextHopProtocol)\n },\n get wasNpnNegotiated() {\n // NPN is deprecated in favor of ALPN, but this implementation returns true\n // for HTTP/2 or HTTP2+QUIC/39 requests negotiated via ALPN.\n const ntEntry =\n performance.getEntriesByType('navigation')[0] || ntEntryFallback\n return ['h2', 'hq'].includes(ntEntry.nextHopProtocol)\n }\n }\n\n const { timing } = window.performance\n\n // Truncate number to specific number of decimals, most of the `loadTimes` stuff has 3\n function toFixed(num, fixed) {\n var re = new RegExp('^-?\\\\d+(?:.\\\\d{0,' + (fixed || -1) + '})?')\n return num.toString().match(re)[0]\n }\n\n const timingInfo = {\n get firstPaintAfterLoadTime() {\n // This was never actually implemented and always returns 0.\n return 0\n },\n get requestTime() {\n return timing.navigationStart / 1000\n },\n get startLoadTime() {\n return timing.navigationStart / 1000\n },\n get commitLoadTime() {\n return timing.responseStart / 1000\n },\n get finishDocumentLoadTime() {\n return timing.domContentLoadedEventEnd / 1000\n },\n get finishLoadTime() {\n return timing.loadEventEnd / 1000\n },\n get firstPaintTime() {\n const fpEntry = performance.getEntriesByType('paint')[0] || {\n startTime: timing.loadEventEnd / 1000 // Fallback if no navigation occured (`about:blank`)\n }\n return toFixed(\n (fpEntry.startTime + performance.timeOrigin) / 1000,\n 3\n )\n }\n }\n\n window.chrome.loadTimes = function() {\n return {\n ...protocolInfo,\n ...timingInfo\n }\n }\n utils.patchToString(window.chrome.loadTimes)\n }",_args:[{opts:{}}]}),(({_utilsFns:_utilsFns,_mainFunction:_mainFunction,_args:_args})=>{const utils=Object.fromEntries(Object.entries(_utilsFns).map((([key,value])=>[key,eval(value)])));utils.init(),eval(_mainFunction)(utils,..._args)})({_utilsFns:{init:"() => {\n utils.preloadCache()\n}",stripProxyFromErrors:"(handler = {}) => {\n const newHandler = {\n setPrototypeOf: function (target, proto) {\n if (proto === null)\n throw new TypeError('Cannot convert object to primitive value')\n if (Object.getPrototypeOf(target) === Object.getPrototypeOf(proto)) {\n throw new TypeError('Cyclic __proto__ value')\n }\n return Reflect.setPrototypeOf(target, proto)\n }\n }\n // We wrap each trap in the handler in a try/catch and modify the error stack if they throw\n const traps = Object.getOwnPropertyNames(handler)\n traps.forEach(trap => {\n newHandler[trap] = function () {\n try {\n // Forward the call to the defined proxy handler\n return handler[trap].apply(this, arguments || [])\n } catch (err) {\n // Stack traces differ per browser, we only support chromium based ones currently\n if (!err || !err.stack || !err.stack.includes(`at `)) {\n throw err\n }\n\n // When something throws within one of our traps the Proxy will show up in error stacks\n // An earlier implementation of this code would simply strip lines with a blacklist,\n // but it makes sense to be more surgical here and only remove lines related to our Proxy.\n // We try to use a known \"anchor\" line for that and strip it with everything above it.\n // If the anchor line cannot be found for some reason we fall back to our blacklist approach.\n\n const stripWithBlacklist = (stack, stripFirstLine = true) => {\n const blacklist = [\n `at Reflect.${trap} `, // e.g. Reflect.get or Reflect.apply\n `at Object.${trap} `, // e.g. Object.get or Object.apply\n `at Object.newHandler. [as ${trap}] ` // caused by this very wrapper :-)\n ]\n return (\n err.stack\n .split('\\n')\n // Always remove the first (file) line in the stack (guaranteed to be our proxy)\n .filter((line, index) => !(index === 1 && stripFirstLine))\n // Check if the line starts with one of our blacklisted strings\n .filter(line => !blacklist.some(bl => line.trim().startsWith(bl)))\n .join('\\n')\n )\n }\n\n const stripWithAnchor = (stack, anchor) => {\n const stackArr = stack.split('\\n')\n anchor = anchor || `at Object.newHandler. [as ${trap}] ` // Known first Proxy line in chromium\n const anchorIndex = stackArr.findIndex(line =>\n line.trim().startsWith(anchor)\n )\n if (anchorIndex === -1) {\n return false // 404, anchor not found\n }\n // Strip everything from the top until we reach the anchor line\n // Note: We're keeping the 1st line (zero index) as it's unrelated (e.g. `TypeError`)\n stackArr.splice(1, anchorIndex)\n return stackArr.join('\\n')\n }\n\n // Special cases due to our nested toString proxies\n err.stack = err.stack.replace(\n 'at Object.toString (',\n 'at Function.toString ('\n )\n if ((err.stack || '').includes('at Function.toString (')) {\n err.stack = stripWithBlacklist(err.stack, false)\n throw err\n }\n\n // Try using the anchor method, fallback to blacklist if necessary\n err.stack = stripWithAnchor(err.stack) || stripWithBlacklist(err.stack)\n\n throw err // Re-throw our now sanitized error\n }\n }\n })\n return newHandler\n}",stripErrorWithAnchor:"(err, anchor) => {\n const stackArr = err.stack.split('\\n')\n const anchorIndex = stackArr.findIndex(line => line.trim().startsWith(anchor))\n if (anchorIndex === -1) {\n return err // 404, anchor not found\n }\n // Strip everything from the top until we reach the anchor line (remove anchor line as well)\n // Note: We're keeping the 1st line (zero index) as it's unrelated (e.g. `TypeError`)\n stackArr.splice(1, anchorIndex)\n err.stack = stackArr.join('\\n')\n return err\n}",replaceProperty:"(obj, propName, descriptorOverrides = {}) => {\n return Object.defineProperty(obj, propName, {\n // Copy over the existing descriptors (writable, enumerable, configurable, etc)\n ...(Object.getOwnPropertyDescriptor(obj, propName) || {}),\n // Add our overrides (e.g. value, get())\n ...descriptorOverrides\n })\n}",preloadCache:"() => {\n if (utils.cache) {\n return\n }\n utils.cache = {\n // Used in our proxies\n Reflect: {\n get: Reflect.get.bind(Reflect),\n apply: Reflect.apply.bind(Reflect)\n },\n // Used in `makeNativeString`\n nativeToStringStr: Function.toString + '' // => `function toString() { [native code] }`\n }\n}",makeNativeString:"(name = '') => {\n return utils.cache.nativeToStringStr.replace('toString', name || '')\n}",patchToString:"(obj, str = '') => {\n const handler = {\n apply: function (target, ctx) {\n // This fixes e.g. `HTMLMediaElement.prototype.canPlayType.toString + \"\"`\n if (ctx === Function.prototype.toString) {\n return utils.makeNativeString('toString')\n }\n // `toString` targeted at our proxied Object detected\n if (ctx === obj) {\n // We either return the optional string verbatim or derive the most desired result automatically\n return str || utils.makeNativeString(obj.name)\n }\n // Check if the toString protype of the context is the same as the global prototype,\n // if not indicates that we are doing a check across different windows., e.g. the iframeWithdirect` test case\n const hasSameProto = Object.getPrototypeOf(\n Function.prototype.toString\n ).isPrototypeOf(ctx.toString) // eslint-disable-line no-prototype-builtins\n if (!hasSameProto) {\n // Pass the call on to the local Function.prototype.toString instead\n return ctx.toString()\n }\n return target.call(ctx)\n }\n }\n\n const toStringProxy = new Proxy(\n Function.prototype.toString,\n utils.stripProxyFromErrors(handler)\n )\n utils.replaceProperty(Function.prototype, 'toString', {\n value: toStringProxy\n })\n}",patchToStringNested:"(obj = {}) => {\n return utils.execRecursively(obj, ['function'], utils.patchToString)\n}",redirectToString:"(proxyObj, originalObj) => {\n const handler = {\n apply: function (target, ctx) {\n // This fixes e.g. `HTMLMediaElement.prototype.canPlayType.toString + \"\"`\n if (ctx === Function.prototype.toString) {\n return utils.makeNativeString('toString')\n }\n\n // `toString` targeted at our proxied Object detected\n if (ctx === proxyObj) {\n const fallback = () =>\n originalObj && originalObj.name\n ? utils.makeNativeString(originalObj.name)\n : utils.makeNativeString(proxyObj.name)\n\n // Return the toString representation of our original object if possible\n return originalObj + '' || fallback()\n }\n\n if (typeof ctx === 'undefined' || ctx === null) {\n return target.call(ctx)\n }\n\n // Check if the toString protype of the context is the same as the global prototype,\n // if not indicates that we are doing a check across different windows., e.g. the iframeWithdirect` test case\n const hasSameProto = Object.getPrototypeOf(\n Function.prototype.toString\n ).isPrototypeOf(ctx.toString) // eslint-disable-line no-prototype-builtins\n if (!hasSameProto) {\n // Pass the call on to the local Function.prototype.toString instead\n return ctx.toString()\n }\n\n return target.call(ctx)\n }\n }\n\n const toStringProxy = new Proxy(\n Function.prototype.toString,\n utils.stripProxyFromErrors(handler)\n )\n utils.replaceProperty(Function.prototype, 'toString', {\n value: toStringProxy\n })\n}",replaceWithProxy:"(obj, propName, handler) => {\n const originalObj = obj[propName]\n const proxyObj = new Proxy(obj[propName], utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { value: proxyObj })\n utils.redirectToString(proxyObj, originalObj)\n\n return true\n}",replaceGetterWithProxy:"(obj, propName, handler) => {\n const fn = Object.getOwnPropertyDescriptor(obj, propName).get\n const fnStr = fn.toString() // special getter function string\n const proxyObj = new Proxy(fn, utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { get: proxyObj })\n utils.patchToString(proxyObj, fnStr)\n\n return true\n}",mockWithProxy:"(obj, propName, pseudoTarget, handler) => {\n const proxyObj = new Proxy(pseudoTarget, utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { value: proxyObj })\n utils.patchToString(proxyObj)\n\n return true\n}",createProxy:"(pseudoTarget, handler) => {\n const proxyObj = new Proxy(pseudoTarget, utils.stripProxyFromErrors(handler))\n utils.patchToString(proxyObj)\n\n return proxyObj\n}",splitObjPath:"objPath => ({\n // Remove last dot entry (property) ==> `HTMLMediaElement.prototype`\n objName: objPath.split('.').slice(0, -1).join('.'),\n // Extract last dot entry ==> `canPlayType`\n propName: objPath.split('.').slice(-1)[0]\n})",replaceObjPathWithProxy:"(objPath, handler) => {\n const { objName, propName } = utils.splitObjPath(objPath)\n const obj = eval(objName) // eslint-disable-line no-eval\n return utils.replaceWithProxy(obj, propName, handler)\n}",execRecursively:"(obj = {}, typeFilter = [], fn) => {\n function recurse(obj) {\n for (const key in obj) {\n if (obj[key] === undefined) {\n continue\n }\n if (obj[key] && typeof obj[key] === 'object') {\n recurse(obj[key])\n } else {\n if (obj[key] && typeFilter.includes(typeof obj[key])) {\n fn.call(this, obj[key])\n }\n }\n }\n }\n recurse(obj)\n return obj\n}",stringifyFns:"(fnObj = { hello: () => 'world' }) => {\n // Object.fromEntries() ponyfill (in 6 lines) - supported only in Node v12+, modern browsers are fine\n // https://github.com/feross/fromentries\n function fromEntries(iterable) {\n return [...iterable].reduce((obj, [key, val]) => {\n obj[key] = val\n return obj\n }, {})\n }\n return (Object.fromEntries || fromEntries)(\n Object.entries(fnObj)\n .filter(([key, value]) => typeof value === 'function')\n .map(([key, value]) => [key, value.toString()]) // eslint-disable-line no-eval\n )\n}",materializeFns:"(fnStrObj = { hello: \"() => 'world'\" }) => {\n return Object.fromEntries(\n Object.entries(fnStrObj).map(([key, value]) => {\n if (value.startsWith('function')) {\n // some trickery is needed to make oldschool functions work :-)\n return [key, eval(`() => ${value}`)()] // eslint-disable-line no-eval\n } else {\n // arrow functions just work\n return [key, eval(value)] // eslint-disable-line no-eval\n }\n })\n )\n}",makeHandler:"() => ({\n // Used by simple `navigator` getter evasions\n getterValue: value => ({\n apply(target, ctx, args) {\n // Let's fetch the value first, to trigger and escalate potential errors\n // Illegal invocations like `navigator.__proto__.vendor` will throw here\n utils.cache.Reflect.apply(...arguments)\n return value\n }\n })\n})"},_mainFunction:"(utils, { opts, STATIC_DATA }) => {\n if (!window.chrome) {\n // Use the exact property descriptor found in headful Chrome\n // fetch it via `Object.getOwnPropertyDescriptor(window, 'chrome')`\n Object.defineProperty(window, 'chrome', {\n writable: true,\n enumerable: true,\n configurable: false, // note!\n value: {} // We'll extend that later\n })\n }\n\n // That means we're running headful and don't need to mock anything\n const existsAlready = 'runtime' in window.chrome\n // `chrome.runtime` is only exposed on secure origins\n const isNotSecure = !window.location.protocol.startsWith('https')\n if (existsAlready || (isNotSecure && !opts.runOnInsecureOrigins)) {\n return // Nothing to do here\n }\n\n window.chrome.runtime = {\n // There's a bunch of static data in that property which doesn't seem to change,\n // we should periodically check for updates: `JSON.stringify(window.chrome.runtime, null, 2)`\n ...STATIC_DATA,\n // `chrome.runtime.id` is extension related and returns undefined in Chrome\n get id() {\n return undefined\n },\n // These two require more sophisticated mocks\n connect: null,\n sendMessage: null\n }\n\n const makeCustomRuntimeErrors = (preamble, method, extensionId) => ({\n NoMatchingSignature: new TypeError(\n preamble + `No matching signature.`\n ),\n MustSpecifyExtensionID: new TypeError(\n preamble +\n `${method} called from a webpage must specify an Extension ID (string) for its first argument.`\n ),\n InvalidExtensionID: new TypeError(\n preamble + `Invalid extension id: '${extensionId}'`\n )\n })\n\n // Valid Extension IDs are 32 characters in length and use the letter `a` to `p`:\n // https://source.chromium.org/chromium/chromium/src/+/master:components/crx_file/id_util.cc;drc=14a055ccb17e8c8d5d437fe080faba4c6f07beac;l=90\n const isValidExtensionID = str =>\n str.length === 32 && str.toLowerCase().match(/^[a-p]+$/)\n\n /** Mock `chrome.runtime.sendMessage` */\n const sendMessageHandler = {\n apply: function(target, ctx, args) {\n const [extensionId, options, responseCallback] = args || []\n\n // Define custom errors\n const errorPreamble = `Error in invocation of runtime.sendMessage(optional string extensionId, any message, optional object options, optional function responseCallback): `\n const Errors = makeCustomRuntimeErrors(\n errorPreamble,\n `chrome.runtime.sendMessage()`,\n extensionId\n )\n\n // Check if the call signature looks ok\n const noArguments = args.length === 0\n const tooManyArguments = args.length > 4\n const incorrectOptions = options && typeof options !== 'object'\n const incorrectResponseCallback =\n responseCallback && typeof responseCallback !== 'function'\n if (\n noArguments ||\n tooManyArguments ||\n incorrectOptions ||\n incorrectResponseCallback\n ) {\n throw Errors.NoMatchingSignature\n }\n\n // At least 2 arguments are required before we even validate the extension ID\n if (args.length < 2) {\n throw Errors.MustSpecifyExtensionID\n }\n\n // Now let's make sure we got a string as extension ID\n if (typeof extensionId !== 'string') {\n throw Errors.NoMatchingSignature\n }\n\n if (!isValidExtensionID(extensionId)) {\n throw Errors.InvalidExtensionID\n }\n\n return undefined // Normal behavior\n }\n }\n utils.mockWithProxy(\n window.chrome.runtime,\n 'sendMessage',\n function sendMessage() {},\n sendMessageHandler\n )\n\n /**\n * Mock `chrome.runtime.connect`\n *\n * @see https://developer.chrome.com/apps/runtime#method-connect\n */\n const connectHandler = {\n apply: function(target, ctx, args) {\n const [extensionId, connectInfo] = args || []\n\n // Define custom errors\n const errorPreamble = `Error in invocation of runtime.connect(optional string extensionId, optional object connectInfo): `\n const Errors = makeCustomRuntimeErrors(\n errorPreamble,\n `chrome.runtime.connect()`,\n extensionId\n )\n\n // Behavior differs a bit from sendMessage:\n const noArguments = args.length === 0\n const emptyStringArgument = args.length === 1 && extensionId === ''\n if (noArguments || emptyStringArgument) {\n throw Errors.MustSpecifyExtensionID\n }\n\n const tooManyArguments = args.length > 2\n const incorrectConnectInfoType =\n connectInfo && typeof connectInfo !== 'object'\n\n if (tooManyArguments || incorrectConnectInfoType) {\n throw Errors.NoMatchingSignature\n }\n\n const extensionIdIsString = typeof extensionId === 'string'\n if (extensionIdIsString && extensionId === '') {\n throw Errors.MustSpecifyExtensionID\n }\n if (extensionIdIsString && !isValidExtensionID(extensionId)) {\n throw Errors.InvalidExtensionID\n }\n\n // There's another edge-case here: extensionId is optional so we might find a connectInfo object as first param, which we need to validate\n const validateConnectInfo = ci => {\n // More than a first param connectInfo as been provided\n if (args.length > 1) {\n throw Errors.NoMatchingSignature\n }\n // An empty connectInfo has been provided\n if (Object.keys(ci).length === 0) {\n throw Errors.MustSpecifyExtensionID\n }\n // Loop over all connectInfo props an check them\n Object.entries(ci).forEach(([k, v]) => {\n const isExpected = ['name', 'includeTlsChannelId'].includes(k)\n if (!isExpected) {\n throw new TypeError(\n errorPreamble + `Unexpected property: '${k}'.`\n )\n }\n const MismatchError = (propName, expected, found) =>\n TypeError(\n errorPreamble +\n `Error at property '${propName}': Invalid type: expected ${expected}, found ${found}.`\n )\n if (k === 'name' && typeof v !== 'string') {\n throw MismatchError(k, 'string', typeof v)\n }\n if (k === 'includeTlsChannelId' && typeof v !== 'boolean') {\n throw MismatchError(k, 'boolean', typeof v)\n }\n })\n }\n if (typeof extensionId === 'object') {\n validateConnectInfo(extensionId)\n throw Errors.MustSpecifyExtensionID\n }\n\n // Unfortunately even when the connect fails Chrome will return an object with methods we need to mock as well\n return utils.patchToStringNested(makeConnectResponse())\n }\n }\n utils.mockWithProxy(\n window.chrome.runtime,\n 'connect',\n function connect() {},\n connectHandler\n )\n\n function makeConnectResponse() {\n const onSomething = () => ({\n addListener: function addListener() {},\n dispatch: function dispatch() {},\n hasListener: function hasListener() {},\n hasListeners: function hasListeners() {\n return false\n },\n removeListener: function removeListener() {}\n })\n\n const response = {\n name: '',\n sender: undefined,\n disconnect: function disconnect() {},\n onDisconnect: onSomething(),\n onMessage: onSomething(),\n postMessage: function postMessage() {\n if (!arguments.length) {\n throw new TypeError(`Insufficient number of arguments.`)\n }\n throw new Error(`Attempting to use a disconnected port object`)\n }\n }\n return response\n }\n }",_args:[{opts:{runOnInsecureOrigins:!1},STATIC_DATA:{OnInstalledReason:{CHROME_UPDATE:"chrome_update",INSTALL:"install",SHARED_MODULE_UPDATE:"shared_module_update",UPDATE:"update"},OnRestartRequiredReason:{APP_UPDATE:"app_update",OS_UPDATE:"os_update",PERIODIC:"periodic"},PlatformArch:{ARM:"arm",ARM64:"arm64",MIPS:"mips",MIPS64:"mips64",X86_32:"x86-32",X86_64:"x86-64"},PlatformNaclArch:{ARM:"arm",MIPS:"mips",MIPS64:"mips64",X86_32:"x86-32",X86_64:"x86-64"},PlatformOs:{ANDROID:"android",CROS:"cros",LINUX:"linux",MAC:"mac",OPENBSD:"openbsd",WIN:"win"},RequestUpdateCheckStatus:{NO_UPDATE:"no_update",THROTTLED:"throttled",UPDATE_AVAILABLE:"update_available"}}}]}),(({_utilsFns:_utilsFns,_mainFunction:_mainFunction,_args:_args})=>{const utils=Object.fromEntries(Object.entries(_utilsFns).map((([key,value])=>[key,eval(value)])));utils.init(),eval(_mainFunction)(utils,..._args)})({_utilsFns:{init:"() => {\n utils.preloadCache()\n}",stripProxyFromErrors:"(handler = {}) => {\n const newHandler = {\n setPrototypeOf: function (target, proto) {\n if (proto === null)\n throw new TypeError('Cannot convert object to primitive value')\n if (Object.getPrototypeOf(target) === Object.getPrototypeOf(proto)) {\n throw new TypeError('Cyclic __proto__ value')\n }\n return Reflect.setPrototypeOf(target, proto)\n }\n }\n // We wrap each trap in the handler in a try/catch and modify the error stack if they throw\n const traps = Object.getOwnPropertyNames(handler)\n traps.forEach(trap => {\n newHandler[trap] = function () {\n try {\n // Forward the call to the defined proxy handler\n return handler[trap].apply(this, arguments || [])\n } catch (err) {\n // Stack traces differ per browser, we only support chromium based ones currently\n if (!err || !err.stack || !err.stack.includes(`at `)) {\n throw err\n }\n\n // When something throws within one of our traps the Proxy will show up in error stacks\n // An earlier implementation of this code would simply strip lines with a blacklist,\n // but it makes sense to be more surgical here and only remove lines related to our Proxy.\n // We try to use a known \"anchor\" line for that and strip it with everything above it.\n // If the anchor line cannot be found for some reason we fall back to our blacklist approach.\n\n const stripWithBlacklist = (stack, stripFirstLine = true) => {\n const blacklist = [\n `at Reflect.${trap} `, // e.g. Reflect.get or Reflect.apply\n `at Object.${trap} `, // e.g. Object.get or Object.apply\n `at Object.newHandler. [as ${trap}] ` // caused by this very wrapper :-)\n ]\n return (\n err.stack\n .split('\\n')\n // Always remove the first (file) line in the stack (guaranteed to be our proxy)\n .filter((line, index) => !(index === 1 && stripFirstLine))\n // Check if the line starts with one of our blacklisted strings\n .filter(line => !blacklist.some(bl => line.trim().startsWith(bl)))\n .join('\\n')\n )\n }\n\n const stripWithAnchor = (stack, anchor) => {\n const stackArr = stack.split('\\n')\n anchor = anchor || `at Object.newHandler. [as ${trap}] ` // Known first Proxy line in chromium\n const anchorIndex = stackArr.findIndex(line =>\n line.trim().startsWith(anchor)\n )\n if (anchorIndex === -1) {\n return false // 404, anchor not found\n }\n // Strip everything from the top until we reach the anchor line\n // Note: We're keeping the 1st line (zero index) as it's unrelated (e.g. `TypeError`)\n stackArr.splice(1, anchorIndex)\n return stackArr.join('\\n')\n }\n\n // Special cases due to our nested toString proxies\n err.stack = err.stack.replace(\n 'at Object.toString (',\n 'at Function.toString ('\n )\n if ((err.stack || '').includes('at Function.toString (')) {\n err.stack = stripWithBlacklist(err.stack, false)\n throw err\n }\n\n // Try using the anchor method, fallback to blacklist if necessary\n err.stack = stripWithAnchor(err.stack) || stripWithBlacklist(err.stack)\n\n throw err // Re-throw our now sanitized error\n }\n }\n })\n return newHandler\n}",stripErrorWithAnchor:"(err, anchor) => {\n const stackArr = err.stack.split('\\n')\n const anchorIndex = stackArr.findIndex(line => line.trim().startsWith(anchor))\n if (anchorIndex === -1) {\n return err // 404, anchor not found\n }\n // Strip everything from the top until we reach the anchor line (remove anchor line as well)\n // Note: We're keeping the 1st line (zero index) as it's unrelated (e.g. `TypeError`)\n stackArr.splice(1, anchorIndex)\n err.stack = stackArr.join('\\n')\n return err\n}",replaceProperty:"(obj, propName, descriptorOverrides = {}) => {\n return Object.defineProperty(obj, propName, {\n // Copy over the existing descriptors (writable, enumerable, configurable, etc)\n ...(Object.getOwnPropertyDescriptor(obj, propName) || {}),\n // Add our overrides (e.g. value, get())\n ...descriptorOverrides\n })\n}",preloadCache:"() => {\n if (utils.cache) {\n return\n }\n utils.cache = {\n // Used in our proxies\n Reflect: {\n get: Reflect.get.bind(Reflect),\n apply: Reflect.apply.bind(Reflect)\n },\n // Used in `makeNativeString`\n nativeToStringStr: Function.toString + '' // => `function toString() { [native code] }`\n }\n}",makeNativeString:"(name = '') => {\n return utils.cache.nativeToStringStr.replace('toString', name || '')\n}",patchToString:"(obj, str = '') => {\n const handler = {\n apply: function (target, ctx) {\n // This fixes e.g. `HTMLMediaElement.prototype.canPlayType.toString + \"\"`\n if (ctx === Function.prototype.toString) {\n return utils.makeNativeString('toString')\n }\n // `toString` targeted at our proxied Object detected\n if (ctx === obj) {\n // We either return the optional string verbatim or derive the most desired result automatically\n return str || utils.makeNativeString(obj.name)\n }\n // Check if the toString protype of the context is the same as the global prototype,\n // if not indicates that we are doing a check across different windows., e.g. the iframeWithdirect` test case\n const hasSameProto = Object.getPrototypeOf(\n Function.prototype.toString\n ).isPrototypeOf(ctx.toString) // eslint-disable-line no-prototype-builtins\n if (!hasSameProto) {\n // Pass the call on to the local Function.prototype.toString instead\n return ctx.toString()\n }\n return target.call(ctx)\n }\n }\n\n const toStringProxy = new Proxy(\n Function.prototype.toString,\n utils.stripProxyFromErrors(handler)\n )\n utils.replaceProperty(Function.prototype, 'toString', {\n value: toStringProxy\n })\n}",patchToStringNested:"(obj = {}) => {\n return utils.execRecursively(obj, ['function'], utils.patchToString)\n}",redirectToString:"(proxyObj, originalObj) => {\n const handler = {\n apply: function (target, ctx) {\n // This fixes e.g. `HTMLMediaElement.prototype.canPlayType.toString + \"\"`\n if (ctx === Function.prototype.toString) {\n return utils.makeNativeString('toString')\n }\n\n // `toString` targeted at our proxied Object detected\n if (ctx === proxyObj) {\n const fallback = () =>\n originalObj && originalObj.name\n ? utils.makeNativeString(originalObj.name)\n : utils.makeNativeString(proxyObj.name)\n\n // Return the toString representation of our original object if possible\n return originalObj + '' || fallback()\n }\n\n if (typeof ctx === 'undefined' || ctx === null) {\n return target.call(ctx)\n }\n\n // Check if the toString protype of the context is the same as the global prototype,\n // if not indicates that we are doing a check across different windows., e.g. the iframeWithdirect` test case\n const hasSameProto = Object.getPrototypeOf(\n Function.prototype.toString\n ).isPrototypeOf(ctx.toString) // eslint-disable-line no-prototype-builtins\n if (!hasSameProto) {\n // Pass the call on to the local Function.prototype.toString instead\n return ctx.toString()\n }\n\n return target.call(ctx)\n }\n }\n\n const toStringProxy = new Proxy(\n Function.prototype.toString,\n utils.stripProxyFromErrors(handler)\n )\n utils.replaceProperty(Function.prototype, 'toString', {\n value: toStringProxy\n })\n}",replaceWithProxy:"(obj, propName, handler) => {\n const originalObj = obj[propName]\n const proxyObj = new Proxy(obj[propName], utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { value: proxyObj })\n utils.redirectToString(proxyObj, originalObj)\n\n return true\n}",replaceGetterWithProxy:"(obj, propName, handler) => {\n const fn = Object.getOwnPropertyDescriptor(obj, propName).get\n const fnStr = fn.toString() // special getter function string\n const proxyObj = new Proxy(fn, utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { get: proxyObj })\n utils.patchToString(proxyObj, fnStr)\n\n return true\n}",mockWithProxy:"(obj, propName, pseudoTarget, handler) => {\n const proxyObj = new Proxy(pseudoTarget, utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { value: proxyObj })\n utils.patchToString(proxyObj)\n\n return true\n}",createProxy:"(pseudoTarget, handler) => {\n const proxyObj = new Proxy(pseudoTarget, utils.stripProxyFromErrors(handler))\n utils.patchToString(proxyObj)\n\n return proxyObj\n}",splitObjPath:"objPath => ({\n // Remove last dot entry (property) ==> `HTMLMediaElement.prototype`\n objName: objPath.split('.').slice(0, -1).join('.'),\n // Extract last dot entry ==> `canPlayType`\n propName: objPath.split('.').slice(-1)[0]\n})",replaceObjPathWithProxy:"(objPath, handler) => {\n const { objName, propName } = utils.splitObjPath(objPath)\n const obj = eval(objName) // eslint-disable-line no-eval\n return utils.replaceWithProxy(obj, propName, handler)\n}",execRecursively:"(obj = {}, typeFilter = [], fn) => {\n function recurse(obj) {\n for (const key in obj) {\n if (obj[key] === undefined) {\n continue\n }\n if (obj[key] && typeof obj[key] === 'object') {\n recurse(obj[key])\n } else {\n if (obj[key] && typeFilter.includes(typeof obj[key])) {\n fn.call(this, obj[key])\n }\n }\n }\n }\n recurse(obj)\n return obj\n}",stringifyFns:"(fnObj = { hello: () => 'world' }) => {\n // Object.fromEntries() ponyfill (in 6 lines) - supported only in Node v12+, modern browsers are fine\n // https://github.com/feross/fromentries\n function fromEntries(iterable) {\n return [...iterable].reduce((obj, [key, val]) => {\n obj[key] = val\n return obj\n }, {})\n }\n return (Object.fromEntries || fromEntries)(\n Object.entries(fnObj)\n .filter(([key, value]) => typeof value === 'function')\n .map(([key, value]) => [key, value.toString()]) // eslint-disable-line no-eval\n )\n}",materializeFns:"(fnStrObj = { hello: \"() => 'world'\" }) => {\n return Object.fromEntries(\n Object.entries(fnStrObj).map(([key, value]) => {\n if (value.startsWith('function')) {\n // some trickery is needed to make oldschool functions work :-)\n return [key, eval(`() => ${value}`)()] // eslint-disable-line no-eval\n } else {\n // arrow functions just work\n return [key, eval(value)] // eslint-disable-line no-eval\n }\n })\n )\n}",makeHandler:"() => ({\n // Used by simple `navigator` getter evasions\n getterValue: value => ({\n apply(target, ctx, args) {\n // Let's fetch the value first, to trigger and escalate potential errors\n // Illegal invocations like `navigator.__proto__.vendor` will throw here\n utils.cache.Reflect.apply(...arguments)\n return value\n }\n })\n})"},_mainFunction:"utils => {\n /**\n * Input might look funky, we need to normalize it so e.g. whitespace isn't an issue for our spoofing.\n *\n * @example\n * video/webm; codecs=\"vp8, vorbis\"\n * video/mp4; codecs=\"avc1.42E01E\"\n * audio/x-m4a;\n * audio/ogg; codecs=\"vorbis\"\n * @param {String} arg\n */\n const parseInput = arg => {\n const [mime, codecStr] = arg.trim().split(';')\n let codecs = []\n if (codecStr && codecStr.includes('codecs=\"')) {\n codecs = codecStr\n .trim()\n .replace(`codecs=\"`, '')\n .replace(`\"`, '')\n .trim()\n .split(',')\n .filter(x => !!x)\n .map(x => x.trim())\n }\n return {\n mime,\n codecStr,\n codecs\n }\n }\n\n const canPlayType = {\n // Intercept certain requests\n apply: function(target, ctx, args) {\n if (!args || !args.length) {\n return target.apply(ctx, args)\n }\n const { mime, codecs } = parseInput(args[0])\n // This specific mp4 codec is missing in Chromium\n if (mime === 'video/mp4') {\n if (codecs.includes('avc1.42E01E')) {\n return 'probably'\n }\n }\n // This mimetype is only supported if no codecs are specified\n if (mime === 'audio/x-m4a' && !codecs.length) {\n return 'maybe'\n }\n\n // This mimetype is only supported if no codecs are specified\n if (mime === 'audio/aac' && !codecs.length) {\n return 'probably'\n }\n // Everything else as usual\n return target.apply(ctx, args)\n }\n }\n\n /* global HTMLMediaElement */\n utils.replaceWithProxy(\n HTMLMediaElement.prototype,\n 'canPlayType',\n canPlayType\n )\n }",_args:[]}),(({_utilsFns:_utilsFns,_mainFunction:_mainFunction,_args:_args})=>{const utils=Object.fromEntries(Object.entries(_utilsFns).map((([key,value])=>[key,eval(value)])));utils.init(),eval(_mainFunction)(utils,..._args)})({_utilsFns:{init:"() => {\n utils.preloadCache()\n}",stripProxyFromErrors:"(handler = {}) => {\n const newHandler = {\n setPrototypeOf: function (target, proto) {\n if (proto === null)\n throw new TypeError('Cannot convert object to primitive value')\n if (Object.getPrototypeOf(target) === Object.getPrototypeOf(proto)) {\n throw new TypeError('Cyclic __proto__ value')\n }\n return Reflect.setPrototypeOf(target, proto)\n }\n }\n // We wrap each trap in the handler in a try/catch and modify the error stack if they throw\n const traps = Object.getOwnPropertyNames(handler)\n traps.forEach(trap => {\n newHandler[trap] = function () {\n try {\n // Forward the call to the defined proxy handler\n return handler[trap].apply(this, arguments || [])\n } catch (err) {\n // Stack traces differ per browser, we only support chromium based ones currently\n if (!err || !err.stack || !err.stack.includes(`at `)) {\n throw err\n }\n\n // When something throws within one of our traps the Proxy will show up in error stacks\n // An earlier implementation of this code would simply strip lines with a blacklist,\n // but it makes sense to be more surgical here and only remove lines related to our Proxy.\n // We try to use a known \"anchor\" line for that and strip it with everything above it.\n // If the anchor line cannot be found for some reason we fall back to our blacklist approach.\n\n const stripWithBlacklist = (stack, stripFirstLine = true) => {\n const blacklist = [\n `at Reflect.${trap} `, // e.g. Reflect.get or Reflect.apply\n `at Object.${trap} `, // e.g. Object.get or Object.apply\n `at Object.newHandler. [as ${trap}] ` // caused by this very wrapper :-)\n ]\n return (\n err.stack\n .split('\\n')\n // Always remove the first (file) line in the stack (guaranteed to be our proxy)\n .filter((line, index) => !(index === 1 && stripFirstLine))\n // Check if the line starts with one of our blacklisted strings\n .filter(line => !blacklist.some(bl => line.trim().startsWith(bl)))\n .join('\\n')\n )\n }\n\n const stripWithAnchor = (stack, anchor) => {\n const stackArr = stack.split('\\n')\n anchor = anchor || `at Object.newHandler. [as ${trap}] ` // Known first Proxy line in chromium\n const anchorIndex = stackArr.findIndex(line =>\n line.trim().startsWith(anchor)\n )\n if (anchorIndex === -1) {\n return false // 404, anchor not found\n }\n // Strip everything from the top until we reach the anchor line\n // Note: We're keeping the 1st line (zero index) as it's unrelated (e.g. `TypeError`)\n stackArr.splice(1, anchorIndex)\n return stackArr.join('\\n')\n }\n\n // Special cases due to our nested toString proxies\n err.stack = err.stack.replace(\n 'at Object.toString (',\n 'at Function.toString ('\n )\n if ((err.stack || '').includes('at Function.toString (')) {\n err.stack = stripWithBlacklist(err.stack, false)\n throw err\n }\n\n // Try using the anchor method, fallback to blacklist if necessary\n err.stack = stripWithAnchor(err.stack) || stripWithBlacklist(err.stack)\n\n throw err // Re-throw our now sanitized error\n }\n }\n })\n return newHandler\n}",stripErrorWithAnchor:"(err, anchor) => {\n const stackArr = err.stack.split('\\n')\n const anchorIndex = stackArr.findIndex(line => line.trim().startsWith(anchor))\n if (anchorIndex === -1) {\n return err // 404, anchor not found\n }\n // Strip everything from the top until we reach the anchor line (remove anchor line as well)\n // Note: We're keeping the 1st line (zero index) as it's unrelated (e.g. `TypeError`)\n stackArr.splice(1, anchorIndex)\n err.stack = stackArr.join('\\n')\n return err\n}",replaceProperty:"(obj, propName, descriptorOverrides = {}) => {\n return Object.defineProperty(obj, propName, {\n // Copy over the existing descriptors (writable, enumerable, configurable, etc)\n ...(Object.getOwnPropertyDescriptor(obj, propName) || {}),\n // Add our overrides (e.g. value, get())\n ...descriptorOverrides\n })\n}",preloadCache:"() => {\n if (utils.cache) {\n return\n }\n utils.cache = {\n // Used in our proxies\n Reflect: {\n get: Reflect.get.bind(Reflect),\n apply: Reflect.apply.bind(Reflect)\n },\n // Used in `makeNativeString`\n nativeToStringStr: Function.toString + '' // => `function toString() { [native code] }`\n }\n}",makeNativeString:"(name = '') => {\n return utils.cache.nativeToStringStr.replace('toString', name || '')\n}",patchToString:"(obj, str = '') => {\n const handler = {\n apply: function (target, ctx) {\n // This fixes e.g. `HTMLMediaElement.prototype.canPlayType.toString + \"\"`\n if (ctx === Function.prototype.toString) {\n return utils.makeNativeString('toString')\n }\n // `toString` targeted at our proxied Object detected\n if (ctx === obj) {\n // We either return the optional string verbatim or derive the most desired result automatically\n return str || utils.makeNativeString(obj.name)\n }\n // Check if the toString protype of the context is the same as the global prototype,\n // if not indicates that we are doing a check across different windows., e.g. the iframeWithdirect` test case\n const hasSameProto = Object.getPrototypeOf(\n Function.prototype.toString\n ).isPrototypeOf(ctx.toString) // eslint-disable-line no-prototype-builtins\n if (!hasSameProto) {\n // Pass the call on to the local Function.prototype.toString instead\n return ctx.toString()\n }\n return target.call(ctx)\n }\n }\n\n const toStringProxy = new Proxy(\n Function.prototype.toString,\n utils.stripProxyFromErrors(handler)\n )\n utils.replaceProperty(Function.prototype, 'toString', {\n value: toStringProxy\n })\n}",patchToStringNested:"(obj = {}) => {\n return utils.execRecursively(obj, ['function'], utils.patchToString)\n}",redirectToString:"(proxyObj, originalObj) => {\n const handler = {\n apply: function (target, ctx) {\n // This fixes e.g. `HTMLMediaElement.prototype.canPlayType.toString + \"\"`\n if (ctx === Function.prototype.toString) {\n return utils.makeNativeString('toString')\n }\n\n // `toString` targeted at our proxied Object detected\n if (ctx === proxyObj) {\n const fallback = () =>\n originalObj && originalObj.name\n ? utils.makeNativeString(originalObj.name)\n : utils.makeNativeString(proxyObj.name)\n\n // Return the toString representation of our original object if possible\n return originalObj + '' || fallback()\n }\n\n if (typeof ctx === 'undefined' || ctx === null) {\n return target.call(ctx)\n }\n\n // Check if the toString protype of the context is the same as the global prototype,\n // if not indicates that we are doing a check across different windows., e.g. the iframeWithdirect` test case\n const hasSameProto = Object.getPrototypeOf(\n Function.prototype.toString\n ).isPrototypeOf(ctx.toString) // eslint-disable-line no-prototype-builtins\n if (!hasSameProto) {\n // Pass the call on to the local Function.prototype.toString instead\n return ctx.toString()\n }\n\n return target.call(ctx)\n }\n }\n\n const toStringProxy = new Proxy(\n Function.prototype.toString,\n utils.stripProxyFromErrors(handler)\n )\n utils.replaceProperty(Function.prototype, 'toString', {\n value: toStringProxy\n })\n}",replaceWithProxy:"(obj, propName, handler) => {\n const originalObj = obj[propName]\n const proxyObj = new Proxy(obj[propName], utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { value: proxyObj })\n utils.redirectToString(proxyObj, originalObj)\n\n return true\n}",replaceGetterWithProxy:"(obj, propName, handler) => {\n const fn = Object.getOwnPropertyDescriptor(obj, propName).get\n const fnStr = fn.toString() // special getter function string\n const proxyObj = new Proxy(fn, utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { get: proxyObj })\n utils.patchToString(proxyObj, fnStr)\n\n return true\n}",mockWithProxy:"(obj, propName, pseudoTarget, handler) => {\n const proxyObj = new Proxy(pseudoTarget, utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { value: proxyObj })\n utils.patchToString(proxyObj)\n\n return true\n}",createProxy:"(pseudoTarget, handler) => {\n const proxyObj = new Proxy(pseudoTarget, utils.stripProxyFromErrors(handler))\n utils.patchToString(proxyObj)\n\n return proxyObj\n}",splitObjPath:"objPath => ({\n // Remove last dot entry (property) ==> `HTMLMediaElement.prototype`\n objName: objPath.split('.').slice(0, -1).join('.'),\n // Extract last dot entry ==> `canPlayType`\n propName: objPath.split('.').slice(-1)[0]\n})",replaceObjPathWithProxy:"(objPath, handler) => {\n const { objName, propName } = utils.splitObjPath(objPath)\n const obj = eval(objName) // eslint-disable-line no-eval\n return utils.replaceWithProxy(obj, propName, handler)\n}",execRecursively:"(obj = {}, typeFilter = [], fn) => {\n function recurse(obj) {\n for (const key in obj) {\n if (obj[key] === undefined) {\n continue\n }\n if (obj[key] && typeof obj[key] === 'object') {\n recurse(obj[key])\n } else {\n if (obj[key] && typeFilter.includes(typeof obj[key])) {\n fn.call(this, obj[key])\n }\n }\n }\n }\n recurse(obj)\n return obj\n}",stringifyFns:"(fnObj = { hello: () => 'world' }) => {\n // Object.fromEntries() ponyfill (in 6 lines) - supported only in Node v12+, modern browsers are fine\n // https://github.com/feross/fromentries\n function fromEntries(iterable) {\n return [...iterable].reduce((obj, [key, val]) => {\n obj[key] = val\n return obj\n }, {})\n }\n return (Object.fromEntries || fromEntries)(\n Object.entries(fnObj)\n .filter(([key, value]) => typeof value === 'function')\n .map(([key, value]) => [key, value.toString()]) // eslint-disable-line no-eval\n )\n}",materializeFns:"(fnStrObj = { hello: \"() => 'world'\" }) => {\n return Object.fromEntries(\n Object.entries(fnStrObj).map(([key, value]) => {\n if (value.startsWith('function')) {\n // some trickery is needed to make oldschool functions work :-)\n return [key, eval(`() => ${value}`)()] // eslint-disable-line no-eval\n } else {\n // arrow functions just work\n return [key, eval(value)] // eslint-disable-line no-eval\n }\n })\n )\n}",makeHandler:"() => ({\n // Used by simple `navigator` getter evasions\n getterValue: value => ({\n apply(target, ctx, args) {\n // Let's fetch the value first, to trigger and escalate potential errors\n // Illegal invocations like `navigator.__proto__.vendor` will throw here\n utils.cache.Reflect.apply(...arguments)\n return value\n }\n })\n})"},_mainFunction:"(utils, { opts }) => {\n utils.replaceGetterWithProxy(\n Object.getPrototypeOf(navigator),\n 'hardwareConcurrency',\n utils.makeHandler().getterValue(opts.hardwareConcurrency)\n )\n }",_args:[{opts:{hardwareConcurrency:4}}]}),(({_utilsFns:_utilsFns,_mainFunction:_mainFunction,_args:_args})=>{const utils=Object.fromEntries(Object.entries(_utilsFns).map((([key,value])=>[key,eval(value)])));utils.init(),eval(_mainFunction)(utils,..._args)})({_utilsFns:{init:"() => {\n utils.preloadCache()\n}",stripProxyFromErrors:"(handler = {}) => {\n const newHandler = {\n setPrototypeOf: function (target, proto) {\n if (proto === null)\n throw new TypeError('Cannot convert object to primitive value')\n if (Object.getPrototypeOf(target) === Object.getPrototypeOf(proto)) {\n throw new TypeError('Cyclic __proto__ value')\n }\n return Reflect.setPrototypeOf(target, proto)\n }\n }\n // We wrap each trap in the handler in a try/catch and modify the error stack if they throw\n const traps = Object.getOwnPropertyNames(handler)\n traps.forEach(trap => {\n newHandler[trap] = function () {\n try {\n // Forward the call to the defined proxy handler\n return handler[trap].apply(this, arguments || [])\n } catch (err) {\n // Stack traces differ per browser, we only support chromium based ones currently\n if (!err || !err.stack || !err.stack.includes(`at `)) {\n throw err\n }\n\n // When something throws within one of our traps the Proxy will show up in error stacks\n // An earlier implementation of this code would simply strip lines with a blacklist,\n // but it makes sense to be more surgical here and only remove lines related to our Proxy.\n // We try to use a known \"anchor\" line for that and strip it with everything above it.\n // If the anchor line cannot be found for some reason we fall back to our blacklist approach.\n\n const stripWithBlacklist = (stack, stripFirstLine = true) => {\n const blacklist = [\n `at Reflect.${trap} `, // e.g. Reflect.get or Reflect.apply\n `at Object.${trap} `, // e.g. Object.get or Object.apply\n `at Object.newHandler. [as ${trap}] ` // caused by this very wrapper :-)\n ]\n return (\n err.stack\n .split('\\n')\n // Always remove the first (file) line in the stack (guaranteed to be our proxy)\n .filter((line, index) => !(index === 1 && stripFirstLine))\n // Check if the line starts with one of our blacklisted strings\n .filter(line => !blacklist.some(bl => line.trim().startsWith(bl)))\n .join('\\n')\n )\n }\n\n const stripWithAnchor = (stack, anchor) => {\n const stackArr = stack.split('\\n')\n anchor = anchor || `at Object.newHandler. [as ${trap}] ` // Known first Proxy line in chromium\n const anchorIndex = stackArr.findIndex(line =>\n line.trim().startsWith(anchor)\n )\n if (anchorIndex === -1) {\n return false // 404, anchor not found\n }\n // Strip everything from the top until we reach the anchor line\n // Note: We're keeping the 1st line (zero index) as it's unrelated (e.g. `TypeError`)\n stackArr.splice(1, anchorIndex)\n return stackArr.join('\\n')\n }\n\n // Special cases due to our nested toString proxies\n err.stack = err.stack.replace(\n 'at Object.toString (',\n 'at Function.toString ('\n )\n if ((err.stack || '').includes('at Function.toString (')) {\n err.stack = stripWithBlacklist(err.stack, false)\n throw err\n }\n\n // Try using the anchor method, fallback to blacklist if necessary\n err.stack = stripWithAnchor(err.stack) || stripWithBlacklist(err.stack)\n\n throw err // Re-throw our now sanitized error\n }\n }\n })\n return newHandler\n}",stripErrorWithAnchor:"(err, anchor) => {\n const stackArr = err.stack.split('\\n')\n const anchorIndex = stackArr.findIndex(line => line.trim().startsWith(anchor))\n if (anchorIndex === -1) {\n return err // 404, anchor not found\n }\n // Strip everything from the top until we reach the anchor line (remove anchor line as well)\n // Note: We're keeping the 1st line (zero index) as it's unrelated (e.g. `TypeError`)\n stackArr.splice(1, anchorIndex)\n err.stack = stackArr.join('\\n')\n return err\n}",replaceProperty:"(obj, propName, descriptorOverrides = {}) => {\n return Object.defineProperty(obj, propName, {\n // Copy over the existing descriptors (writable, enumerable, configurable, etc)\n ...(Object.getOwnPropertyDescriptor(obj, propName) || {}),\n // Add our overrides (e.g. value, get())\n ...descriptorOverrides\n })\n}",preloadCache:"() => {\n if (utils.cache) {\n return\n }\n utils.cache = {\n // Used in our proxies\n Reflect: {\n get: Reflect.get.bind(Reflect),\n apply: Reflect.apply.bind(Reflect)\n },\n // Used in `makeNativeString`\n nativeToStringStr: Function.toString + '' // => `function toString() { [native code] }`\n }\n}",makeNativeString:"(name = '') => {\n return utils.cache.nativeToStringStr.replace('toString', name || '')\n}",patchToString:"(obj, str = '') => {\n const handler = {\n apply: function (target, ctx) {\n // This fixes e.g. `HTMLMediaElement.prototype.canPlayType.toString + \"\"`\n if (ctx === Function.prototype.toString) {\n return utils.makeNativeString('toString')\n }\n // `toString` targeted at our proxied Object detected\n if (ctx === obj) {\n // We either return the optional string verbatim or derive the most desired result automatically\n return str || utils.makeNativeString(obj.name)\n }\n // Check if the toString protype of the context is the same as the global prototype,\n // if not indicates that we are doing a check across different windows., e.g. the iframeWithdirect` test case\n const hasSameProto = Object.getPrototypeOf(\n Function.prototype.toString\n ).isPrototypeOf(ctx.toString) // eslint-disable-line no-prototype-builtins\n if (!hasSameProto) {\n // Pass the call on to the local Function.prototype.toString instead\n return ctx.toString()\n }\n return target.call(ctx)\n }\n }\n\n const toStringProxy = new Proxy(\n Function.prototype.toString,\n utils.stripProxyFromErrors(handler)\n )\n utils.replaceProperty(Function.prototype, 'toString', {\n value: toStringProxy\n })\n}",patchToStringNested:"(obj = {}) => {\n return utils.execRecursively(obj, ['function'], utils.patchToString)\n}",redirectToString:"(proxyObj, originalObj) => {\n const handler = {\n apply: function (target, ctx) {\n // This fixes e.g. `HTMLMediaElement.prototype.canPlayType.toString + \"\"`\n if (ctx === Function.prototype.toString) {\n return utils.makeNativeString('toString')\n }\n\n // `toString` targeted at our proxied Object detected\n if (ctx === proxyObj) {\n const fallback = () =>\n originalObj && originalObj.name\n ? utils.makeNativeString(originalObj.name)\n : utils.makeNativeString(proxyObj.name)\n\n // Return the toString representation of our original object if possible\n return originalObj + '' || fallback()\n }\n\n if (typeof ctx === 'undefined' || ctx === null) {\n return target.call(ctx)\n }\n\n // Check if the toString protype of the context is the same as the global prototype,\n // if not indicates that we are doing a check across different windows., e.g. the iframeWithdirect` test case\n const hasSameProto = Object.getPrototypeOf(\n Function.prototype.toString\n ).isPrototypeOf(ctx.toString) // eslint-disable-line no-prototype-builtins\n if (!hasSameProto) {\n // Pass the call on to the local Function.prototype.toString instead\n return ctx.toString()\n }\n\n return target.call(ctx)\n }\n }\n\n const toStringProxy = new Proxy(\n Function.prototype.toString,\n utils.stripProxyFromErrors(handler)\n )\n utils.replaceProperty(Function.prototype, 'toString', {\n value: toStringProxy\n })\n}",replaceWithProxy:"(obj, propName, handler) => {\n const originalObj = obj[propName]\n const proxyObj = new Proxy(obj[propName], utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { value: proxyObj })\n utils.redirectToString(proxyObj, originalObj)\n\n return true\n}",replaceGetterWithProxy:"(obj, propName, handler) => {\n const fn = Object.getOwnPropertyDescriptor(obj, propName).get\n const fnStr = fn.toString() // special getter function string\n const proxyObj = new Proxy(fn, utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { get: proxyObj })\n utils.patchToString(proxyObj, fnStr)\n\n return true\n}",mockWithProxy:"(obj, propName, pseudoTarget, handler) => {\n const proxyObj = new Proxy(pseudoTarget, utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { value: proxyObj })\n utils.patchToString(proxyObj)\n\n return true\n}",createProxy:"(pseudoTarget, handler) => {\n const proxyObj = new Proxy(pseudoTarget, utils.stripProxyFromErrors(handler))\n utils.patchToString(proxyObj)\n\n return proxyObj\n}",splitObjPath:"objPath => ({\n // Remove last dot entry (property) ==> `HTMLMediaElement.prototype`\n objName: objPath.split('.').slice(0, -1).join('.'),\n // Extract last dot entry ==> `canPlayType`\n propName: objPath.split('.').slice(-1)[0]\n})",replaceObjPathWithProxy:"(objPath, handler) => {\n const { objName, propName } = utils.splitObjPath(objPath)\n const obj = eval(objName) // eslint-disable-line no-eval\n return utils.replaceWithProxy(obj, propName, handler)\n}",execRecursively:"(obj = {}, typeFilter = [], fn) => {\n function recurse(obj) {\n for (const key in obj) {\n if (obj[key] === undefined) {\n continue\n }\n if (obj[key] && typeof obj[key] === 'object') {\n recurse(obj[key])\n } else {\n if (obj[key] && typeFilter.includes(typeof obj[key])) {\n fn.call(this, obj[key])\n }\n }\n }\n }\n recurse(obj)\n return obj\n}",stringifyFns:"(fnObj = { hello: () => 'world' }) => {\n // Object.fromEntries() ponyfill (in 6 lines) - supported only in Node v12+, modern browsers are fine\n // https://github.com/feross/fromentries\n function fromEntries(iterable) {\n return [...iterable].reduce((obj, [key, val]) => {\n obj[key] = val\n return obj\n }, {})\n }\n return (Object.fromEntries || fromEntries)(\n Object.entries(fnObj)\n .filter(([key, value]) => typeof value === 'function')\n .map(([key, value]) => [key, value.toString()]) // eslint-disable-line no-eval\n )\n}",materializeFns:"(fnStrObj = { hello: \"() => 'world'\" }) => {\n return Object.fromEntries(\n Object.entries(fnStrObj).map(([key, value]) => {\n if (value.startsWith('function')) {\n // some trickery is needed to make oldschool functions work :-)\n return [key, eval(`() => ${value}`)()] // eslint-disable-line no-eval\n } else {\n // arrow functions just work\n return [key, eval(value)] // eslint-disable-line no-eval\n }\n })\n )\n}",makeHandler:"() => ({\n // Used by simple `navigator` getter evasions\n getterValue: value => ({\n apply(target, ctx, args) {\n // Let's fetch the value first, to trigger and escalate potential errors\n // Illegal invocations like `navigator.__proto__.vendor` will throw here\n utils.cache.Reflect.apply(...arguments)\n return value\n }\n })\n})"},_mainFunction:"(utils, { opts }) => {\n const languages = opts.languages.length\n ? opts.languages\n : ['en-US', 'en']\n utils.replaceGetterWithProxy(\n Object.getPrototypeOf(navigator),\n 'languages',\n utils.makeHandler().getterValue(Object.freeze([...languages]))\n )\n }",_args:[{opts:{languages:[]}}]}),(({_utilsFns:_utilsFns,_mainFunction:_mainFunction,_args:_args})=>{const utils=Object.fromEntries(Object.entries(_utilsFns).map((([key,value])=>[key,eval(value)])));utils.init(),eval(_mainFunction)(utils,..._args)})({_utilsFns:{init:"() => {\n utils.preloadCache()\n}",stripProxyFromErrors:"(handler = {}) => {\n const newHandler = {\n setPrototypeOf: function (target, proto) {\n if (proto === null)\n throw new TypeError('Cannot convert object to primitive value')\n if (Object.getPrototypeOf(target) === Object.getPrototypeOf(proto)) {\n throw new TypeError('Cyclic __proto__ value')\n }\n return Reflect.setPrototypeOf(target, proto)\n }\n }\n // We wrap each trap in the handler in a try/catch and modify the error stack if they throw\n const traps = Object.getOwnPropertyNames(handler)\n traps.forEach(trap => {\n newHandler[trap] = function () {\n try {\n // Forward the call to the defined proxy handler\n return handler[trap].apply(this, arguments || [])\n } catch (err) {\n // Stack traces differ per browser, we only support chromium based ones currently\n if (!err || !err.stack || !err.stack.includes(`at `)) {\n throw err\n }\n\n // When something throws within one of our traps the Proxy will show up in error stacks\n // An earlier implementation of this code would simply strip lines with a blacklist,\n // but it makes sense to be more surgical here and only remove lines related to our Proxy.\n // We try to use a known \"anchor\" line for that and strip it with everything above it.\n // If the anchor line cannot be found for some reason we fall back to our blacklist approach.\n\n const stripWithBlacklist = (stack, stripFirstLine = true) => {\n const blacklist = [\n `at Reflect.${trap} `, // e.g. Reflect.get or Reflect.apply\n `at Object.${trap} `, // e.g. Object.get or Object.apply\n `at Object.newHandler. [as ${trap}] ` // caused by this very wrapper :-)\n ]\n return (\n err.stack\n .split('\\n')\n // Always remove the first (file) line in the stack (guaranteed to be our proxy)\n .filter((line, index) => !(index === 1 && stripFirstLine))\n // Check if the line starts with one of our blacklisted strings\n .filter(line => !blacklist.some(bl => line.trim().startsWith(bl)))\n .join('\\n')\n )\n }\n\n const stripWithAnchor = (stack, anchor) => {\n const stackArr = stack.split('\\n')\n anchor = anchor || `at Object.newHandler. [as ${trap}] ` // Known first Proxy line in chromium\n const anchorIndex = stackArr.findIndex(line =>\n line.trim().startsWith(anchor)\n )\n if (anchorIndex === -1) {\n return false // 404, anchor not found\n }\n // Strip everything from the top until we reach the anchor line\n // Note: We're keeping the 1st line (zero index) as it's unrelated (e.g. `TypeError`)\n stackArr.splice(1, anchorIndex)\n return stackArr.join('\\n')\n }\n\n // Special cases due to our nested toString proxies\n err.stack = err.stack.replace(\n 'at Object.toString (',\n 'at Function.toString ('\n )\n if ((err.stack || '').includes('at Function.toString (')) {\n err.stack = stripWithBlacklist(err.stack, false)\n throw err\n }\n\n // Try using the anchor method, fallback to blacklist if necessary\n err.stack = stripWithAnchor(err.stack) || stripWithBlacklist(err.stack)\n\n throw err // Re-throw our now sanitized error\n }\n }\n })\n return newHandler\n}",stripErrorWithAnchor:"(err, anchor) => {\n const stackArr = err.stack.split('\\n')\n const anchorIndex = stackArr.findIndex(line => line.trim().startsWith(anchor))\n if (anchorIndex === -1) {\n return err // 404, anchor not found\n }\n // Strip everything from the top until we reach the anchor line (remove anchor line as well)\n // Note: We're keeping the 1st line (zero index) as it's unrelated (e.g. `TypeError`)\n stackArr.splice(1, anchorIndex)\n err.stack = stackArr.join('\\n')\n return err\n}",replaceProperty:"(obj, propName, descriptorOverrides = {}) => {\n return Object.defineProperty(obj, propName, {\n // Copy over the existing descriptors (writable, enumerable, configurable, etc)\n ...(Object.getOwnPropertyDescriptor(obj, propName) || {}),\n // Add our overrides (e.g. value, get())\n ...descriptorOverrides\n })\n}",preloadCache:"() => {\n if (utils.cache) {\n return\n }\n utils.cache = {\n // Used in our proxies\n Reflect: {\n get: Reflect.get.bind(Reflect),\n apply: Reflect.apply.bind(Reflect)\n },\n // Used in `makeNativeString`\n nativeToStringStr: Function.toString + '' // => `function toString() { [native code] }`\n }\n}",makeNativeString:"(name = '') => {\n return utils.cache.nativeToStringStr.replace('toString', name || '')\n}",patchToString:"(obj, str = '') => {\n const handler = {\n apply: function (target, ctx) {\n // This fixes e.g. `HTMLMediaElement.prototype.canPlayType.toString + \"\"`\n if (ctx === Function.prototype.toString) {\n return utils.makeNativeString('toString')\n }\n // `toString` targeted at our proxied Object detected\n if (ctx === obj) {\n // We either return the optional string verbatim or derive the most desired result automatically\n return str || utils.makeNativeString(obj.name)\n }\n // Check if the toString protype of the context is the same as the global prototype,\n // if not indicates that we are doing a check across different windows., e.g. the iframeWithdirect` test case\n const hasSameProto = Object.getPrototypeOf(\n Function.prototype.toString\n ).isPrototypeOf(ctx.toString) // eslint-disable-line no-prototype-builtins\n if (!hasSameProto) {\n // Pass the call on to the local Function.prototype.toString instead\n return ctx.toString()\n }\n return target.call(ctx)\n }\n }\n\n const toStringProxy = new Proxy(\n Function.prototype.toString,\n utils.stripProxyFromErrors(handler)\n )\n utils.replaceProperty(Function.prototype, 'toString', {\n value: toStringProxy\n })\n}",patchToStringNested:"(obj = {}) => {\n return utils.execRecursively(obj, ['function'], utils.patchToString)\n}",redirectToString:"(proxyObj, originalObj) => {\n const handler = {\n apply: function (target, ctx) {\n // This fixes e.g. `HTMLMediaElement.prototype.canPlayType.toString + \"\"`\n if (ctx === Function.prototype.toString) {\n return utils.makeNativeString('toString')\n }\n\n // `toString` targeted at our proxied Object detected\n if (ctx === proxyObj) {\n const fallback = () =>\n originalObj && originalObj.name\n ? utils.makeNativeString(originalObj.name)\n : utils.makeNativeString(proxyObj.name)\n\n // Return the toString representation of our original object if possible\n return originalObj + '' || fallback()\n }\n\n if (typeof ctx === 'undefined' || ctx === null) {\n return target.call(ctx)\n }\n\n // Check if the toString protype of the context is the same as the global prototype,\n // if not indicates that we are doing a check across different windows., e.g. the iframeWithdirect` test case\n const hasSameProto = Object.getPrototypeOf(\n Function.prototype.toString\n ).isPrototypeOf(ctx.toString) // eslint-disable-line no-prototype-builtins\n if (!hasSameProto) {\n // Pass the call on to the local Function.prototype.toString instead\n return ctx.toString()\n }\n\n return target.call(ctx)\n }\n }\n\n const toStringProxy = new Proxy(\n Function.prototype.toString,\n utils.stripProxyFromErrors(handler)\n )\n utils.replaceProperty(Function.prototype, 'toString', {\n value: toStringProxy\n })\n}",replaceWithProxy:"(obj, propName, handler) => {\n const originalObj = obj[propName]\n const proxyObj = new Proxy(obj[propName], utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { value: proxyObj })\n utils.redirectToString(proxyObj, originalObj)\n\n return true\n}",replaceGetterWithProxy:"(obj, propName, handler) => {\n const fn = Object.getOwnPropertyDescriptor(obj, propName).get\n const fnStr = fn.toString() // special getter function string\n const proxyObj = new Proxy(fn, utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { get: proxyObj })\n utils.patchToString(proxyObj, fnStr)\n\n return true\n}",mockWithProxy:"(obj, propName, pseudoTarget, handler) => {\n const proxyObj = new Proxy(pseudoTarget, utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { value: proxyObj })\n utils.patchToString(proxyObj)\n\n return true\n}",createProxy:"(pseudoTarget, handler) => {\n const proxyObj = new Proxy(pseudoTarget, utils.stripProxyFromErrors(handler))\n utils.patchToString(proxyObj)\n\n return proxyObj\n}",splitObjPath:"objPath => ({\n // Remove last dot entry (property) ==> `HTMLMediaElement.prototype`\n objName: objPath.split('.').slice(0, -1).join('.'),\n // Extract last dot entry ==> `canPlayType`\n propName: objPath.split('.').slice(-1)[0]\n})",replaceObjPathWithProxy:"(objPath, handler) => {\n const { objName, propName } = utils.splitObjPath(objPath)\n const obj = eval(objName) // eslint-disable-line no-eval\n return utils.replaceWithProxy(obj, propName, handler)\n}",execRecursively:"(obj = {}, typeFilter = [], fn) => {\n function recurse(obj) {\n for (const key in obj) {\n if (obj[key] === undefined) {\n continue\n }\n if (obj[key] && typeof obj[key] === 'object') {\n recurse(obj[key])\n } else {\n if (obj[key] && typeFilter.includes(typeof obj[key])) {\n fn.call(this, obj[key])\n }\n }\n }\n }\n recurse(obj)\n return obj\n}",stringifyFns:"(fnObj = { hello: () => 'world' }) => {\n // Object.fromEntries() ponyfill (in 6 lines) - supported only in Node v12+, modern browsers are fine\n // https://github.com/feross/fromentries\n function fromEntries(iterable) {\n return [...iterable].reduce((obj, [key, val]) => {\n obj[key] = val\n return obj\n }, {})\n }\n return (Object.fromEntries || fromEntries)(\n Object.entries(fnObj)\n .filter(([key, value]) => typeof value === 'function')\n .map(([key, value]) => [key, value.toString()]) // eslint-disable-line no-eval\n )\n}",materializeFns:"(fnStrObj = { hello: \"() => 'world'\" }) => {\n return Object.fromEntries(\n Object.entries(fnStrObj).map(([key, value]) => {\n if (value.startsWith('function')) {\n // some trickery is needed to make oldschool functions work :-)\n return [key, eval(`() => ${value}`)()] // eslint-disable-line no-eval\n } else {\n // arrow functions just work\n return [key, eval(value)] // eslint-disable-line no-eval\n }\n })\n )\n}",makeHandler:"() => ({\n // Used by simple `navigator` getter evasions\n getterValue: value => ({\n apply(target, ctx, args) {\n // Let's fetch the value first, to trigger and escalate potential errors\n // Illegal invocations like `navigator.__proto__.vendor` will throw here\n utils.cache.Reflect.apply(...arguments)\n return value\n }\n })\n})"},_mainFunction:"(utils, opts) => {\n const isSecure = document.location.protocol.startsWith('https')\n\n // In headful on secure origins the permission should be \"default\", not \"denied\"\n if (isSecure) {\n utils.replaceGetterWithProxy(Notification, 'permission', {\n apply() {\n return 'default'\n }\n })\n }\n\n // Another weird behavior:\n // On insecure origins in headful the state is \"denied\",\n // whereas in headless it's \"prompt\"\n if (!isSecure) {\n const handler = {\n apply(target, ctx, args) {\n const param = (args || [])[0]\n\n const isNotifications =\n param && param.name && param.name === 'notifications'\n if (!isNotifications) {\n return utils.cache.Reflect.apply(...arguments)\n }\n\n return Promise.resolve(\n Object.setPrototypeOf(\n {\n state: 'denied',\n onchange: null\n },\n PermissionStatus.prototype\n )\n )\n }\n }\n // Note: Don't use `Object.getPrototypeOf` here\n utils.replaceWithProxy(Permissions.prototype, 'query', handler)\n }\n }",_args:[{}]}),(({_utilsFns:_utilsFns,_mainFunction:_mainFunction,_args:_args})=>{const utils=Object.fromEntries(Object.entries(_utilsFns).map((([key,value])=>[key,eval(value)])));utils.init(),eval(_mainFunction)(utils,..._args)})({_utilsFns:{init:"() => {\n utils.preloadCache()\n}",stripProxyFromErrors:"(handler = {}) => {\n const newHandler = {\n setPrototypeOf: function (target, proto) {\n if (proto === null)\n throw new TypeError('Cannot convert object to primitive value')\n if (Object.getPrototypeOf(target) === Object.getPrototypeOf(proto)) {\n throw new TypeError('Cyclic __proto__ value')\n }\n return Reflect.setPrototypeOf(target, proto)\n }\n }\n // We wrap each trap in the handler in a try/catch and modify the error stack if they throw\n const traps = Object.getOwnPropertyNames(handler)\n traps.forEach(trap => {\n newHandler[trap] = function () {\n try {\n // Forward the call to the defined proxy handler\n return handler[trap].apply(this, arguments || [])\n } catch (err) {\n // Stack traces differ per browser, we only support chromium based ones currently\n if (!err || !err.stack || !err.stack.includes(`at `)) {\n throw err\n }\n\n // When something throws within one of our traps the Proxy will show up in error stacks\n // An earlier implementation of this code would simply strip lines with a blacklist,\n // but it makes sense to be more surgical here and only remove lines related to our Proxy.\n // We try to use a known \"anchor\" line for that and strip it with everything above it.\n // If the anchor line cannot be found for some reason we fall back to our blacklist approach.\n\n const stripWithBlacklist = (stack, stripFirstLine = true) => {\n const blacklist = [\n `at Reflect.${trap} `, // e.g. Reflect.get or Reflect.apply\n `at Object.${trap} `, // e.g. Object.get or Object.apply\n `at Object.newHandler. [as ${trap}] ` // caused by this very wrapper :-)\n ]\n return (\n err.stack\n .split('\\n')\n // Always remove the first (file) line in the stack (guaranteed to be our proxy)\n .filter((line, index) => !(index === 1 && stripFirstLine))\n // Check if the line starts with one of our blacklisted strings\n .filter(line => !blacklist.some(bl => line.trim().startsWith(bl)))\n .join('\\n')\n )\n }\n\n const stripWithAnchor = (stack, anchor) => {\n const stackArr = stack.split('\\n')\n anchor = anchor || `at Object.newHandler. [as ${trap}] ` // Known first Proxy line in chromium\n const anchorIndex = stackArr.findIndex(line =>\n line.trim().startsWith(anchor)\n )\n if (anchorIndex === -1) {\n return false // 404, anchor not found\n }\n // Strip everything from the top until we reach the anchor line\n // Note: We're keeping the 1st line (zero index) as it's unrelated (e.g. `TypeError`)\n stackArr.splice(1, anchorIndex)\n return stackArr.join('\\n')\n }\n\n // Special cases due to our nested toString proxies\n err.stack = err.stack.replace(\n 'at Object.toString (',\n 'at Function.toString ('\n )\n if ((err.stack || '').includes('at Function.toString (')) {\n err.stack = stripWithBlacklist(err.stack, false)\n throw err\n }\n\n // Try using the anchor method, fallback to blacklist if necessary\n err.stack = stripWithAnchor(err.stack) || stripWithBlacklist(err.stack)\n\n throw err // Re-throw our now sanitized error\n }\n }\n })\n return newHandler\n}",stripErrorWithAnchor:"(err, anchor) => {\n const stackArr = err.stack.split('\\n')\n const anchorIndex = stackArr.findIndex(line => line.trim().startsWith(anchor))\n if (anchorIndex === -1) {\n return err // 404, anchor not found\n }\n // Strip everything from the top until we reach the anchor line (remove anchor line as well)\n // Note: We're keeping the 1st line (zero index) as it's unrelated (e.g. `TypeError`)\n stackArr.splice(1, anchorIndex)\n err.stack = stackArr.join('\\n')\n return err\n}",replaceProperty:"(obj, propName, descriptorOverrides = {}) => {\n return Object.defineProperty(obj, propName, {\n // Copy over the existing descriptors (writable, enumerable, configurable, etc)\n ...(Object.getOwnPropertyDescriptor(obj, propName) || {}),\n // Add our overrides (e.g. value, get())\n ...descriptorOverrides\n })\n}",preloadCache:"() => {\n if (utils.cache) {\n return\n }\n utils.cache = {\n // Used in our proxies\n Reflect: {\n get: Reflect.get.bind(Reflect),\n apply: Reflect.apply.bind(Reflect)\n },\n // Used in `makeNativeString`\n nativeToStringStr: Function.toString + '' // => `function toString() { [native code] }`\n }\n}",makeNativeString:"(name = '') => {\n return utils.cache.nativeToStringStr.replace('toString', name || '')\n}",patchToString:"(obj, str = '') => {\n const handler = {\n apply: function (target, ctx) {\n // This fixes e.g. `HTMLMediaElement.prototype.canPlayType.toString + \"\"`\n if (ctx === Function.prototype.toString) {\n return utils.makeNativeString('toString')\n }\n // `toString` targeted at our proxied Object detected\n if (ctx === obj) {\n // We either return the optional string verbatim or derive the most desired result automatically\n return str || utils.makeNativeString(obj.name)\n }\n // Check if the toString protype of the context is the same as the global prototype,\n // if not indicates that we are doing a check across different windows., e.g. the iframeWithdirect` test case\n const hasSameProto = Object.getPrototypeOf(\n Function.prototype.toString\n ).isPrototypeOf(ctx.toString) // eslint-disable-line no-prototype-builtins\n if (!hasSameProto) {\n // Pass the call on to the local Function.prototype.toString instead\n return ctx.toString()\n }\n return target.call(ctx)\n }\n }\n\n const toStringProxy = new Proxy(\n Function.prototype.toString,\n utils.stripProxyFromErrors(handler)\n )\n utils.replaceProperty(Function.prototype, 'toString', {\n value: toStringProxy\n })\n}",patchToStringNested:"(obj = {}) => {\n return utils.execRecursively(obj, ['function'], utils.patchToString)\n}",redirectToString:"(proxyObj, originalObj) => {\n const handler = {\n apply: function (target, ctx) {\n // This fixes e.g. `HTMLMediaElement.prototype.canPlayType.toString + \"\"`\n if (ctx === Function.prototype.toString) {\n return utils.makeNativeString('toString')\n }\n\n // `toString` targeted at our proxied Object detected\n if (ctx === proxyObj) {\n const fallback = () =>\n originalObj && originalObj.name\n ? utils.makeNativeString(originalObj.name)\n : utils.makeNativeString(proxyObj.name)\n\n // Return the toString representation of our original object if possible\n return originalObj + '' || fallback()\n }\n\n if (typeof ctx === 'undefined' || ctx === null) {\n return target.call(ctx)\n }\n\n // Check if the toString protype of the context is the same as the global prototype,\n // if not indicates that we are doing a check across different windows., e.g. the iframeWithdirect` test case\n const hasSameProto = Object.getPrototypeOf(\n Function.prototype.toString\n ).isPrototypeOf(ctx.toString) // eslint-disable-line no-prototype-builtins\n if (!hasSameProto) {\n // Pass the call on to the local Function.prototype.toString instead\n return ctx.toString()\n }\n\n return target.call(ctx)\n }\n }\n\n const toStringProxy = new Proxy(\n Function.prototype.toString,\n utils.stripProxyFromErrors(handler)\n )\n utils.replaceProperty(Function.prototype, 'toString', {\n value: toStringProxy\n })\n}",replaceWithProxy:"(obj, propName, handler) => {\n const originalObj = obj[propName]\n const proxyObj = new Proxy(obj[propName], utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { value: proxyObj })\n utils.redirectToString(proxyObj, originalObj)\n\n return true\n}",replaceGetterWithProxy:"(obj, propName, handler) => {\n const fn = Object.getOwnPropertyDescriptor(obj, propName).get\n const fnStr = fn.toString() // special getter function string\n const proxyObj = new Proxy(fn, utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { get: proxyObj })\n utils.patchToString(proxyObj, fnStr)\n\n return true\n}",mockWithProxy:"(obj, propName, pseudoTarget, handler) => {\n const proxyObj = new Proxy(pseudoTarget, utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { value: proxyObj })\n utils.patchToString(proxyObj)\n\n return true\n}",createProxy:"(pseudoTarget, handler) => {\n const proxyObj = new Proxy(pseudoTarget, utils.stripProxyFromErrors(handler))\n utils.patchToString(proxyObj)\n\n return proxyObj\n}",splitObjPath:"objPath => ({\n // Remove last dot entry (property) ==> `HTMLMediaElement.prototype`\n objName: objPath.split('.').slice(0, -1).join('.'),\n // Extract last dot entry ==> `canPlayType`\n propName: objPath.split('.').slice(-1)[0]\n})",replaceObjPathWithProxy:"(objPath, handler) => {\n const { objName, propName } = utils.splitObjPath(objPath)\n const obj = eval(objName) // eslint-disable-line no-eval\n return utils.replaceWithProxy(obj, propName, handler)\n}",execRecursively:"(obj = {}, typeFilter = [], fn) => {\n function recurse(obj) {\n for (const key in obj) {\n if (obj[key] === undefined) {\n continue\n }\n if (obj[key] && typeof obj[key] === 'object') {\n recurse(obj[key])\n } else {\n if (obj[key] && typeFilter.includes(typeof obj[key])) {\n fn.call(this, obj[key])\n }\n }\n }\n }\n recurse(obj)\n return obj\n}",stringifyFns:"(fnObj = { hello: () => 'world' }) => {\n // Object.fromEntries() ponyfill (in 6 lines) - supported only in Node v12+, modern browsers are fine\n // https://github.com/feross/fromentries\n function fromEntries(iterable) {\n return [...iterable].reduce((obj, [key, val]) => {\n obj[key] = val\n return obj\n }, {})\n }\n return (Object.fromEntries || fromEntries)(\n Object.entries(fnObj)\n .filter(([key, value]) => typeof value === 'function')\n .map(([key, value]) => [key, value.toString()]) // eslint-disable-line no-eval\n )\n}",materializeFns:"(fnStrObj = { hello: \"() => 'world'\" }) => {\n return Object.fromEntries(\n Object.entries(fnStrObj).map(([key, value]) => {\n if (value.startsWith('function')) {\n // some trickery is needed to make oldschool functions work :-)\n return [key, eval(`() => ${value}`)()] // eslint-disable-line no-eval\n } else {\n // arrow functions just work\n return [key, eval(value)] // eslint-disable-line no-eval\n }\n })\n )\n}",makeHandler:"() => ({\n // Used by simple `navigator` getter evasions\n getterValue: value => ({\n apply(target, ctx, args) {\n // Let's fetch the value first, to trigger and escalate potential errors\n // Illegal invocations like `navigator.__proto__.vendor` will throw here\n utils.cache.Reflect.apply(...arguments)\n return value\n }\n })\n})"},_mainFunction:"(utils, { fns, data }) => {\n fns = utils.materializeFns(fns)\n\n // That means we're running headful\n const hasPlugins = 'plugins' in navigator && navigator.plugins.length\n if (hasPlugins) {\n return // nothing to do here\n }\n\n const mimeTypes = fns.generateMimeTypeArray(utils, fns)(data.mimeTypes)\n const plugins = fns.generatePluginArray(utils, fns)(data.plugins)\n\n // Plugin and MimeType cross-reference each other, let's do that now\n // Note: We're looping through `data.plugins` here, not the generated `plugins`\n for (const pluginData of data.plugins) {\n pluginData.__mimeTypes.forEach((type, index) => {\n plugins[pluginData.name][index] = mimeTypes[type]\n\n Object.defineProperty(plugins[pluginData.name], type, {\n value: mimeTypes[type],\n writable: false,\n enumerable: false, // Not enumerable\n configurable: true\n })\n Object.defineProperty(mimeTypes[type], 'enabledPlugin', {\n value:\n type === 'application/x-pnacl'\n ? mimeTypes['application/x-nacl'].enabledPlugin // these reference the same plugin, so we need to re-use the Proxy in order to avoid leaks\n : new Proxy(plugins[pluginData.name], {}), // Prevent circular references\n writable: false,\n enumerable: false, // Important: `JSON.stringify(navigator.plugins)`\n configurable: true\n })\n })\n }\n\n const patchNavigator = (name, value) =>\n utils.replaceProperty(Object.getPrototypeOf(navigator), name, {\n get() {\n return value\n }\n })\n\n patchNavigator('mimeTypes', mimeTypes)\n patchNavigator('plugins', plugins)\n\n // All done\n }",_args:[{fns:{generateMimeTypeArray:"(utils, fns) => mimeTypesData => {\n return fns.generateMagicArray(utils, fns)(\n mimeTypesData,\n MimeTypeArray.prototype,\n MimeType.prototype,\n 'type'\n )\n}",generatePluginArray:"(utils, fns) => pluginsData => {\n return fns.generateMagicArray(utils, fns)(\n pluginsData,\n PluginArray.prototype,\n Plugin.prototype,\n 'name'\n )\n}",generateMagicArray:"(utils, fns) =>\n function(\n dataArray = [],\n proto = MimeTypeArray.prototype,\n itemProto = MimeType.prototype,\n itemMainProp = 'type'\n ) {\n // Quick helper to set props with the same descriptors vanilla is using\n const defineProp = (obj, prop, value) =>\n Object.defineProperty(obj, prop, {\n value,\n writable: false,\n enumerable: false, // Important for mimeTypes & plugins: `JSON.stringify(navigator.mimeTypes)`\n configurable: true\n })\n\n // Loop over our fake data and construct items\n const makeItem = data => {\n const item = {}\n for (const prop of Object.keys(data)) {\n if (prop.startsWith('__')) {\n continue\n }\n defineProp(item, prop, data[prop])\n }\n return patchItem(item, data)\n }\n\n const patchItem = (item, data) => {\n let descriptor = Object.getOwnPropertyDescriptors(item)\n\n // Special case: Plugins have a magic length property which is not enumerable\n // e.g. `navigator.plugins[i].length` should always be the length of the assigned mimeTypes\n if (itemProto === Plugin.prototype) {\n descriptor = {\n ...descriptor,\n length: {\n value: data.__mimeTypes.length,\n writable: false,\n enumerable: false,\n configurable: true // Important to be able to use the ownKeys trap in a Proxy to strip `length`\n }\n }\n }\n\n // We need to spoof a specific `MimeType` or `Plugin` object\n const obj = Object.create(itemProto, descriptor)\n\n // Virtually all property keys are not enumerable in vanilla\n const blacklist = [...Object.keys(data), 'length', 'enabledPlugin']\n return new Proxy(obj, {\n ownKeys(target) {\n return Reflect.ownKeys(target).filter(k => !blacklist.includes(k))\n },\n getOwnPropertyDescriptor(target, prop) {\n if (blacklist.includes(prop)) {\n return undefined\n }\n return Reflect.getOwnPropertyDescriptor(target, prop)\n }\n })\n }\n\n const magicArray = []\n\n // Loop through our fake data and use that to create convincing entities\n dataArray.forEach(data => {\n magicArray.push(makeItem(data))\n })\n\n // Add direct property access based on types (e.g. `obj['application/pdf']`) afterwards\n magicArray.forEach(entry => {\n defineProp(magicArray, entry[itemMainProp], entry)\n })\n\n // This is the best way to fake the type to make sure this is false: `Array.isArray(navigator.mimeTypes)`\n const magicArrayObj = Object.create(proto, {\n ...Object.getOwnPropertyDescriptors(magicArray),\n\n // There's one ugly quirk we unfortunately need to take care of:\n // The `MimeTypeArray` prototype has an enumerable `length` property,\n // but headful Chrome will still skip it when running `Object.getOwnPropertyNames(navigator.mimeTypes)`.\n // To strip it we need to make it first `configurable` and can then overlay a Proxy with an `ownKeys` trap.\n length: {\n value: magicArray.length,\n writable: false,\n enumerable: false,\n configurable: true // Important to be able to use the ownKeys trap in a Proxy to strip `length`\n }\n })\n\n // Generate our functional function mocks :-)\n const functionMocks = fns.generateFunctionMocks(utils)(\n proto,\n itemMainProp,\n magicArray\n )\n\n // We need to overlay our custom object with a JS Proxy\n const magicArrayObjProxy = new Proxy(magicArrayObj, {\n get(target, key = '') {\n // Redirect function calls to our custom proxied versions mocking the vanilla behavior\n if (key === 'item') {\n return functionMocks.item\n }\n if (key === 'namedItem') {\n return functionMocks.namedItem\n }\n if (proto === PluginArray.prototype && key === 'refresh') {\n return functionMocks.refresh\n }\n // Everything else can pass through as normal\n return utils.cache.Reflect.get(...arguments)\n },\n ownKeys(target) {\n // There are a couple of quirks where the original property demonstrates \"magical\" behavior that makes no sense\n // This can be witnessed when calling `Object.getOwnPropertyNames(navigator.mimeTypes)` and the absense of `length`\n // My guess is that it has to do with the recent change of not allowing data enumeration and this being implemented weirdly\n // For that reason we just completely fake the available property names based on our data to match what regular Chrome is doing\n // Specific issues when not patching this: `length` property is available, direct `types` props (e.g. `obj['application/pdf']`) are missing\n const keys = []\n const typeProps = magicArray.map(mt => mt[itemMainProp])\n typeProps.forEach((_, i) => keys.push(`${i}`))\n typeProps.forEach(propName => keys.push(propName))\n return keys\n },\n getOwnPropertyDescriptor(target, prop) {\n if (prop === 'length') {\n return undefined\n }\n return Reflect.getOwnPropertyDescriptor(target, prop)\n }\n })\n\n return magicArrayObjProxy\n }",generateFunctionMocks:"utils => (\n proto,\n itemMainProp,\n dataArray\n) => ({\n /** Returns the MimeType object with the specified index. */\n item: utils.createProxy(proto.item, {\n apply(target, ctx, args) {\n if (!args.length) {\n throw new TypeError(\n `Failed to execute 'item' on '${\n proto[Symbol.toStringTag]\n }': 1 argument required, but only 0 present.`\n )\n }\n // Special behavior alert:\n // - Vanilla tries to cast strings to Numbers (only integers!) and use them as property index lookup\n // - If anything else than an integer (including as string) is provided it will return the first entry\n const isInteger = args[0] && Number.isInteger(Number(args[0])) // Cast potential string to number first, then check for integer\n // Note: Vanilla never returns `undefined`\n return (isInteger ? dataArray[Number(args[0])] : dataArray[0]) || null\n }\n }),\n /** Returns the MimeType object with the specified name. */\n namedItem: utils.createProxy(proto.namedItem, {\n apply(target, ctx, args) {\n if (!args.length) {\n throw new TypeError(\n `Failed to execute 'namedItem' on '${\n proto[Symbol.toStringTag]\n }': 1 argument required, but only 0 present.`\n )\n }\n return dataArray.find(mt => mt[itemMainProp] === args[0]) || null // Not `undefined`!\n }\n }),\n /** Does nothing and shall return nothing */\n refresh: proto.refresh\n ? utils.createProxy(proto.refresh, {\n apply(target, ctx, args) {\n return undefined\n }\n })\n : undefined\n})"},data:{mimeTypes:[{type:"application/pdf",suffixes:"pdf",description:"",__pluginName:"Chrome PDF Viewer"},{type:"application/x-google-chrome-pdf",suffixes:"pdf",description:"Portable Document Format",__pluginName:"Chrome PDF Plugin"},{type:"application/x-nacl",suffixes:"",description:"Native Client Executable",__pluginName:"Native Client"},{type:"application/x-pnacl",suffixes:"",description:"Portable Native Client Executable",__pluginName:"Native Client"}],plugins:[{name:"Chrome PDF Plugin",filename:"internal-pdf-viewer",description:"Portable Document Format",__mimeTypes:["application/x-google-chrome-pdf"]},{name:"Chrome PDF Viewer",filename:"mhjfbmdgcfjbbpaeojofohoefgiehjai",description:"",__mimeTypes:["application/pdf"]},{name:"Native Client",filename:"internal-nacl-plugin",description:"",__mimeTypes:["application/x-nacl","application/x-pnacl"]}]}}]}),!1===navigator.webdriver||void 0===navigator.webdriver||delete Object.getPrototypeOf(navigator).webdriver,(({_utilsFns:_utilsFns,_mainFunction:_mainFunction,_args:_args})=>{const utils=Object.fromEntries(Object.entries(_utilsFns).map((([key,value])=>[key,eval(value)])));utils.init(),eval(_mainFunction)(utils,..._args)})({_utilsFns:{init:"() => {\n utils.preloadCache()\n}",stripProxyFromErrors:"(handler = {}) => {\n const newHandler = {\n setPrototypeOf: function (target, proto) {\n if (proto === null)\n throw new TypeError('Cannot convert object to primitive value')\n if (Object.getPrototypeOf(target) === Object.getPrototypeOf(proto)) {\n throw new TypeError('Cyclic __proto__ value')\n }\n return Reflect.setPrototypeOf(target, proto)\n }\n }\n // We wrap each trap in the handler in a try/catch and modify the error stack if they throw\n const traps = Object.getOwnPropertyNames(handler)\n traps.forEach(trap => {\n newHandler[trap] = function () {\n try {\n // Forward the call to the defined proxy handler\n return handler[trap].apply(this, arguments || [])\n } catch (err) {\n // Stack traces differ per browser, we only support chromium based ones currently\n if (!err || !err.stack || !err.stack.includes(`at `)) {\n throw err\n }\n\n // When something throws within one of our traps the Proxy will show up in error stacks\n // An earlier implementation of this code would simply strip lines with a blacklist,\n // but it makes sense to be more surgical here and only remove lines related to our Proxy.\n // We try to use a known \"anchor\" line for that and strip it with everything above it.\n // If the anchor line cannot be found for some reason we fall back to our blacklist approach.\n\n const stripWithBlacklist = (stack, stripFirstLine = true) => {\n const blacklist = [\n `at Reflect.${trap} `, // e.g. Reflect.get or Reflect.apply\n `at Object.${trap} `, // e.g. Object.get or Object.apply\n `at Object.newHandler. [as ${trap}] ` // caused by this very wrapper :-)\n ]\n return (\n err.stack\n .split('\\n')\n // Always remove the first (file) line in the stack (guaranteed to be our proxy)\n .filter((line, index) => !(index === 1 && stripFirstLine))\n // Check if the line starts with one of our blacklisted strings\n .filter(line => !blacklist.some(bl => line.trim().startsWith(bl)))\n .join('\\n')\n )\n }\n\n const stripWithAnchor = (stack, anchor) => {\n const stackArr = stack.split('\\n')\n anchor = anchor || `at Object.newHandler. [as ${trap}] ` // Known first Proxy line in chromium\n const anchorIndex = stackArr.findIndex(line =>\n line.trim().startsWith(anchor)\n )\n if (anchorIndex === -1) {\n return false // 404, anchor not found\n }\n // Strip everything from the top until we reach the anchor line\n // Note: We're keeping the 1st line (zero index) as it's unrelated (e.g. `TypeError`)\n stackArr.splice(1, anchorIndex)\n return stackArr.join('\\n')\n }\n\n // Special cases due to our nested toString proxies\n err.stack = err.stack.replace(\n 'at Object.toString (',\n 'at Function.toString ('\n )\n if ((err.stack || '').includes('at Function.toString (')) {\n err.stack = stripWithBlacklist(err.stack, false)\n throw err\n }\n\n // Try using the anchor method, fallback to blacklist if necessary\n err.stack = stripWithAnchor(err.stack) || stripWithBlacklist(err.stack)\n\n throw err // Re-throw our now sanitized error\n }\n }\n })\n return newHandler\n}",stripErrorWithAnchor:"(err, anchor) => {\n const stackArr = err.stack.split('\\n')\n const anchorIndex = stackArr.findIndex(line => line.trim().startsWith(anchor))\n if (anchorIndex === -1) {\n return err // 404, anchor not found\n }\n // Strip everything from the top until we reach the anchor line (remove anchor line as well)\n // Note: We're keeping the 1st line (zero index) as it's unrelated (e.g. `TypeError`)\n stackArr.splice(1, anchorIndex)\n err.stack = stackArr.join('\\n')\n return err\n}",replaceProperty:"(obj, propName, descriptorOverrides = {}) => {\n return Object.defineProperty(obj, propName, {\n // Copy over the existing descriptors (writable, enumerable, configurable, etc)\n ...(Object.getOwnPropertyDescriptor(obj, propName) || {}),\n // Add our overrides (e.g. value, get())\n ...descriptorOverrides\n })\n}",preloadCache:"() => {\n if (utils.cache) {\n return\n }\n utils.cache = {\n // Used in our proxies\n Reflect: {\n get: Reflect.get.bind(Reflect),\n apply: Reflect.apply.bind(Reflect)\n },\n // Used in `makeNativeString`\n nativeToStringStr: Function.toString + '' // => `function toString() { [native code] }`\n }\n}",makeNativeString:"(name = '') => {\n return utils.cache.nativeToStringStr.replace('toString', name || '')\n}",patchToString:"(obj, str = '') => {\n const handler = {\n apply: function (target, ctx) {\n // This fixes e.g. `HTMLMediaElement.prototype.canPlayType.toString + \"\"`\n if (ctx === Function.prototype.toString) {\n return utils.makeNativeString('toString')\n }\n // `toString` targeted at our proxied Object detected\n if (ctx === obj) {\n // We either return the optional string verbatim or derive the most desired result automatically\n return str || utils.makeNativeString(obj.name)\n }\n // Check if the toString protype of the context is the same as the global prototype,\n // if not indicates that we are doing a check across different windows., e.g. the iframeWithdirect` test case\n const hasSameProto = Object.getPrototypeOf(\n Function.prototype.toString\n ).isPrototypeOf(ctx.toString) // eslint-disable-line no-prototype-builtins\n if (!hasSameProto) {\n // Pass the call on to the local Function.prototype.toString instead\n return ctx.toString()\n }\n return target.call(ctx)\n }\n }\n\n const toStringProxy = new Proxy(\n Function.prototype.toString,\n utils.stripProxyFromErrors(handler)\n )\n utils.replaceProperty(Function.prototype, 'toString', {\n value: toStringProxy\n })\n}",patchToStringNested:"(obj = {}) => {\n return utils.execRecursively(obj, ['function'], utils.patchToString)\n}",redirectToString:"(proxyObj, originalObj) => {\n const handler = {\n apply: function (target, ctx) {\n // This fixes e.g. `HTMLMediaElement.prototype.canPlayType.toString + \"\"`\n if (ctx === Function.prototype.toString) {\n return utils.makeNativeString('toString')\n }\n\n // `toString` targeted at our proxied Object detected\n if (ctx === proxyObj) {\n const fallback = () =>\n originalObj && originalObj.name\n ? utils.makeNativeString(originalObj.name)\n : utils.makeNativeString(proxyObj.name)\n\n // Return the toString representation of our original object if possible\n return originalObj + '' || fallback()\n }\n\n if (typeof ctx === 'undefined' || ctx === null) {\n return target.call(ctx)\n }\n\n // Check if the toString protype of the context is the same as the global prototype,\n // if not indicates that we are doing a check across different windows., e.g. the iframeWithdirect` test case\n const hasSameProto = Object.getPrototypeOf(\n Function.prototype.toString\n ).isPrototypeOf(ctx.toString) // eslint-disable-line no-prototype-builtins\n if (!hasSameProto) {\n // Pass the call on to the local Function.prototype.toString instead\n return ctx.toString()\n }\n\n return target.call(ctx)\n }\n }\n\n const toStringProxy = new Proxy(\n Function.prototype.toString,\n utils.stripProxyFromErrors(handler)\n )\n utils.replaceProperty(Function.prototype, 'toString', {\n value: toStringProxy\n })\n}",replaceWithProxy:"(obj, propName, handler) => {\n const originalObj = obj[propName]\n const proxyObj = new Proxy(obj[propName], utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { value: proxyObj })\n utils.redirectToString(proxyObj, originalObj)\n\n return true\n}",replaceGetterWithProxy:"(obj, propName, handler) => {\n const fn = Object.getOwnPropertyDescriptor(obj, propName).get\n const fnStr = fn.toString() // special getter function string\n const proxyObj = new Proxy(fn, utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { get: proxyObj })\n utils.patchToString(proxyObj, fnStr)\n\n return true\n}",mockWithProxy:"(obj, propName, pseudoTarget, handler) => {\n const proxyObj = new Proxy(pseudoTarget, utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { value: proxyObj })\n utils.patchToString(proxyObj)\n\n return true\n}",createProxy:"(pseudoTarget, handler) => {\n const proxyObj = new Proxy(pseudoTarget, utils.stripProxyFromErrors(handler))\n utils.patchToString(proxyObj)\n\n return proxyObj\n}",splitObjPath:"objPath => ({\n // Remove last dot entry (property) ==> `HTMLMediaElement.prototype`\n objName: objPath.split('.').slice(0, -1).join('.'),\n // Extract last dot entry ==> `canPlayType`\n propName: objPath.split('.').slice(-1)[0]\n})",replaceObjPathWithProxy:"(objPath, handler) => {\n const { objName, propName } = utils.splitObjPath(objPath)\n const obj = eval(objName) // eslint-disable-line no-eval\n return utils.replaceWithProxy(obj, propName, handler)\n}",execRecursively:"(obj = {}, typeFilter = [], fn) => {\n function recurse(obj) {\n for (const key in obj) {\n if (obj[key] === undefined) {\n continue\n }\n if (obj[key] && typeof obj[key] === 'object') {\n recurse(obj[key])\n } else {\n if (obj[key] && typeFilter.includes(typeof obj[key])) {\n fn.call(this, obj[key])\n }\n }\n }\n }\n recurse(obj)\n return obj\n}",stringifyFns:"(fnObj = { hello: () => 'world' }) => {\n // Object.fromEntries() ponyfill (in 6 lines) - supported only in Node v12+, modern browsers are fine\n // https://github.com/feross/fromentries\n function fromEntries(iterable) {\n return [...iterable].reduce((obj, [key, val]) => {\n obj[key] = val\n return obj\n }, {})\n }\n return (Object.fromEntries || fromEntries)(\n Object.entries(fnObj)\n .filter(([key, value]) => typeof value === 'function')\n .map(([key, value]) => [key, value.toString()]) // eslint-disable-line no-eval\n )\n}",materializeFns:"(fnStrObj = { hello: \"() => 'world'\" }) => {\n return Object.fromEntries(\n Object.entries(fnStrObj).map(([key, value]) => {\n if (value.startsWith('function')) {\n // some trickery is needed to make oldschool functions work :-)\n return [key, eval(`() => ${value}`)()] // eslint-disable-line no-eval\n } else {\n // arrow functions just work\n return [key, eval(value)] // eslint-disable-line no-eval\n }\n })\n )\n}",makeHandler:"() => ({\n // Used by simple `navigator` getter evasions\n getterValue: value => ({\n apply(target, ctx, args) {\n // Let's fetch the value first, to trigger and escalate potential errors\n // Illegal invocations like `navigator.__proto__.vendor` will throw here\n utils.cache.Reflect.apply(...arguments)\n return value\n }\n })\n})"},_mainFunction:"(utils, opts) => {\n const getParameterProxyHandler = {\n apply: function(target, ctx, args) {\n const param = (args || [])[0]\n const result = utils.cache.Reflect.apply(target, ctx, args)\n // UNMASKED_VENDOR_WEBGL\n if (param === 37445) {\n return opts.vendor || 'Intel Inc.' // default in headless: Google Inc.\n }\n // UNMASKED_RENDERER_WEBGL\n if (param === 37446) {\n return opts.renderer || 'Intel Iris OpenGL Engine' // default in headless: Google SwiftShader\n }\n return result\n }\n }\n\n // There's more than one WebGL rendering context\n // https://developer.mozilla.org/en-US/docs/Web/API/WebGL2RenderingContext#Browser_compatibility\n // To find out the original values here: Object.getOwnPropertyDescriptors(WebGLRenderingContext.prototype.getParameter)\n const addProxy = (obj, propName) => {\n utils.replaceWithProxy(obj, propName, getParameterProxyHandler)\n }\n // For whatever weird reason loops don't play nice with Object.defineProperty, here's the next best thing:\n addProxy(WebGLRenderingContext.prototype, 'getParameter')\n addProxy(WebGL2RenderingContext.prototype, 'getParameter')\n }",_args:[{}]}),(()=>{try{if(window.outerWidth&&window.outerHeight)return;const n=85;window.outerWidth=window.innerWidth,window.outerHeight=window.innerHeight+n}catch(n){}})(),(({_utilsFns:_utilsFns,_mainFunction:_mainFunction,_args:_args})=>{const utils=Object.fromEntries(Object.entries(_utilsFns).map((([key,value])=>[key,eval(value)])));utils.init(),eval(_mainFunction)(utils,..._args)})({_utilsFns:{init:"() => {\n utils.preloadCache()\n}",stripProxyFromErrors:"(handler = {}) => {\n const newHandler = {\n setPrototypeOf: function (target, proto) {\n if (proto === null)\n throw new TypeError('Cannot convert object to primitive value')\n if (Object.getPrototypeOf(target) === Object.getPrototypeOf(proto)) {\n throw new TypeError('Cyclic __proto__ value')\n }\n return Reflect.setPrototypeOf(target, proto)\n }\n }\n // We wrap each trap in the handler in a try/catch and modify the error stack if they throw\n const traps = Object.getOwnPropertyNames(handler)\n traps.forEach(trap => {\n newHandler[trap] = function () {\n try {\n // Forward the call to the defined proxy handler\n return handler[trap].apply(this, arguments || [])\n } catch (err) {\n // Stack traces differ per browser, we only support chromium based ones currently\n if (!err || !err.stack || !err.stack.includes(`at `)) {\n throw err\n }\n\n // When something throws within one of our traps the Proxy will show up in error stacks\n // An earlier implementation of this code would simply strip lines with a blacklist,\n // but it makes sense to be more surgical here and only remove lines related to our Proxy.\n // We try to use a known \"anchor\" line for that and strip it with everything above it.\n // If the anchor line cannot be found for some reason we fall back to our blacklist approach.\n\n const stripWithBlacklist = (stack, stripFirstLine = true) => {\n const blacklist = [\n `at Reflect.${trap} `, // e.g. Reflect.get or Reflect.apply\n `at Object.${trap} `, // e.g. Object.get or Object.apply\n `at Object.newHandler. [as ${trap}] ` // caused by this very wrapper :-)\n ]\n return (\n err.stack\n .split('\\n')\n // Always remove the first (file) line in the stack (guaranteed to be our proxy)\n .filter((line, index) => !(index === 1 && stripFirstLine))\n // Check if the line starts with one of our blacklisted strings\n .filter(line => !blacklist.some(bl => line.trim().startsWith(bl)))\n .join('\\n')\n )\n }\n\n const stripWithAnchor = (stack, anchor) => {\n const stackArr = stack.split('\\n')\n anchor = anchor || `at Object.newHandler. [as ${trap}] ` // Known first Proxy line in chromium\n const anchorIndex = stackArr.findIndex(line =>\n line.trim().startsWith(anchor)\n )\n if (anchorIndex === -1) {\n return false // 404, anchor not found\n }\n // Strip everything from the top until we reach the anchor line\n // Note: We're keeping the 1st line (zero index) as it's unrelated (e.g. `TypeError`)\n stackArr.splice(1, anchorIndex)\n return stackArr.join('\\n')\n }\n\n // Special cases due to our nested toString proxies\n err.stack = err.stack.replace(\n 'at Object.toString (',\n 'at Function.toString ('\n )\n if ((err.stack || '').includes('at Function.toString (')) {\n err.stack = stripWithBlacklist(err.stack, false)\n throw err\n }\n\n // Try using the anchor method, fallback to blacklist if necessary\n err.stack = stripWithAnchor(err.stack) || stripWithBlacklist(err.stack)\n\n throw err // Re-throw our now sanitized error\n }\n }\n })\n return newHandler\n}",stripErrorWithAnchor:"(err, anchor) => {\n const stackArr = err.stack.split('\\n')\n const anchorIndex = stackArr.findIndex(line => line.trim().startsWith(anchor))\n if (anchorIndex === -1) {\n return err // 404, anchor not found\n }\n // Strip everything from the top until we reach the anchor line (remove anchor line as well)\n // Note: We're keeping the 1st line (zero index) as it's unrelated (e.g. `TypeError`)\n stackArr.splice(1, anchorIndex)\n err.stack = stackArr.join('\\n')\n return err\n}",replaceProperty:"(obj, propName, descriptorOverrides = {}) => {\n return Object.defineProperty(obj, propName, {\n // Copy over the existing descriptors (writable, enumerable, configurable, etc)\n ...(Object.getOwnPropertyDescriptor(obj, propName) || {}),\n // Add our overrides (e.g. value, get())\n ...descriptorOverrides\n })\n}",preloadCache:"() => {\n if (utils.cache) {\n return\n }\n utils.cache = {\n // Used in our proxies\n Reflect: {\n get: Reflect.get.bind(Reflect),\n apply: Reflect.apply.bind(Reflect)\n },\n // Used in `makeNativeString`\n nativeToStringStr: Function.toString + '' // => `function toString() { [native code] }`\n }\n}",makeNativeString:"(name = '') => {\n return utils.cache.nativeToStringStr.replace('toString', name || '')\n}",patchToString:"(obj, str = '') => {\n const handler = {\n apply: function (target, ctx) {\n // This fixes e.g. `HTMLMediaElement.prototype.canPlayType.toString + \"\"`\n if (ctx === Function.prototype.toString) {\n return utils.makeNativeString('toString')\n }\n // `toString` targeted at our proxied Object detected\n if (ctx === obj) {\n // We either return the optional string verbatim or derive the most desired result automatically\n return str || utils.makeNativeString(obj.name)\n }\n // Check if the toString protype of the context is the same as the global prototype,\n // if not indicates that we are doing a check across different windows., e.g. the iframeWithdirect` test case\n const hasSameProto = Object.getPrototypeOf(\n Function.prototype.toString\n ).isPrototypeOf(ctx.toString) // eslint-disable-line no-prototype-builtins\n if (!hasSameProto) {\n // Pass the call on to the local Function.prototype.toString instead\n return ctx.toString()\n }\n return target.call(ctx)\n }\n }\n\n const toStringProxy = new Proxy(\n Function.prototype.toString,\n utils.stripProxyFromErrors(handler)\n )\n utils.replaceProperty(Function.prototype, 'toString', {\n value: toStringProxy\n })\n}",patchToStringNested:"(obj = {}) => {\n return utils.execRecursively(obj, ['function'], utils.patchToString)\n}",redirectToString:"(proxyObj, originalObj) => {\n const handler = {\n apply: function (target, ctx) {\n // This fixes e.g. `HTMLMediaElement.prototype.canPlayType.toString + \"\"`\n if (ctx === Function.prototype.toString) {\n return utils.makeNativeString('toString')\n }\n\n // `toString` targeted at our proxied Object detected\n if (ctx === proxyObj) {\n const fallback = () =>\n originalObj && originalObj.name\n ? utils.makeNativeString(originalObj.name)\n : utils.makeNativeString(proxyObj.name)\n\n // Return the toString representation of our original object if possible\n return originalObj + '' || fallback()\n }\n\n if (typeof ctx === 'undefined' || ctx === null) {\n return target.call(ctx)\n }\n\n // Check if the toString protype of the context is the same as the global prototype,\n // if not indicates that we are doing a check across different windows., e.g. the iframeWithdirect` test case\n const hasSameProto = Object.getPrototypeOf(\n Function.prototype.toString\n ).isPrototypeOf(ctx.toString) // eslint-disable-line no-prototype-builtins\n if (!hasSameProto) {\n // Pass the call on to the local Function.prototype.toString instead\n return ctx.toString()\n }\n\n return target.call(ctx)\n }\n }\n\n const toStringProxy = new Proxy(\n Function.prototype.toString,\n utils.stripProxyFromErrors(handler)\n )\n utils.replaceProperty(Function.prototype, 'toString', {\n value: toStringProxy\n })\n}",replaceWithProxy:"(obj, propName, handler) => {\n const originalObj = obj[propName]\n const proxyObj = new Proxy(obj[propName], utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { value: proxyObj })\n utils.redirectToString(proxyObj, originalObj)\n\n return true\n}",replaceGetterWithProxy:"(obj, propName, handler) => {\n const fn = Object.getOwnPropertyDescriptor(obj, propName).get\n const fnStr = fn.toString() // special getter function string\n const proxyObj = new Proxy(fn, utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { get: proxyObj })\n utils.patchToString(proxyObj, fnStr)\n\n return true\n}",mockWithProxy:"(obj, propName, pseudoTarget, handler) => {\n const proxyObj = new Proxy(pseudoTarget, utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { value: proxyObj })\n utils.patchToString(proxyObj)\n\n return true\n}",createProxy:"(pseudoTarget, handler) => {\n const proxyObj = new Proxy(pseudoTarget, utils.stripProxyFromErrors(handler))\n utils.patchToString(proxyObj)\n\n return proxyObj\n}",splitObjPath:"objPath => ({\n // Remove last dot entry (property) ==> `HTMLMediaElement.prototype`\n objName: objPath.split('.').slice(0, -1).join('.'),\n // Extract last dot entry ==> `canPlayType`\n propName: objPath.split('.').slice(-1)[0]\n})",replaceObjPathWithProxy:"(objPath, handler) => {\n const { objName, propName } = utils.splitObjPath(objPath)\n const obj = eval(objName) // eslint-disable-line no-eval\n return utils.replaceWithProxy(obj, propName, handler)\n}",execRecursively:"(obj = {}, typeFilter = [], fn) => {\n function recurse(obj) {\n for (const key in obj) {\n if (obj[key] === undefined) {\n continue\n }\n if (obj[key] && typeof obj[key] === 'object') {\n recurse(obj[key])\n } else {\n if (obj[key] && typeFilter.includes(typeof obj[key])) {\n fn.call(this, obj[key])\n }\n }\n }\n }\n recurse(obj)\n return obj\n}",stringifyFns:"(fnObj = { hello: () => 'world' }) => {\n // Object.fromEntries() ponyfill (in 6 lines) - supported only in Node v12+, modern browsers are fine\n // https://github.com/feross/fromentries\n function fromEntries(iterable) {\n return [...iterable].reduce((obj, [key, val]) => {\n obj[key] = val\n return obj\n }, {})\n }\n return (Object.fromEntries || fromEntries)(\n Object.entries(fnObj)\n .filter(([key, value]) => typeof value === 'function')\n .map(([key, value]) => [key, value.toString()]) // eslint-disable-line no-eval\n )\n}",materializeFns:"(fnStrObj = { hello: \"() => 'world'\" }) => {\n return Object.fromEntries(\n Object.entries(fnStrObj).map(([key, value]) => {\n if (value.startsWith('function')) {\n // some trickery is needed to make oldschool functions work :-)\n return [key, eval(`() => ${value}`)()] // eslint-disable-line no-eval\n } else {\n // arrow functions just work\n return [key, eval(value)] // eslint-disable-line no-eval\n }\n })\n )\n}",makeHandler:"() => ({\n // Used by simple `navigator` getter evasions\n getterValue: value => ({\n apply(target, ctx, args) {\n // Let's fetch the value first, to trigger and escalate potential errors\n // Illegal invocations like `navigator.__proto__.vendor` will throw here\n utils.cache.Reflect.apply(...arguments)\n return value\n }\n })\n})"},_mainFunction:"(utils, opts) => {\n try {\n // Adds a contentWindow proxy to the provided iframe element\n const addContentWindowProxy = iframe => {\n const contentWindowProxy = {\n get(target, key) {\n // Now to the interesting part:\n // We actually make this thing behave like a regular iframe window,\n // by intercepting calls to e.g. `.self` and redirect it to the correct thing. :)\n // That makes it possible for these assertions to be correct:\n // iframe.contentWindow.self === window.top // must be false\n if (key === 'self') {\n return this\n }\n // iframe.contentWindow.frameElement === iframe // must be true\n if (key === 'frameElement') {\n return iframe\n }\n // Intercept iframe.contentWindow[0] to hide the property 0 added by the proxy.\n if (key === '0') {\n return undefined\n }\n return Reflect.get(target, key)\n }\n }\n\n if (!iframe.contentWindow) {\n const proxy = new Proxy(window, contentWindowProxy)\n Object.defineProperty(iframe, 'contentWindow', {\n get() {\n return proxy\n },\n set(newValue) {\n return newValue // contentWindow is immutable\n },\n enumerable: true,\n configurable: false\n })\n }\n }\n\n // Handles iframe element creation, augments `srcdoc` property so we can intercept further\n const handleIframeCreation = (target, thisArg, args) => {\n const iframe = target.apply(thisArg, args)\n\n // We need to keep the originals around\n const _iframe = iframe\n const _srcdoc = _iframe.srcdoc\n\n // Add hook for the srcdoc property\n // We need to be very surgical here to not break other iframes by accident\n Object.defineProperty(iframe, 'srcdoc', {\n configurable: true, // Important, so we can reset this later\n get: function() {\n return _srcdoc\n },\n set: function(newValue) {\n addContentWindowProxy(this)\n // Reset property, the hook is only needed once\n Object.defineProperty(iframe, 'srcdoc', {\n configurable: false,\n writable: false,\n value: _srcdoc\n })\n _iframe.srcdoc = newValue\n }\n })\n return iframe\n }\n\n // Adds a hook to intercept iframe creation events\n const addIframeCreationSniffer = () => {\n /* global document */\n const createElementHandler = {\n // Make toString() native\n get(target, key) {\n return Reflect.get(target, key)\n },\n apply: function(target, thisArg, args) {\n const isIframe =\n args && args.length && `${args[0]}`.toLowerCase() === 'iframe'\n if (!isIframe) {\n // Everything as usual\n return target.apply(thisArg, args)\n } else {\n return handleIframeCreation(target, thisArg, args)\n }\n }\n }\n // All this just due to iframes with srcdoc bug\n utils.replaceWithProxy(\n document,\n 'createElement',\n createElementHandler\n )\n }\n\n // Let's go\n addIframeCreationSniffer()\n } catch (err) {\n // console.warn(err)\n }\n }",_args:[]}); \ No newline at end of file diff --git a/feapder/utils/log.py b/feapder/utils/log.py index 6d6311d6..e993f760 100644 --- a/feapder/utils/log.py +++ b/feapder/utils/log.py @@ -5,7 +5,7 @@ @summary: --------- @author: Boris -@email: boris@bzkj.tech +@email: boris_liu@foxmail.com """ import logging @@ -13,34 +13,37 @@ import sys from logging.handlers import BaseRotatingHandler +import loguru from better_exceptions import format_exception import feapder.setting as setting -LOG_FORMAT = "%(threadName)s|%(asctime)s|%(filename)s|%(funcName)s|line:%(lineno)d|%(levelname)s| %(message)s" -PRINT_EXCEPTION_DETAILS = True + +class InterceptHandler(logging.Handler): + def emit(self, record): + # Retrieve context where the logging call occurred, this happens to be in the 6th frame upward + logger_opt = loguru.logger.opt(depth=6, exception=record.exc_info) + logger_opt.log(record.levelname, record.getMessage()) # 重写 RotatingFileHandler 自定义log的文件名 # 原来 xxx.log xxx.log.1 xxx.log.2 xxx.log.3 文件由近及远 -# 现在 xxx.log xxx1.log xxx2.log 如果backupCount 是2位数时 则 01 02 03 三位数 001 002 .. 文件由近及远 +# 现在 xxx.log xxx1.log xxx2.log 如果backup_count 是2位数时 则 01 02 03 三位数 001 002 .. 文件由近及远 class RotatingFileHandler(BaseRotatingHandler): def __init__( - self, filename, mode="a", maxBytes=0, backupCount=0, encoding=None, delay=0 + self, filename, mode="a", max_bytes=0, backup_count=0, encoding=None, delay=0 ): - # if maxBytes > 0: - # mode = 'a' BaseRotatingHandler.__init__(self, filename, mode, encoding, delay) - self.maxBytes = maxBytes - self.backupCount = backupCount - self.placeholder = str(len(str(backupCount))) + self.max_bytes = max_bytes + self.backup_count = backup_count + self.placeholder = str(len(str(backup_count))) def doRollover(self): if self.stream: self.stream.close() self.stream = None - if self.backupCount > 0: - for i in range(self.backupCount - 1, 0, -1): + if self.backup_count > 0: + for i in range(self.backup_count - 1, 0, -1): sfn = ("%0" + self.placeholder + "d.") % i # '%2d.'%i -> 02 sfn = sfn.join(self.baseFilename.split(".")) # sfn = "%d_%s" % (i, self.baseFilename) @@ -64,19 +67,27 @@ def doRollover(self): self.stream = self._open() def shouldRollover(self, record): - if self.stream is None: # delay was set... self.stream = self._open() - if self.maxBytes > 0: # are we rolling over? + if self.max_bytes > 0: # are we rolling over? msg = "%s\n" % self.format(record) self.stream.seek(0, 2) # due to non-posix-compliant Windows feature - if self.stream.tell() + len(msg) >= self.maxBytes: + if self.stream.tell() + len(msg) >= self.max_bytes: return 1 return 0 def get_logger( - name, path="", log_level="DEBUG", is_write_to_file=False, is_write_to_stdout=True + name=None, + path=None, + log_level=None, + is_write_to_console=None, + is_write_to_file=None, + color=None, + mode=None, + max_bytes=None, + backup_count=None, + encoding=None, ): """ @summary: 获取log @@ -84,17 +95,44 @@ def get_logger( @param name: log名 @param path: log文件存储路径 如 D://xxx.log @param log_level: log等级 CRITICAL/ERROR/WARNING/INFO/DEBUG + @param is_write_to_console: 是否输出到控制台 @param is_write_to_file: 是否写入到文件 默认否 + @param color：是否有颜色 + @param mode：写文件模式 + @param max_bytes： 每个日志文件的最大字节数 + @param backup_count：日志文件保留数量 + @param encoding：日志文件编码 --------- @result: """ + # 加载setting里最新的值 + name = name or setting.LOG_NAME + path = path or setting.LOG_PATH + log_level = log_level or setting.LOG_LEVEL + is_write_to_console = ( + is_write_to_console + if is_write_to_console is not None + else setting.LOG_IS_WRITE_TO_CONSOLE + ) + is_write_to_file = ( + is_write_to_file + if is_write_to_file is not None + else setting.LOG_IS_WRITE_TO_FILE + ) + color = color if color is not None else setting.LOG_COLOR + mode = mode or setting.LOG_MODE + max_bytes = max_bytes or setting.LOG_MAX_BYTES + backup_count = backup_count or setting.LOG_BACKUP_COUNT + encoding = encoding or setting.LOG_ENCODING + + # logger 配置 name = name.split(os.sep)[-1].split(".")[0] # 取文件名 logger = logging.getLogger(name) logger.setLevel(log_level) - formatter = logging.Formatter(LOG_FORMAT) - if PRINT_EXCEPTION_DETAILS: + formatter = logging.Formatter(setting.LOG_FORMAT) + if setting.PRINT_EXCEPTION_DETAILS: formatter.formatException = lambda exc_info: format_exception(*exc_info) # 定义一个RotatingFileHandler，最多备份5个日志文件，每个日志文件最大10M @@ -103,11 +141,20 @@ def get_logger( os.makedirs(os.path.dirname(path)) rf_handler = RotatingFileHandler( - path, mode="w", maxBytes=10 * 1024 * 1024, backupCount=20, encoding="utf8" + path, + mode=mode, + max_bytes=max_bytes, + backup_count=backup_count, + encoding=encoding, ) rf_handler.setFormatter(formatter) logger.addHandler(rf_handler) - if is_write_to_stdout: + if color and is_write_to_console: + loguru_handler = InterceptHandler() + loguru_handler.setFormatter(formatter) + # logging.basicConfig(handlers=[loguru_handler], level=0) + logger.addHandler(loguru_handler) + elif is_write_to_console: stream_handler = logging.StreamHandler() stream_handler.stream = sys.stdout stream_handler.setFormatter(formatter) @@ -165,16 +212,60 @@ def get_logger( ] # 关闭日志打印 +OTHERS_LOG_LEVAL = eval("logging." + setting.OTHERS_LOG_LEVAL) for STOP_LOG in STOP_LOGS: - log_level = eval("logging." + setting.OTHERS_LOG_LEVAL) - logging.getLogger(STOP_LOG).setLevel(log_level) + logging.getLogger(STOP_LOG).setLevel(OTHERS_LOG_LEVAL) # print(logging.Logger.manager.loggerDict) # 取使用debug模块的name -# 日志级别大小关系为：critical > error > warning > info > debug -log = get_logger( - name=setting.LOG_NAME, - path=setting.LOG_PATH, - log_level=setting.LOG_LEVEL, - is_write_to_file=setting.LOG_IS_WRITE_TO_FILE, -) +# 日志级别大小关系为：CRITICAL > ERROR > WARNING > INFO > DEBUG + + +class Log: + log = None + + def func(self, log_level): + def wrapper(msg, *args, **kwargs): + if self.isEnabledFor(log_level): + self._log(log_level, msg, args, **kwargs) + + return wrapper + + def __getattr__(self, name): + # 调用log时再初始化，为了加载最新的setting + if self.__class__.log is None: + self.__class__.log = get_logger() + return getattr(self.__class__.log, name) + + @property + def debug(self): + return self.__class__.log.debug + + @property + def info(self): + return self.__class__.log.info + + @property + def success(self): + log_level = logging.INFO + 1 + logging.addLevelName(log_level, "success".upper()) + return self.func(log_level) + + @property + def warning(self): + return self.__class__.log.warning + + @property + def exception(self): + return self.__class__.log.exception + + @property + def error(self): + return self.__class__.log.error + + @property + def critical(self): + return self.__class__.log.critical + + +log = Log() diff --git a/feapder/utils/metrics.py b/feapder/utils/metrics.py new file mode 100644 index 00000000..ab88ee1e --- /dev/null +++ b/feapder/utils/metrics.py @@ -0,0 +1,562 @@ +import concurrent.futures +import json +import os +import queue +import random +import socket +import string +import threading +import time +from collections import Counter +from typing import Any + +from influxdb import InfluxDBClient + +from feapder import setting +from feapder.utils.log import log +from feapder.utils.tools import aio_wrap, ensure_float, ensure_int + +_inited_pid = None +# this thread should stop running in the forked process +_executor = concurrent.futures.ThreadPoolExecutor( + max_workers=1, thread_name_prefix="metrics" +) + + +class MetricsEmitter: + def __init__( + self, + influxdb, + *, + batch_size=10, + max_timer_seq=0, + emit_interval=10, + retention_policy=None, + ratio=1.0, + debug=False, + add_hostname=False, + max_points=10240, + default_tags=None, + ): + """ + Args: + influxdb: influxdb instance + batch_size: 打点的批次大小 + max_timer_seq: 每个时间间隔内最多收集多少个 timer 类型点, 0 表示不限制 + emit_interval: 最多等待多长时间必须打点 + retention_policy: 对应的 retention policy + ratio: store 和 timer 类型采样率，比如 0.1 表示只有 10% 的点会留下 + debug: 是否打印调试日志 + add_hostname: 是否添加 hostname 作为 tag + max_points: 本地 buffer 最多累计多少个点 + """ + self.pending_points = queue.Queue() + self.batch_size = batch_size + self.influxdb: InfluxDBClient = influxdb + self.tagkv = {} + self.max_timer_seq = max_timer_seq + self.lock = threading.Lock() + self.hostname = socket.gethostname() + self.last_emit_ts = time.time() # 上次提交时间 + self.emit_interval = emit_interval # 提交间隔 + self.max_points = max_points + self.retention_policy = retention_policy # 支持自定义保留策略 + self.debug = debug + self.add_hostname = add_hostname + self.ratio = ratio + self.default_tags = default_tags or {} + + def define_tagkv(self, tagk, tagvs): + self.tagkv[tagk] = set(tagvs) + + def _point_tagset(self, p): + return f"{p['measurement']}-{sorted(p['tags'].items())}-{p['time']}" + + def _make_time_to_ns(self, _time): + """ + 将时间转换为 ns 级别的时间戳，补足长度 19 位 + Args: + _time: + + Returns: + + """ + time_len = len(str(_time)) + random_str = "".join(random.sample(string.digits, 19 - time_len)) + return int(str(_time) + random_str) + + def _accumulate_points(self, points): + """ + 对于处于同一个 key 的点做聚合 + + - 对于 counter 类型，同一个 key 的值(_count)可以累加 + - 对于 store 类型，不做任何操作，influxdb 会自行覆盖 + - 对于 timer 类型，通过添加一个 _seq 值来区分每个不同的点 + """ + counters = {} # 临时保留 counter 类型的值 + timer_seqs = Counter() # 记录不同 key 的 timer 序列号 + new_points = [] + + for point in points: + point_type = point["tags"].get("_type", None) + tagset = self._point_tagset(point) + + # counter 类型全部聚合，不做丢弃 + if point_type == "counter": + if tagset not in counters: + counters[tagset] = point + else: + counters[tagset]["fields"]["_count"] += point["fields"]["_count"] + elif point_type == "timer": + if self.max_timer_seq and timer_seqs[tagset] > self.max_timer_seq: + continue + # 掷一把骰子，如果足够幸运才打点 + if self.ratio < 1.0 and random.random() > self.ratio: + continue + # 增加 _seq tag，以便区分不同的点 + point["tags"]["_seq"] = timer_seqs[tagset] + point["time"] = self._make_time_to_ns(point["time"]) + timer_seqs[tagset] += 1 + new_points.append(point) + else: + if self.ratio < 1.0 and random.random() > self.ratio: + continue + point["time"] = self._make_time_to_ns(point["time"]) + new_points.append(point) + + for point in counters.values(): + # 修改下counter类型的点的时间戳，补足19位, 伪装成纳秒级时间戳，防止influxdb对同一秒内的数据进行覆盖 + point["time"] = self._make_time_to_ns(point["time"]) + new_points.append(point) + + # 把拟合后的 counter 值添加进来 + new_points.append(point) + return new_points + + def _get_ready_emit(self, force=False): + """ + 把当前 pending 的值做聚合并返回 + """ + if self.debug: + log.info("got %s raw points", self.pending_points.qsize()) + + # 从 pending 中读取点, 设定一个最大值，避免一直打点，一直获取 + points = [] + while len(points) < self.max_points or force: + try: + points.append(self.pending_points.get_nowait()) + except queue.Empty: + break + + # 聚合点 + points = self._accumulate_points(points) + + if self.debug: + log.info("got %s point", len(points)) + log.info(json.dumps(points, indent=4)) + + return points + + def emit(self, point=None, force=False): + """ + 1. 添加新点到 pending + 2. 如果符合条件，尝试聚合并打点 + 3. 更新打点时间 + + :param point: + :param force: 强制提交所有点 默认False + :return: + """ + if point: + self.pending_points.put(point) + + # 判断是否需要提交点 1、数量 2、间隔 3、强力打点 + if not ( + force + or self.pending_points.qsize() >= self.max_points # noqa: W503 + or time.time() - self.last_emit_ts > self.emit_interval # noqa: W503 + ): + return + + # 需要打点，读取可以打点的值, 确保只有一个线程在做点的压缩 + with self.lock: + points = self._get_ready_emit(force=force) + + if not points: + return + try: + # h(hour) m(minutes), s(seconds), ms(milliseconds), u(microseconds), n(nanoseconds) + self.influxdb.write_points( + points, + batch_size=self.batch_size, + time_precision="n", + retention_policy=self.retention_policy, + ) + except Exception: + log.exception("error writing points") + + self.last_emit_ts = time.time() + + def flush(self): + if self.debug: + log.info("start draining points %s", self.pending_points.qsize()) + self.emit(force=True) + + def close(self): + self.flush() + try: + self.influxdb.close() + except Exception as e: + log.exception(e) + + def make_point(self, measurement, tags: dict, fields: dict, timestamp=None): + """ + 默认的时间戳是"秒"级别的 + """ + assert measurement, "measurement can't be null" + tags = tags.copy() if tags else {} + tags.update(self.default_tags) + fields = fields.copy() if fields else {} + if timestamp is None: + timestamp = int(time.time()) + # 支持自定义hostname + if self.add_hostname and "hostname" not in tags: + tags["hostname"] = self.hostname + point = dict(measurement=measurement, tags=tags, fields=fields, time=timestamp) + if self.tagkv: + for tagk, tagv in tags.items(): + if tagv not in self.tagkv[tagk]: + raise ValueError("tag value = %s not in %s", tagv, self.tagkv[tagk]) + return point + + def get_counter_point( + self, + measurement: str, + key: str = None, + count: int = 1, + tags: dict = None, + timestamp: int = None, + ): + """ + counter 不能被覆盖 + """ + tags = tags.copy() if tags else {} + if key is not None: + tags["_key"] = key + tags["_type"] = "counter" + count = ensure_int(count) + fields = dict(_count=count) + point = self.make_point(measurement, tags, fields, timestamp=timestamp) + return point + + def get_store_point( + self, + measurement: str, + key: str = None, + value: Any = 0, + tags: dict = None, + timestamp=None, + ): + tags = tags.copy() if tags else {} + if key is not None: + tags["_key"] = key + tags["_type"] = "store" + fields = dict(_value=value) + point = self.make_point(measurement, tags, fields, timestamp=timestamp) + return point + + def get_timer_point( + self, + measurement: str, + key: str = None, + duration: float = 0, + tags: dict = None, + timestamp=None, + ): + tags = tags.copy() if tags else {} + if key is not None: + tags["_key"] = key + tags["_type"] = "timer" + fields = dict(_duration=ensure_float(duration)) + point = self.make_point(measurement, tags, fields, timestamp=timestamp) + return point + + def emit_any(self, *args, **kwargs): + point = self.make_point(*args, **kwargs) + self.emit(point) + + def emit_counter(self, *args, **kwargs): + point = self.get_counter_point(*args, **kwargs) + self.emit(point) + + def emit_store(self, *args, **kwargs): + point = self.get_store_point(*args, **kwargs) + self.emit(point) + + def emit_timer(self, *args, **kwargs): + point = self.get_timer_point(*args, **kwargs) + self.emit(point) + + +_emitter: MetricsEmitter = None +_measurement: str = None + + +def init( + *, + influxdb_host=None, + influxdb_port=None, + influxdb_udp_port=None, + influxdb_database=None, + influxdb_user=None, + influxdb_password=None, + influxdb_measurement=None, + retention_policy=None, + retention_policy_duration="180d", + emit_interval=60, + batch_size=100, + debug=False, + use_udp=False, + timeout=22, + ssl=False, + retention_policy_replication: str = "1", + set_retention_policy_default=True, + **kwargs, +): + """ + 打点监控初始化 + Args: + influxdb_host: + influxdb_port: + influxdb_udp_port: + influxdb_database: + influxdb_user: + influxdb_password: + influxdb_measurement: 存储的表，也可以在打点的时候指定 + retention_policy: 保留策略 + retention_policy_duration: 保留策略过期时间 + emit_interval: 打点最大间隔 + batch_size: 打点的批次大小 + debug: 是否开启调试 + use_udp: 是否使用udp协议打点 + timeout: 与influxdb建立连接时的超时时间 + ssl: 是否使用https协议 + retention_policy_replication: 保留策略的副本数, 确保数据的可靠性和高可用性。如果一个节点发生故障，其他节点可以继续提供服务，从而避免数据丢失和服务不可用的情况 + set_retention_policy_default: 是否设置为默认的保留策略，当retention_policy初次创建时有效 + **kwargs: 可传递MetricsEmitter类的参数 + + Returns: + + """ + global _inited_pid, _emitter, _measurement + if _inited_pid == os.getpid(): + return + + influxdb_host = influxdb_host or setting.INFLUXDB_HOST + influxdb_port = influxdb_port or setting.INFLUXDB_PORT + influxdb_udp_port = influxdb_udp_port or setting.INFLUXDB_UDP_PORT + influxdb_database = influxdb_database or setting.INFLUXDB_DATABASE + influxdb_user = influxdb_user or setting.INFLUXDB_USER + influxdb_password = influxdb_password or setting.INFLUXDB_PASSWORD + _measurement = influxdb_measurement or setting.INFLUXDB_MEASUREMENT + retention_policy = ( + retention_policy or f"{influxdb_database}_{retention_policy_duration}" + ) + + if not all( + [ + influxdb_host, + influxdb_port, + influxdb_udp_port, + influxdb_database, + influxdb_user, + influxdb_password, + ] + ): + return + + influxdb_client = InfluxDBClient( + host=influxdb_host, + port=influxdb_port, + udp_port=influxdb_udp_port, + database=influxdb_database, + use_udp=use_udp, + timeout=timeout, + username=influxdb_user, + password=influxdb_password, + ssl=ssl, + ) + # 创建数据库 + if influxdb_database: + try: + influxdb_client.create_database(influxdb_database) + influxdb_client.create_retention_policy( + retention_policy, + retention_policy_duration, + replication=retention_policy_replication, + default=set_retention_policy_default, + ) + except Exception as e: + log.error("metrics init falied: {}".format(e)) + return + + _emitter = MetricsEmitter( + influxdb_client, + debug=debug, + batch_size=batch_size, + retention_policy=retention_policy, + emit_interval=emit_interval, + **kwargs, + ) + _inited_pid = os.getpid() + log.info("metrics init successfully") + + +def emit_any( + tags: dict, + fields: dict, + *, + classify: str = "", + measurement: str = None, + timestamp=None, +): + """ + 原生的打点，不进行额外的处理 + Args: + tags: influxdb的tag的字段和值 + fields: influxdb的field的字段和值 + classify: 点的类别 + measurement: 存储的表 + timestamp: 点的时间戳，默认为当前时间 + + Returns: + + """ + if not _emitter: + return + + tags = tags or {} + tags["_classify"] = classify + measurement = measurement or _measurement + _emitter.emit_any(measurement, tags, fields, timestamp) + + +def emit_counter( + key: str = None, + count: int = 1, + *, + classify: str = "", + tags: dict = None, + measurement: str = None, + timestamp: int = None, +): + """ + 聚合打点，即会将一段时间内的点求和，然后打一个点数和 + Args: + key: 与点绑定的key值 + count: 点数 + classify: 点的类别 + tags: influxdb的tag的字段和值 + measurement: 存储的表 + timestamp: 点的时间戳，默认为当前时间 + + Returns: + + """ + if not _emitter: + return + + tags = tags or {} + tags["_classify"] = classify + measurement = measurement or _measurement + _emitter.emit_counter(measurement, key, count, tags, timestamp) + + +def emit_timer( + key: str = None, + duration: float = 0, + *, + classify: str = "", + tags: dict = None, + measurement: str = None, + timestamp=None, +): + """ + 时间打点，用于监控程序的运行时长等，每个duration一个点，不会被覆盖 + Args: + key: 与点绑定的key值 + duration: 时长 + classify: 点的类别 + tags: influxdb的tag的字段和值 + measurement: 存储的表 + timestamp: 点的时间戳，默认为当前时间 + + Returns: + + """ + if not _emitter: + return + + tags = tags or {} + tags["_classify"] = classify + measurement = measurement or _measurement + _emitter.emit_timer(measurement, key, duration, tags, timestamp) + + +def emit_store( + key: str = None, + value: Any = 0, + *, + classify: str = "", + tags: dict = None, + measurement: str = None, + timestamp=None, +): + """ + 直接打点，不进行额外的处理 + Args: + key: 与点绑定的key值 + value: 点的值 + classify: 点的类别 + tags: influxdb的tag的字段和值 + measurement: 存储的表 + timestamp: 点的时间戳，默认为当前时间 + + Returns: + + """ + if not _emitter: + return + + tags = tags or {} + tags["_classify"] = classify + measurement = measurement or _measurement + _emitter.emit_store(measurement, key, value, tags, timestamp) + + +def flush(): + """ + 强刷点到influxdb + Returns: + + """ + if not _emitter: + return + _emitter.flush() + + +def close(): + """ + 关闭 + Returns: + + """ + if not _emitter: + return + _emitter.close() + + +# 协程打点 +aemit_counter = aio_wrap(executor=_executor)(emit_counter) +aemit_store = aio_wrap(executor=_executor)(emit_store) +aemit_timer = aio_wrap(executor=_executor)(emit_timer) diff --git a/feapder/utils/perfect_dict.py b/feapder/utils/perfect_dict.py new file mode 100644 index 00000000..45081966 --- /dev/null +++ b/feapder/utils/perfect_dict.py @@ -0,0 +1,94 @@ +# -*- coding: utf-8 -*- +""" +Created on 2021/4/8 11:32 上午 +--------- +@summary: +--------- +@author: Boris +@email: boris_liu@foxmail.com +""" + + +def ensure_value(value): + if isinstance(value, (list, tuple)): + _value = [] + for v in value: + _value.append(ensure_value(v)) + + if isinstance(value, tuple): + value = tuple(_value) + else: + value = _value + + if isinstance(value, dict): + return PerfectDict(value) + else: + return value + + +class PerfectDict(dict): + """ + >>> data = PerfectDict({"id":1, "url":"xxx"}) + >>> data + {'id': 1, 'url': 'xxx'} + >>> data = PerfectDict(id=1, url="xxx") + >>> data + {'id': 1, 'url': 'xxx'} + >>> data.id + 1 + >>> data.get("id") + 1 + >>> data["id"] + 1 + >>> id, url = data + >>> id + 1 + >>> url + 'xxx' + >>> data[0] + 1 + >>> data[1] + 'xxx' + >>> data = PerfectDict({"a": 1, "b": {"b1": 2}, "c": [{"c1": [{"d": 1}]}]}) + >>> data.b.b1 + 2 + >>> data[1].b1 + 2 + >>> data.get("b").b1 + 2 + >>> data.c[0].c1 + [{'d': 1}] + >>> data.c[0].c1[0] + {'d': 1} + """ + + def __init__(self, _dict: dict = None, _values: list = None, **kwargs): + self.__dict__ = _dict or kwargs or {} + self.__dict__.pop("__values__", None) + super().__init__(self.__dict__, **kwargs) + self.__values__ = _values or list(self.__dict__.values()) + + def __getitem__(self, key): + if isinstance(key, int): + value = self.__values__[key] + else: + value = self.__dict__[key] + + return ensure_value(value) + + def __iter__(self, *args, **kwargs): + for value in self.__values__: + yield ensure_value(value) + + def __getattribute__(self, item): + value = object.__getattribute__(self, item) + if item == "__dict__" or item == "__values__": + return value + return ensure_value(value) + + def get(self, key, default=None): + if key in self.__dict__: + value = self.__dict__[key] + return ensure_value(value) + + return default diff --git a/feapder/utils/redis_lock.py b/feapder/utils/redis_lock.py index 7b902877..9df0b85d 100644 --- a/feapder/utils/redis_lock.py +++ b/feapder/utils/redis_lock.py @@ -5,102 +5,90 @@ @summary: --------- @author: Boris -@email: boris@bzkj.tech +@email: boris_liu@foxmail.com """ +import threading import time -import feapder.utils.log as log +from feapder.db.redisdb import RedisDB +from feapder.utils.log import log -class RedisLock(object): +class RedisLock: + redis_cli = None + def __init__( - self, - key, - timeout=300, - wait_timeout=8 * 3600, - break_wait=None, - redis_cli=None, - logger=None, + self, key, *, wait_timeout=0, lock_timeout=86400, redis_cli=None, redis_url=None ): """ redis超时锁 - :param key: 关键字 不同项目区分 - :param timeout: 锁超时时间 - :param wait_timeout: 等待加锁超时时间 默认8小时 防止多线程竞争时可能出现的 某个线程无限等待 - <=0 则不等待 直接加锁失败 - :param break_wait: 可自定义函数 灵活控制 wait_timeout 时间 当此函数返回True时 不再wait - :param redis_cli: redis客户端 + :param key: 存储锁的key redis_lock:[key] + :param wait_timeout: 等待加锁超时时间，为0时则不等待加锁，加锁失败 + :param lock_timeout: 锁超时时间 为0时则不会超时，直到锁释放或意外退出，默认超时为1天 + :param redis_cli: redis客户端对象 + :param redis_url: redis连接地址，若redis_cli传值，则不使用redis_url 用法示例: - with RedisLock(key="test", timeout=10, wait_timeout=100, redis_uri="") as _lock: + with RedisLock(key="test") as _lock: if _lock.locked: # 用来判断是否加上了锁 # do somethings """ - self.redis_index = -1 - if not key: - raise Exception("lock key is empty") - if not redis_cli: - raise Exception("redis_cli is empty") - self.redis_conn = redis_cli - - self.logger = logger or log.get_logger(__file__) - + self.redis_url = redis_url self.lock_key = "redis_lock:{}".format(key) # 锁超时时间 - self.timeout = timeout + self.lock_timeout = lock_timeout # 等待加锁时间 self.wait_timeout = wait_timeout - # wait中断函数 - self.break_wait = break_wait - if self.break_wait is None: - self.break_wait = lambda: False - if not callable(self.break_wait): - raise TypeError( - "break_wait must be function or None, but: {}".format( - type(self.break_wait) - ) - ) - self.locked = False + self.stop_prolong_life = False + + @property + def redis_conn(self): + if not self.__class__.redis_cli: + self.__class__.redis_cli = RedisDB(url=self.redis_url).get_redis_obj() + + return self.__class__.redis_cli + + @redis_conn.setter + def redis_conn(self, cli): + if cli: + self.__class__.redis_cli = cli def __enter__(self): if not self.locked: self.acquire() + if self.locked: + # 延长锁的时间 + thread = threading.Thread(target=self.prolong_life) + thread.daemon = True + thread.start() return self def __exit__(self, exc_type, exc_val, exc_tb): + self.stop_prolong_life = True self.release() def __repr__(self): - return "".format(self.lock_key, self.redis_index) + return "".format(self.lock_key) def acquire(self): start = time.time() - self.logger.debug("准备获取锁{} ...".format(self)) - while 1: + while True: # 尝试加锁 - if self.redis_conn.setnx(self.lock_key, time.time()): - self.redis_conn.expire(self.lock_key, self.timeout) + if self.redis_conn.set(self.lock_key, time.time(), nx=True, ex=5): self.locked = True - self.logger.debug("加锁成功: {}".format(self)) break - else: - # 修复bug： 当加锁时被干掉 导致没有设置expire成功 锁无限存在 - if self.redis_conn.ttl(self.lock_key) < 0: - self.redis_conn.delete(self.lock_key) if self.wait_timeout > 0: if time.time() - start > self.wait_timeout: + log.debug("获取锁失败") break else: - # 不等待 - break - if self.break_wait(): - self.logger.debug("break_wait 生效 不再等待加锁") + log.debug("获取锁失败") break - self.logger.debug("等待加锁: {} wait:{}".format(self, time.time() - start)) + log.debug("等待锁: {} wait:{}".format(self, time.time() - start)) if self.wait_timeout > 10: time.sleep(5) else: @@ -113,15 +101,22 @@ def release(self): self.locked = False return - def prolong_life(self, life_time: int) -> int: + def prolong_life(self): """ - 延长这个锁的超时时间 - :param life_time: 延长时间 + 延长锁的过期时间 :return: """ - expire = self.redis_conn.ttl(self.lock_key) - if expire < 0: - return expire - expire += life_time - self.redis_conn.expire(self.lock_key, expire) - return self.redis_conn.ttl(self.lock_key) + + spend_time = 0 + while not self.stop_prolong_life: + expire = self.redis_conn.ttl(self.lock_key) + if expire < 0: # key 不存在 + time.sleep(1) + continue + self.redis_conn.expire(self.lock_key, expire + 5) # 延长5秒 + time.sleep(expire) # 临过期5秒前，再次延长 + spend_time += expire + if self.lock_timeout and spend_time > self.lock_timeout: + log.info("锁超时，释放") + self.redis_conn.delete(self.lock_key) + break diff --git a/feapder/utils/tail_thread.py b/feapder/utils/tail_thread.py new file mode 100644 index 00000000..eda266d5 --- /dev/null +++ b/feapder/utils/tail_thread.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +""" +Created on 2024/3/19 20:00 +--------- +@summary: +--------- +@author: Boris +@email: boris_liu@foxmail.com +""" +import sys +import threading + + +class TailThread(threading.Thread): + """ + 所有子线程结束后，主线程才会退出 + """ + + def start(self) -> None: + """ + 解决python3.12 RuntimeError: cannot join thread before it is started的报错 + """ + super().start() + + if sys.version_info.minor >= 12 and sys.version_info.major >= 3: + for thread in threading.enumerate(): + if ( + thread.daemon + or thread is threading.current_thread() + or not thread.is_alive() + ): + continue + thread.join() diff --git a/feapder/utils/tools.py b/feapder/utils/tools.py index 632ba2e3..31952876 100644 --- a/feapder/utils/tools.py +++ b/feapder/utils/tools.py @@ -5,20 +5,25 @@ @summary: 工具 --------- @author: Boris -@email: boris@bzkj.tech +@email: boris_liu@foxmail.com """ +import asyncio +import base64 import calendar import codecs import configparser # 读配置文件的 import datetime import functools import hashlib +import hmac import html +import importlib import json import os import pickle import random import re +import signal import socket import ssl import string @@ -28,22 +33,30 @@ import urllib import urllib.parse import uuid +import weakref +from functools import partial, wraps from hashlib import md5 from pprint import pformat from pprint import pprint from urllib import request from urllib.parse import urljoin -import execjs # pip install PyExecJS import redis import requests import six from requests.cookies import RequestsCookieJar -from w3lib.url import canonicalize_url as sort_url +from w3lib.url import canonicalize_url as _canonicalize_url import feapder.setting as setting +from feapder.db.redisdb import RedisDB +from feapder.utils.email_sender import EmailSender from feapder.utils.log import log +try: + import execjs # pip install PyExecJS +except Exception as e: + pass + os.environ["EXECJS_RUNTIME"] = "Node" # 设置使用node执行js # 全局取消ssl证书验证 @@ -58,18 +71,39 @@ def get_redisdb(): global redisdb if not redisdb: - ip, port = setting.REDISDB_IP_PORTS.split(":") - redisdb = redis.Redis( - host=ip, - port=port, - db=setting.REDISDB_DB, - password=setting.REDISDB_USER_PASS, - decode_responses=True, - ) # redis默认端口是6379 + redisdb = RedisDB() return redisdb # 装饰器 +class Singleton(object): + def __init__(self, cls): + self._cls = cls + self._instance = {} + + def __call__(self, *args, **kwargs): + if self._cls not in self._instance: + self._instance[self._cls] = self._cls(*args, **kwargs) + return self._instance[self._cls] + + +class LazyProperty: + """ + 属性延时初始化，且只初始化一次 + """ + + def __init__(self, func): + self.func = func + + def __get__(self, instance, owner): + if instance is None: + return self + else: + value = self.func(instance) + setattr(instance, self.func.__name__, value) + return value + + def log_function_time(func): try: @@ -110,6 +144,115 @@ def run_func(*args, **kw): return inner_run_safe_model +def memoizemethod_noargs(method): + """Decorator to cache the result of a method (without arguments) using a + weak reference to its object + """ + cache = weakref.WeakKeyDictionary() + + @functools.wraps(method) + def new_method(self, *args, **kwargs): + if self not in cache: + cache[self] = method(self, *args, **kwargs) + return cache[self] + + return new_method + + +def retry(retry_times=3, interval=0): + """ + 普通函数的重试装饰器 + Args: + retry_times: 重试次数 + interval: 每次重试之间的间隔 + + Returns: + + """ + + def _retry(func): + @functools.wraps(func) # 将函数的原来属性付给新函数 + def wapper(*args, **kwargs): + for i in range(retry_times): + try: + return func(*args, **kwargs) + except Exception as e: + log.error( + "函数 {} 执行失败 重试 {} 次. error {}".format(func.__name__, i + 1, e) + ) + time.sleep(interval) + if i + 1 >= retry_times: + raise e + + return wapper + + return _retry + + +def retry_asyncio(retry_times=3, interval=0): + """ + 协程的重试装饰器 + Args: + retry_times: 重试次数 + interval: 每次重试之间的间隔 + + Returns: + + """ + + def _retry(func): + @functools.wraps(func) # 将函数的原来属性付给新函数 + async def wapper(*args, **kwargs): + for i in range(retry_times): + try: + return await func(*args, **kwargs) + except Exception as e: + log.error( + "函数 {} 执行失败 重试 {} 次. error {}".format(func.__name__, i + 1, e) + ) + await asyncio.sleep(interval) + if i + 1 >= retry_times: + raise e + + return wapper + + return _retry + + +def func_timeout(timeout): + """ + 函数运行时间限制装饰器 + 注: 不支持window + Args: + timeout: 超时的时间 + + Eg: + @set_timeout(3) + def test(): + ... + + Returns: + + """ + + def wapper(func): + def handle( + signum, frame + ): # 收到信号 SIGALRM 后的回调函数，第一个参数是信号的数字，第二个参数是the interrupted stack frame. + raise TimeoutError + + def new_method(*args, **kwargs): + signal.signal(signal.SIGALRM, handle) # 设置信号和回调函数 + signal.alarm(timeout) # 设置 timeout 秒的闹钟 + r = func(*args, **kwargs) + signal.alarm(0) # 关闭闹钟 + return r + + return new_method + + return wapper + + ########################【网页解析相关】############################### @@ -192,6 +335,30 @@ def get_cookies(response): return cookies +def get_cookies_from_str(cookie_str): + """ + >>> get_cookies_from_str("key=value; key2=value2; key3=; key4=; ") + {'key': 'value', 'key2': 'value2', 'key3': '', 'key4': ''} + + Args: + cookie_str: key=value; key2=value2; key3=; key4= + + Returns: + + """ + cookies = {} + for cookie in cookie_str.split(";"): + cookie = cookie.strip() + if not cookie: + continue + key, value = cookie.split("=", 1) + key = key.strip() + value = value.strip() + cookies[key] = value + + return cookies + + def get_cookies_jar(cookies): """ @summary: 适用于selenium生成的cookies转requests的cookies @@ -321,7 +488,7 @@ def canonicalize_url(url): """ url 归一化 会参数排序 及去掉锚点 """ - return sort_url(url) + return _canonicalize_url(url) def get_url_md5(url): @@ -341,12 +508,63 @@ def fit_url(urls, identis): def get_param(url, key): - params = url.split("?")[-1].split("&") + pattern = r"(?:[?&])" + re.escape(key) + r"=([^&]+)" + match = re.search(pattern, url) + if match: + return match.group(1) + return None + + +def get_all_params(url): + """ + >>> get_all_params("https://www.baidu.com/s?wd=feapder") + {'wd': 'feapder'} + """ + params_json = {} + params = url.split("?", 1)[-1].split("&") for param in params: key_value = param.split("=", 1) - if key == key_value[0]: - return key_value[1] - return None + if len(key_value) == 2: + params_json[key_value[0]] = unquote_url(key_value[1]) + else: + params_json[key_value[0]] = "" + + return params_json + + +def parse_url_params(url): + """ + 解析url参数 + :param url: + :return: + + >>> parse_url_params("https://www.baidu.com/s?wd=%E4%BD%A0%E5%A5%BD") + ('https://www.baidu.com/s', {'wd': '你好'}) + >>> parse_url_params("wd=%E4%BD%A0%E5%A5%BD") + ('', {'wd': '你好'}) + >>> parse_url_params("https://www.baidu.com/s?wd=%E4%BD%A0%E5%A5%BD&pn=10") + ('https://www.baidu.com/s', {'wd': '你好', 'pn': '10'}) + >>> parse_url_params("wd=%E4%BD%A0%E5%A5%BD&pn=10") + ('', {'wd': '你好', 'pn': '10'}) + >>> parse_url_params("https://www.baidu.com") + ('https://www.baidu.com', {}) + >>> parse_url_params("https://www.spidertools.cn/#/") + ('https://www.spidertools.cn/#/', {}) + """ + root_url = "" + params = {} + if "?" not in url: + if re.search("[&=]", url) and not re.search("/", url): + # 只有参数 + params = get_all_params(url) + else: + root_url = url + + else: + root_url = url.split("?", 1)[0] + params = get_all_params(url) + + return root_url, params def urlencode(params): @@ -375,7 +593,7 @@ def urldecode(url): params_json = {} params = url.split("?")[-1].split("&") for param in params: - key, value = param.split("=") + key, value = param.split("=", 1) params_json[key] = unquote_url(value) return params_json @@ -545,20 +763,8 @@ def get_form_data(form): return data -# mac上不好使 -# def get_domain(url): -# domain = '' -# try: -# domain = get_tld(url) -# except Exception as e: -# log.debug(e) -# return domain - - def get_domain(url): - proto, rest = urllib.parse.splittype(url) - domain, rest = urllib.parse.splithost(rest) - return domain + return urllib.parse.urlparse(url).netloc def get_index_url(url): @@ -581,6 +787,8 @@ def get_localhost_ip(): s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) s.connect(("8.8.8.8", 80)) ip = s.getsockname()[0] + except: + ip = "" finally: if s: s.close() @@ -657,36 +865,46 @@ def get_text(soup, *args): return "" -def del_html_tag(content, except_line_break=False, save_img=False, white_replaced=""): +def del_html_tag(content, save_line_break=True, save_p=False, save_img=False): """ 删除html标签 @param content: html内容 - @param except_line_break: 保留p标签 - @param save_img: 保留图片 - @param white_replaced: 空白符替换 + @param save_p: 保留p标签 + @param save_img: 保留图片标签 + @param save_line_break: 保留\n换行 @return: """ - content = replace_str(content, "(?i)") # (?)忽略大小写 - content = replace_str(content, "(?i)") - content = replace_str(content, "") - content = replace_str( - content, "(?!&[a-z]+=)&[a-z]+;?" - ) # 干掉 等无用的字符 但&xxx= 这种表示参数的除外 - if except_line_break: - content = content.replace("

", "/p") - content = replace_str(content, "<[^p].*?>") - content = content.replace("/p", "

") - content = replace_str(content, "[ \f\r\t\v]") - + if not content: + return content + # js + content = re.sub("(?i)", "", content) # (?)忽略大小写 + # css + content = re.sub("(?i)", "", content) # (?)忽略大小写 + # 注释 + content = re.sub("", "", content) + # 干掉 等无用的字符 但&xxx= 这种表示参数的除外 + content = re.sub("(?!&[a-z]+=)&[a-z]+;?", "", content) + + if save_p and save_img: + content = re.sub("<(?!(p[ >]|/p>|img ))(.|\n)+?>", "", content) + elif save_p: + content = re.sub("<(?!(p[ >]|/p>))(.|\n)+?>", "", content) elif save_img: - content = replace_str(content, "(?!)<.+?>") # 替换掉除图片外的其他标签 - content = replace_str(content, "(?! +)\s+", "\n") # 保留空格 - content = content.strip() + content = re.sub("<(?!img )(.|\n)+?>", "", content) + elif save_line_break: + content = re.sub("<(?!/p>)(.|\n)+?>", "", content) + content = re.sub("

", "\n", content) + else: + content = re.sub("<(.|\n)*?>", "", content) + if save_line_break: + # 把非换行符的空白符替换为一个空格 + content = re.sub("[^\S\n]+", " ", content) + # 把多个换行符替换为一个换行符 如\n\n\n 或 \n \n \n 替换为\n + content = re.sub("(\n ?)+", "\n", content) else: - content = replace_str(content, "<(.|\n)*?>") - content = replace_str(content, "\s", white_replaced) - content = content.strip() + content = re.sub("\s+", " ", content) + content = content.strip() return content @@ -770,27 +988,31 @@ def jsonp2json(jsonp): raise ValueError("Invalid Input") -def dumps_json(json_, indent=4, sort_keys=False): +def dumps_json(data, indent=4, sort_keys=False): """ @summary: 格式化json 用于打印 --------- - @param json_: json格式的字符串或json对象 + @param data: json格式的字符串或json对象 --------- @result: 格式化后的字符串 """ try: - if isinstance(json_, str): - json_ = get_json(json_) - - json_ = json.dumps( - json_, ensure_ascii=False, indent=indent, skipkeys=True, sort_keys=sort_keys + if isinstance(data, str): + data = get_json(data) + + data = json.dumps( + data, + ensure_ascii=False, + indent=indent, + skipkeys=True, + sort_keys=sort_keys, + default=str, ) except Exception as e: - log.error(e) - json_ = pformat(json_) + data = pformat(data) - return json_ + return data def get_json_value(json_object, key): @@ -888,11 +1110,32 @@ def get_conf_value(config_file, section, key): def mkdir(path): try: - os.makedirs(path) + if not os.path.exists(path): + os.makedirs(path) except OSError as exc: # Python >2.5 pass +def get_cache_path(filename, root_dir=None, local=False): + """ + Args: + filename: + root_dir: + local: 是否存储到当前目录 + + Returns: + + """ + if root_dir is None: + if local: + root_dir = os.path.join(sys.path[0], ".cache") + else: + root_dir = os.path.join(os.path.expanduser("~"), ".feapder/cache") + file_path = f"{root_dir}{os.sep}{filename}" + os.makedirs(os.path.dirname(file_path), exist_ok=True) + return f"{root_dir}{os.sep}{filename}" + + def write_file(filename, content, mode="w", encoding="utf-8"): """ @summary: 写文件 @@ -933,10 +1176,10 @@ def read_file(filename, readlines=False, encoding="utf-8"): def get_oss_file_list(oss_handler, prefix, date_range_min, date_range_max=None): """ 获取文件列表 - @param prefix: 路径前缀 如 data/car_service_line/yiche/yiche_serial_zongshu_info + @param prefix: 路径前缀 如 xxx/xxx @param date_range_min: 时间范围 最小值 日期分隔符为/ 如 2019/03/01 或 2019/03/01/00/00/00 @param date_range_max: 时间范围 最大值 日期分隔符为/ 如 2019/03/01 或 2019/03/01/00/00/00 - @return: 每个文件路径 如 html/e_commerce_service_line/alibaba/alibaba_shop_info/2019/03/22/15/53/15/8ca8b9e4-4c77-11e9-9dee-acde48001122.json.snappy + @return: 每个文件路径 如 html/xxx/xxx/2019/03/22/15/53/15/8ca8b9e4-4c77-11e9-9dee-acde48001122.json.snappy """ # 计算时间范围 @@ -992,8 +1235,19 @@ def is_exist(file_path): return os.path.exists(file_path) -def download_file(url, base_path, filename="", call_func="", proxies=None, data=None): - file_path = base_path + filename +def download_file(url, file_path, *, call_func=None, proxies=None, data=None): + """ + 下载文件，会自动创建文件存储目录 + Args: + url: 地址 + file_path: 文件存储地址 + call_func: 下载成功的回调 + proxies: 代理 + data: 请求体 + + Returns: + + """ directory = os.path.dirname(file_path) mkdir(directory) @@ -1013,13 +1267,6 @@ def progress_callfunc(blocknum, blocksize, totalsize): if url: try: - log.debug( - """ - 正在下载 %s - 存储路径 %s - """ - % (url, file_path) - ) if proxies: # create the object, assign it to a variable proxy = request.ProxyHandler(proxies) @@ -1030,15 +1277,8 @@ def progress_callfunc(blocknum, blocksize, totalsize): request.urlretrieve(url, file_path, progress_callfunc, data) - log.debug( - """ - 下载完毕 %s - 文件路径 %s - """ - % (url, file_path) - ) - - call_func and call_func() + if callable(call_func): + call_func() return 1 except Exception as e: log.error(e) @@ -1149,8 +1389,6 @@ def compile_js(js_func): return ctx.call -############################################### - ############################################# @@ -1439,7 +1677,7 @@ def format_date(date, old_format="", new_format="%Y-%m-%d %H:%M:%S"): %S 秒（00-59） @param new_format: 输出的日期格式 --------- - @result: 格式化后的日期，类型为字符串 如2017-4-17 3:27:12 + @result: 格式化后的日期，类型为字符串 如2017-4-17 03:27:12 """ if not date: return "" @@ -1472,45 +1710,101 @@ def format_date(date, old_format="", new_format="%Y-%m-%d %H:%M:%S"): return date_str +def transform_lower_num(data_str: str): + num_map = { + "一": "1", + "二": "2", + "两": "2", + "三": "3", + "四": "4", + "五": "5", + "六": "6", + "七": "7", + "八": "8", + "九": "9", + "十": "0", + } + pattern = f'[{"|".join(num_map.keys())}|零]' + res = re.search(pattern, data_str) + if not res: + # 如果字符串中没有包含中文数字 不做处理 直接返回 + return data_str + + data_str = data_str.replace("0", "零") + for n in num_map: + data_str = data_str.replace(n, num_map[n]) + + re_data_str = re.findall("\d+", data_str) + for i in re_data_str: + if len(i) == 3: + new_i = i.replace("0", "") + data_str = data_str.replace(i, new_i, 1) + elif len(i) == 4: + new_i = i.replace("10", "") + data_str = data_str.replace(i, new_i, 1) + elif len(i) == 2 and int(i) < 10: + new_i = int(i) + 10 + data_str = data_str.replace(i, str(new_i), 1) + elif len(i) == 1 and int(i) == 0: + new_i = int(i) + 10 + data_str = data_str.replace(i, str(new_i), 1) + + return data_str.replace("零", "0") + + @run_safe_model("format_time") def format_time(release_time, date_format="%Y-%m-%d %H:%M:%S"): + """ + >>> format_time("2个月前") + '2021-08-15 16:24:21' + >>> format_time("2月前") + '2021-08-15 16:24:36' + """ + release_time = transform_lower_num(release_time) + release_time = release_time.replace("日", "天").replace("/", "-") + if "年前" in release_time: - years = re.compile("(\d+)年前").findall(release_time) + years = re.compile("(\d+)\s*年前").findall(release_time) years_ago = datetime.datetime.now() - datetime.timedelta( days=int(years[0]) * 365 ) release_time = years_ago.strftime("%Y-%m-%d %H:%M:%S") elif "月前" in release_time: - months = re.compile("(\d+)月前").findall(release_time) + months = re.compile("(\d+)[\s个]*月前").findall(release_time) months_ago = datetime.datetime.now() - datetime.timedelta( days=int(months[0]) * 30 ) release_time = months_ago.strftime("%Y-%m-%d %H:%M:%S") elif "周前" in release_time: - weeks = re.compile("(\d+)周前").findall(release_time) + weeks = re.compile("(\d+)\s*周前").findall(release_time) weeks_ago = datetime.datetime.now() - datetime.timedelta(days=int(weeks[0]) * 7) release_time = weeks_ago.strftime("%Y-%m-%d %H:%M:%S") elif "天前" in release_time: - ndays = re.compile("(\d+)天前").findall(release_time) + ndays = re.compile("(\d+)\s*天前").findall(release_time) days_ago = datetime.datetime.now() - datetime.timedelta(days=int(ndays[0])) release_time = days_ago.strftime("%Y-%m-%d %H:%M:%S") elif "小时前" in release_time: - nhours = re.compile("(\d+)小时前").findall(release_time) + nhours = re.compile("(\d+)\s*小时前").findall(release_time) hours_ago = datetime.datetime.now() - datetime.timedelta(hours=int(nhours[0])) release_time = hours_ago.strftime("%Y-%m-%d %H:%M:%S") elif "分钟前" in release_time: - nminutes = re.compile("(\d+)分钟前").findall(release_time) + nminutes = re.compile("(\d+)\s*分钟前").findall(release_time) minutes_ago = datetime.datetime.now() - datetime.timedelta( minutes=int(nminutes[0]) ) release_time = minutes_ago.strftime("%Y-%m-%d %H:%M:%S") - elif "昨天" in release_time or "昨日" in release_time: + elif "前天" in release_time: + today = datetime.date.today() + yesterday = today - datetime.timedelta(days=2) + release_time = release_time.replace("前天", str(yesterday)) + + elif "昨天" in release_time: today = datetime.date.today() yesterday = today - datetime.timedelta(days=1) release_time = release_time.replace("昨天", str(yesterday)) @@ -1531,6 +1825,9 @@ def format_time(release_time, date_format="%Y-%m-%d %H:%M:%S"): else: release_time = str(int(get_current_date("%Y")) - 1) + "-" + release_time + # 把日和小时粘在一起的拆开 + template = re.compile("(\d{4}-\d{1,2}-\d{2})(\d{1,2})") + release_time = re.sub(template, r"\1 \2", release_time) release_time = format_date(release_time, new_format=date_format) return release_time @@ -1561,7 +1858,7 @@ def get_before_date( return datetime.datetime.strftime(date_obj, return_date_format) -def delay_time(sleep_time=160): +def delay_time(sleep_time=60): """ @summary: 睡眠 默认1分钟 --------- @@ -1633,28 +1930,10 @@ def get_sha1(*args): return sha1.hexdigest() # 40位 -def get_base64(secret, message): - """ - @summary: 数字证书签名算法是："HMAC-SHA256" - 参考：https://www.jokecamp.com/blog/examples-of-creating-base64-hashes-using-hmac-sha256-in-different-languages/ - --------- - @param secret: 秘钥 - @param message: 消息 - --------- - @result: 签名输出类型是："base64" - """ - - import hashlib - import hmac - import base64 - - message = bytes(message, "utf-8") - secret = bytes(secret, "utf-8") - - signature = base64.b64encode( - hmac.new(secret, message, digestmod=hashlib.sha256).digest() - ).decode("utf8") - return signature +def get_base64(data): + if data is None: + return data + return base64.b64encode(str(data).encode()).decode("utf8") def get_uuid(key1="", key2=""): @@ -1788,7 +2067,7 @@ def get_method(obj, name): return None -def witch_workspace(project_path): +def switch_workspace(project_path): """ @summary: --------- @@ -1856,14 +2135,14 @@ def make_insert_sql( ["{key}=values({key})".format(key=key) for key in update_columns] ) sql = ( - "insert%s into {table} {keys} values {values} on duplicate key update %s" + "insert%s into `{table}` {keys} values {values} on duplicate key update %s" % (" ignore" if insert_ignore else "", update_columns_) ) elif auto_update: - sql = "replace into {table} {keys} values {values}" + sql = "replace into `{table}` {keys} values {values}" else: - sql = "insert%s into {table} {keys} values {values}" % ( + sql = "insert%s into `{table}` {keys} values {values}" % ( " ignore" if insert_ignore else "" ) @@ -1886,7 +2165,7 @@ def make_update_sql(table, data, condition): for key, value in data.items(): value = format_sql_value(value) if isinstance(value, str): - key_values.append("`{}`='{}'".format(key, value)) + key_values.append("`{}`={}".format(key, repr(value))) elif value is None: key_values.append("`{}`={}".format(key, "null")) else: @@ -1894,7 +2173,7 @@ def make_update_sql(table, data, condition): key_values = ", ".join(key_values) - sql = "update {table} set {key_values} where {condition}" + sql = "update `{table}` set {key_values} where {condition}" sql = sql.format(table=table, key_values=key_values, condition=condition) return sql @@ -1916,7 +2195,7 @@ def make_batch_sql( if not datas: return - keys = list(datas[0].keys()) + keys = list(set([key for data in datas for key in data])) values_placeholder = ["%s"] * len(keys) values = [] @@ -1949,18 +2228,18 @@ def make_batch_sql( update_columns_ = ", ".join( ["`{key}`=values(`{key}`)".format(key=key) for key in update_columns] ) - sql = "insert into {table} {keys} values {values_placeholder} on duplicate key update {update_columns}".format( + sql = "insert into `{table}` {keys} values {values_placeholder} on duplicate key update {update_columns}".format( table=table, keys=keys, values_placeholder=values_placeholder, update_columns=update_columns_, ) elif auto_update: - sql = "replace into {table} {keys} values {values_placeholder}".format( + sql = "replace into `{table}` {keys} values {values_placeholder}".format( table=table, keys=keys, values_placeholder=values_placeholder ) else: - sql = "insert ignore into {table} {keys} values {values_placeholder}".format( + sql = "insert ignore into `{table}` {keys} values {values_placeholder}".format( table=table, keys=keys, values_placeholder=values_placeholder ) @@ -1970,17 +2249,33 @@ def make_batch_sql( ############### json相关 ####################### -def key2underline(key): - regex = "[A-Z]*" +def key2underline(key: str, strict=True): + """ + >>> key2underline("HelloWord") + 'hello_word' + >>> key2underline("SHData", strict=True) + 's_h_data' + >>> key2underline("SHData", strict=False) + 'sh_data' + >>> key2underline("SHDataHi", strict=False) + 'sh_data_hi' + >>> key2underline("SHDataHi", strict=True) + 's_h_data_hi' + >>> key2underline("dataHi", strict=True) + 'data_hi' + """ + regex = "[A-Z]*" if not strict else "[A-Z]" capitals = re.findall(regex, key) if capitals: - for pos, capital in enumerate(capitals): + for capital in capitals: if not capital: continue - if pos == 0: + if key.startswith(capital): if len(capital) > 1: - key = key.replace(capital, capital.lower() + "_", 1) + key = key.replace( + capital, capital[:-1].lower() + "_" + capital[-1].lower(), 1 + ) else: key = key.replace(capital, capital.lower(), 1) else: @@ -2136,59 +2431,173 @@ def re_def_supper_class(obj, supper_class): ################### +freq_limit_record = {} -def is_in_rate_limit(rate_limit, *key): +def reach_freq_limit(rate_limit, *key): """ 频率限制 :param rate_limit: 限制时间 单位秒 - :param key: 限制频率的key + :param key: 频率限制的key :return: True / False """ + if rate_limit == 0: + return False + msg_md5 = get_md5(*key) key = "rate_limit:{}".format(msg_md5) - if get_redisdb().get(key): - return True + try: + if get_redisdb().strget(key): + return True + + get_redisdb().strset(key, time.time(), ex=rate_limit) + except redis.exceptions.ConnectionError as e: + # 使用内存做频率限制 + global freq_limit_record + + if key not in freq_limit_record: + freq_limit_record[key] = time.time() + return False + + if time.time() - freq_limit_record.get(key) < rate_limit: + return True + else: + freq_limit_record[key] = time.time() - get_redisdb().set(key, time.time(), ex=rate_limit) return False def dingding_warning( message, - rate_limit=3600, + *, message_prefix=None, - url=setting.DINGDING_WARNING_URL, - user_phone=setting.DINGDING_WARNING_PHONE, + rate_limit=None, + url=None, + user_phone=None, + user_id=None, + secret=None, ): - if not url: - log.info("未设置叮叮的地址，不支持报警") + """ + 钉钉报警，user_phone与user_id 二选一即可 + Args: + message: + message_prefix: 消息摘要，用于去重 + rate_limit: 包名频率，单位秒，相同的报警内容在rate_limit时间内只会报警一次 + url: 钉钉报警url + user_phone: 被@的群成员手机号，支持列表，可指定多个。 + user_id: 被@的群成员userId，支持列表，可指定多个 + secret: 钉钉报警加签密钥 + Returns: + + """ + # 为了加载最新的配置 + rate_limit = rate_limit if rate_limit is not None else setting.WARNING_INTERVAL + url = url or setting.DINGDING_WARNING_URL + user_phone = user_phone or setting.DINGDING_WARNING_PHONE + user_id = user_id or setting.DINGDING_WARNING_USER_ID + secret = secret or setting.DINGDING_WARNING_SECRET + if secret: + timestamp = str(round(time.time() * 1000)) + secret_enc = secret.encode("utf-8") + string_to_sign_enc = f"{timestamp}\n{secret}".encode("utf-8") + hmac_code = hmac.new( + secret_enc, string_to_sign_enc, digestmod=hashlib.sha256 + ).digest() + sign = urllib.parse.quote_plus(base64.b64encode(hmac_code)) + url = f"{url}×tamp={timestamp}&sign={sign}" + + if not all([url, message]): return - if is_in_rate_limit(rate_limit, url, user_phone, message_prefix or message): + if reach_freq_limit(rate_limit, url, user_phone, message_prefix or message): log.info("报警时间间隔过短，此次报警忽略。 内容 {}".format(message)) return + if isinstance(user_phone, str): + user_phone = [user_phone] if user_phone else [] + + if isinstance(user_id, str): + user_id = [user_id] if user_id else [] + data = { "msgtype": "text", "text": {"content": message}, - "at": {"atMobiles": [user_phone], "isAtAll": False}, + "at": { + "atMobiles": user_phone, + "atUserIds": user_id, + "isAtAll": setting.DINGDING_WARNING_ALL, + }, } headers = {"Content-Type": "application/json"} - response = requests.post(url, headers=headers, data=json.dumps(data).encode("utf8")) - return response + try: + response = requests.post( + url, headers=headers, data=json.dumps(data).encode("utf8") + ) + result = response.json() + response.close() + if result.get("errcode") == 0: + return True + else: + raise Exception(result.get("errmsg")) + except Exception as e: + log.error("报警发送失败。 报警内容 {}, error: {}".format(message, e)) + return False -def linkedsee_warning( - message, rate_limit=3600, message_prefix=None, token=setting.LINGXI_TOKEN +def email_warning( + message, + title, + message_prefix=None, + email_sender=None, + email_password=None, + email_receiver=None, + email_smtpserver=None, + rate_limit=None, ): + # 为了加载最新的配置 + email_sender = email_sender or setting.EMAIL_SENDER + email_password = email_password or setting.EMAIL_PASSWORD + email_receiver = email_receiver or setting.EMAIL_RECEIVER + email_smtpserver = email_smtpserver or setting.EMAIL_SMTPSERVER + rate_limit = rate_limit if rate_limit is not None else setting.WARNING_INTERVAL + + if not all([message, email_sender, email_password, email_receiver]): + return + + if reach_freq_limit( + rate_limit, email_receiver, email_sender, message_prefix or message + ): + log.info("报警时间间隔过短，此次报警忽略。 内容 {}".format(message)) + return + + if isinstance(email_receiver, str): + email_receiver = [email_receiver] + + with EmailSender( + username=email_sender, password=email_password, smtpserver=email_smtpserver + ) as email: + return email.send(receivers=email_receiver, title=title, content=message) + + +def linkedsee_warning(message, rate_limit=3600, message_prefix=None, token=None): + """ + 灵犀电话报警 + Args: + message: + rate_limit: + message_prefix: + token: + + Returns: + + """ if not token: log.info("未设置灵犀token，不支持报警") return - if is_in_rate_limit(rate_limit, token, message_prefix or message): + if reach_freq_limit(rate_limit, token, message_prefix or message): log.info("报警时间间隔过短，此次报警忽略。 内容 {}".format(message)) return @@ -2199,3 +2608,268 @@ def linkedsee_warning( data = {"content": message} response = requests.post(url, data=json.dumps(data), headers=headers) return response + + +def wechat_warning( + message, + message_prefix=None, + rate_limit=None, + url=None, + user_phone=None, + all_users: bool = None, +): + """企业微信报警""" + + # 为了加载最新的配置 + rate_limit = rate_limit if rate_limit is not None else setting.WARNING_INTERVAL + url = url or setting.WECHAT_WARNING_URL + user_phone = user_phone or setting.WECHAT_WARNING_PHONE + all_users = all_users if all_users is not None else setting.WECHAT_WARNING_ALL + + if isinstance(user_phone, str): + user_phone = [user_phone] if user_phone else [] + + if all_users is True or not user_phone: + user_phone = ["@all"] + + if not all([url, message]): + return + + if reach_freq_limit(rate_limit, url, user_phone, message_prefix or message): + log.info("报警时间间隔过短，此次报警忽略。 内容 {}".format(message)) + return + + data = { + "msgtype": "text", + "text": {"content": message, "mentioned_mobile_list": user_phone}, + } + + headers = {"Content-Type": "application/json"} + + try: + response = requests.post( + url, headers=headers, data=json.dumps(data).encode("utf8") + ) + result = response.json() + response.close() + if result.get("errcode") == 0: + return True + else: + raise Exception(result.get("errmsg")) + except Exception as e: + log.error("报警发送失败。 报警内容 {}, error: {}".format(message, e)) + return False + + +def feishu_warning(message, message_prefix=None, rate_limit=None, url=None, user=None): + """ + + Args: + message: + message_prefix: + rate_limit: + url: + user: {"open_id":"ou_xxxxx", "name":"xxxx"} 或 [{"open_id":"ou_xxxxx", "name":"xxxx"}] + + Returns: + + """ + # 为了加载最新的配置 + rate_limit = rate_limit if rate_limit is not None else setting.WARNING_INTERVAL + url = url or setting.FEISHU_WARNING_URL + user = user or setting.FEISHU_WARNING_USER + + if not all([url, message]): + return + + if reach_freq_limit(rate_limit, url, user, message_prefix or message): + log.info("报警时间间隔过短，此次报警忽略。 内容 {}".format(message)) + return + + if isinstance(user, dict): + user = [user] if user else [] + + at = "" + if setting.FEISHU_WARNING_ALL: + at = '所有人' + elif user: + at = " ".join( + [f'{u.get("name")}' for u in user] + ) + + data = {"msg_type": "text", "content": {"text": at + message}} + headers = {"Content-Type": "application/json"} + + try: + response = requests.post( + url, headers=headers, data=json.dumps(data).encode("utf8") + ) + result = response.json() + response.close() + if result.get("StatusCode") == 0: + return True + else: + raise Exception(result.get("msg")) + except Exception as e: + log.error("报警发送失败。 报警内容 {}, error: {}".format(message, e)) + return False + + +def qmsg_warning( + message, + message_prefix=None, + rate_limit=None, + url=None, + user_qq=None, + bot_qq=None +): + """qmsg报警""" + + # 为了加载最新的配置 + rate_limit = rate_limit if rate_limit is not None else setting.WARNING_INTERVAL + url = url or setting.QMSG_WARNING_URL + user_qq = user_qq or setting.QMSG_WARNING_QQ + bot_qq = bot_qq or setting.QMSG_WARNING_BOT + + if isinstance(user_qq, list): + user_qq = ','.join(map(str, user_qq)) + + if not all([url, message]): + return + + if reach_freq_limit(rate_limit, url, user_qq, message_prefix or message): + log.info("报警时间间隔过短，此次报警忽略。 内容 {}".format(message)) + return + + data = { + "msg": message, + "qq": user_qq, + "bot": bot_qq, + } + + headers = {"Content-Type": "application/json"} + + try: + response = requests.post( + url, headers=headers, data=json.dumps(data).encode("utf8") + ) + result = response.json() + response.close() + if result.get("code") == 0: + return True + else: + raise Exception(result.get("reason")) + except Exception as e: + log.error("报警发送失败。 报警内容 {}, error: {}".format(message, e)) + return False + + +def send_msg(msg, level="DEBUG", message_prefix="", keyword="feapder报警系统\n"): + if setting.WARNING_LEVEL == "ERROR": + if level.upper() != "ERROR": + return + + if setting.DINGDING_WARNING_URL: + dingding_warning(keyword + msg, message_prefix=message_prefix) + + if setting.EMAIL_RECEIVER: + title = message_prefix or msg + if len(title) > 50: + title = title[:50] + "..." + email_warning(msg, message_prefix=message_prefix, title=title) + + if setting.WECHAT_WARNING_URL: + wechat_warning(keyword + msg, message_prefix=message_prefix) + + if setting.FEISHU_WARNING_URL: + feishu_warning(keyword + msg, message_prefix=message_prefix) + + if setting.QMSG_WARNING_URL: + qmsg_warning(keyword + msg, message_prefix=message_prefix) + + +################### + + +def make_item(cls, data: dict): + """提供Item类与原数据，快速构建Item实例 + :param cls: Item类 + :param data: 字典格式的数据 + """ + item = cls() + for key, val in data.items(): + setattr(item, key, val) + return item + + +################### + + +def aio_wrap(loop=None, executor=None): + """ + wrap a normal sync version of a function to an async version + """ + outer_loop = loop + outer_executor = executor + + def wrap(fn): + @wraps(fn) + async def run(*args, loop=None, executor=None, **kwargs): + if loop is None: + if outer_loop is None: + loop = asyncio.get_event_loop() + else: + loop = outer_loop + if executor is None: + executor = outer_executor + pfunc = partial(fn, *args, **kwargs) + return await loop.run_in_executor(executor, pfunc) + + return run + + return wrap + + +######### number ########## + + +def ensure_int(n): + """ + >>> ensure_int(None) + 0 + >>> ensure_int(False) + 0 + >>> ensure_int(12) + 12 + >>> ensure_int("72") + 72 + >>> ensure_int('') + 0 + >>> ensure_int('1') + 1 + """ + if not n: + return 0 + return int(n) + + +def ensure_float(n): + """ + >>> ensure_float(None) + 0.0 + >>> ensure_float(False) + 0.0 + >>> ensure_float(12) + 12.0 + >>> ensure_float("72") + 72.0 + """ + if not n: + return 0.0 + return float(n) + + +def import_cls(cls_info): + module, class_name = cls_info.rsplit(".", 1) + cls = importlib.import_module(module).__getattribute__(class_name) + return cls diff --git a/feapder/utils/webdriver/__init__.py b/feapder/utils/webdriver/__init__.py new file mode 100644 index 00000000..16f8bd93 --- /dev/null +++ b/feapder/utils/webdriver/__init__.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- +""" +Created on 2022/9/7 4:39 PM +--------- +@summary: +--------- +@author: Boris +@email: boris_liu@foxmail.com +""" +from .playwright_driver import PlaywrightDriver +from .selenium_driver import SeleniumDriver +from .webdirver import InterceptRequest, InterceptResponse +from .webdriver_pool import WebDriverPool + +# 为了兼容老代码 +WebDriver = SeleniumDriver diff --git a/feapder/utils/webdriver/playwright_driver.py b/feapder/utils/webdriver/playwright_driver.py new file mode 100644 index 00000000..fe7e5062 --- /dev/null +++ b/feapder/utils/webdriver/playwright_driver.py @@ -0,0 +1,298 @@ +# -*- coding: utf-8 -*- +""" +Created on 2022/9/7 4:11 PM +--------- +@summary: +--------- +@author: Boris +@email: boris_liu@foxmail.com +""" + +import json +import os +import re +from collections import defaultdict +from typing import Union, List + +try: + from typing import Literal # python >= 3.8 +except ImportError: # python <3.8 + from typing_extensions import Literal + + +from playwright.sync_api import Page, BrowserContext, ViewportSize, ProxySettings +from playwright.sync_api import Playwright, Browser +from playwright.sync_api import Response +from playwright.sync_api import sync_playwright + +from feapder.utils import tools +from feapder.utils.log import log +from feapder.utils.webdriver.webdirver import * + + +class PlaywrightDriver(WebDriver): + def __init__( + self, + *, + page_on_event_callback: dict = None, + storage_state_path: str = None, + driver_type: Literal["chromium", "firefox", "webkit"] = "chromium", + url_regexes: list = None, + save_all: bool = False, + **kwargs + ): + """ + + Args: + page_on_event_callback: page.on() 事件的回调 如 page_on_event_callback={"dialog": lambda dialog: dialog.accept()} + storage_state_path: 保存浏览器状态的路径 + driver_type: 浏览器类型 chromium, firefox, webkit + url_regexes: 拦截接口，支持正则，数组类型 + save_all: 是否保存所有拦截的接口, 默认只保存最后一个 + **kwargs: + """ + super(PlaywrightDriver, self).__init__(**kwargs) + self.driver: Playwright = None + self.browser: Browser = None + self.context: BrowserContext = None + self.page: Page = None + self.url = None + self.storage_state_path = storage_state_path + + self._driver_type = driver_type or "chromium" + self._page_on_event_callback = page_on_event_callback + self._url_regexes = url_regexes + self._save_all = save_all + + if self._save_all and self._url_regexes: + log.warning( + "获取完拦截的数据后, 请主动调用PlaywrightDriver的clear_cache()方法清空拦截的数据，否则数据会一直累加，导致内存溢出" + ) + self._cache_data = defaultdict(list) + else: + self._cache_data = {} + + self._setup() + + def _setup(self): + # 处理参数 + if self._proxy: + proxy = self._proxy() if callable(self._proxy) else self._proxy + proxy = self.format_context_proxy(proxy) + else: + proxy = None + + user_agent = ( + self._user_agent() if callable(self._user_agent) else self._user_agent + ) + + view_size = ViewportSize( + width=self._window_size[0], height=self._window_size[1] + ) + + # 初始化浏览器对象 + self.driver = sync_playwright().start() + self.browser = getattr(self.driver, self._driver_type).launch( + headless=self._headless, + args=["--no-sandbox"], + proxy=proxy, + executable_path=self._executable_path, + downloads_path=self._download_path, + ) + + if self.storage_state_path and os.path.exists(self.storage_state_path): + self.context = self.browser.new_context( + user_agent=user_agent, + screen=view_size, + viewport=view_size, + proxy=proxy, + storage_state=self.storage_state_path, + ) + else: + self.context = self.browser.new_context( + user_agent=user_agent, + screen=view_size, + viewport=view_size, + proxy=proxy, + ) + + if self._use_stealth_js: + path = os.path.join(os.path.dirname(__file__), "../js/stealth.min.js") + self.context.add_init_script(path=path) + + self.page = self.context.new_page() + self.page.set_default_timeout(self._timeout * 1000) + + if self._page_on_event_callback: + for event, callback in self._page_on_event_callback.items(): + self.page.on(event, callback) + + if self._url_regexes: + self.page.on("response", self.on_response) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + if exc_val: + log.error(exc_val) + + self.quit() + return True + + def format_context_proxy(self, proxy) -> ProxySettings: + """ + Args: + proxy: username:password@ip:port / ip:port + Returns: + { + "server": "ip:port" + "username": username, + "password": password, + } + server: http://ip:port or socks5://ip:port. Short form ip:port is considered an HTTP proxy. + """ + + if "@" in proxy: + certification, _proxy = proxy.split("@") + username, password = certification.split(":") + + context_proxy = ProxySettings( + server=_proxy, + username=username, + password=password, + ) + else: + context_proxy = ProxySettings(server=proxy) + + return context_proxy + + def save_storage_stage(self): + if self.storage_state_path: + os.makedirs(os.path.dirname(self.storage_state_path), exist_ok=True) + self.context.storage_state(path=self.storage_state_path) + + def quit(self): + self.page.close() + self.context.close() + self.browser.close() + self.driver.stop() + + @property + def domain(self): + return tools.get_domain(self.url or self.page.url) + + @property + def cookies(self): + cookies_json = {} + for cookie in self.page.context.cookies(): + cookies_json[cookie["name"]] = cookie["value"] + + return cookies_json + + @cookies.setter + def cookies(self, val: Union[dict, List[dict]]): + """ + 设置cookie + Args: + val: List[{name: str, value: str, url: Union[str, NoneType], domain: Union[str, NoneType], path: Union[str, NoneType], expires: Union[float, NoneType], httpOnly: Union[bool, NoneType], secure: Union[bool, NoneType], sameSite: Union["Lax", "None", "Strict", NoneType]}] + + Returns: + + """ + if isinstance(val, list): + self.page.context.add_cookies(val) + else: + cookies = [] + for key, value in val.items(): + cookies.append( + {"name": key, "value": value, "url": self.url or self.page.url} + ) + self.page.context.add_cookies(cookies) + + @property + def user_agent(self): + return self.page.evaluate("() => navigator.userAgent") + + def on_response(self, response: Response): + for regex in self._url_regexes: + if re.search(regex, response.request.url): + intercept_request = InterceptRequest( + url=response.request.url, + headers=response.request.headers, + data=response.request.post_data, + ) + + intercept_response = InterceptResponse( + request=intercept_request, + url=response.url, + headers=response.headers, + content=response.body(), + status_code=response.status, + ) + if self._save_all: + self._cache_data[regex].append(intercept_response) + else: + self._cache_data[regex] = intercept_response + + def get_response(self, url_regex) -> InterceptResponse: + if self._save_all: + response_list = self._cache_data.get(url_regex) + if response_list: + return response_list[-1] + return self._cache_data.get(url_regex) + + def get_all_response(self, url_regex) -> List[InterceptResponse]: + """ + 获取所有匹配的响应, 仅在save_all=True时有效 + Args: + url_regex: + + Returns: + + """ + response_list = self._cache_data.get(url_regex, []) + if not isinstance(response_list, list): + return [response_list] + return response_list + + def get_text(self, url_regex): + return ( + self.get_response(url_regex).content.decode() + if self.get_response(url_regex) + else None + ) + + def get_all_text(self, url_regex): + """ + 获取所有匹配的响应文本, 仅在save_all=True时有效 + Args: + url_regex: + + Returns: + + """ + return [ + response.content.decode() for response in self.get_all_response(url_regex) + ] + + def get_json(self, url_regex): + return ( + json.loads(self.get_text(url_regex)) + if self.get_response(url_regex) + else None + ) + + def get_all_json(self, url_regex): + """ + 获取所有匹配的响应json, 仅在save_all=True时有效 + Args: + url_regex: + + Returns: + + """ + return [json.loads(text) for text in self.get_all_text(url_regex)] + + def clear_cache(self): + self._cache_data = defaultdict(list) diff --git a/feapder/utils/webdriver/selenium_driver.py b/feapder/utils/webdriver/selenium_driver.py new file mode 100644 index 00000000..06eb5493 --- /dev/null +++ b/feapder/utils/webdriver/selenium_driver.py @@ -0,0 +1,471 @@ +# -*- coding: utf-8 -*- +""" +Created on 2021/3/18 4:59 下午 +--------- +@summary: +--------- +@author: Boris +@email: boris_liu@foxmail.com +""" + +import inspect +import json +import logging +import os +from typing import Optional, Union, List + +from selenium import webdriver +from selenium.webdriver.remote.webdriver import WebDriver as RemoteWebDriver +from webdriver_manager.chrome import ChromeDriverManager +from webdriver_manager.firefox import GeckoDriverManager + +from feapder.utils import tools +from feapder.utils.log import log, OTHERS_LOG_LEVAL +from feapder.utils.webdriver.webdirver import * + +# 屏蔽webdriver_manager日志 +logging.getLogger("WDM").setLevel(OTHERS_LOG_LEVAL) + + +class SeleniumDriver(WebDriver, RemoteWebDriver): + CHROME = "CHROME" + EDGE = "EDGE" + PHANTOMJS = "PHANTOMJS" + FIREFOX = "FIREFOX" + + __DRIVER_ATTRS__ = {"keep_alive"} + + def __init__(self, xhr_url_regexes: list = None, **kwargs): + """ + + Args: + xhr_url_regexes: 拦截xhr接口，支持正则，数组类型 + **kwargs: + """ + super(SeleniumDriver, self).__init__(**kwargs) + self._xhr_url_regexes = xhr_url_regexes + self._driver_type = self._driver_type or SeleniumDriver.CHROME + + if self._xhr_url_regexes and self._driver_type != SeleniumDriver.CHROME: + raise Exception( + "xhr_url_regexes only support by chrome now! eg: driver_type=SeleniumDriver.CHROME" + ) + + if self._driver_type == SeleniumDriver.CHROME: + self.driver = self.chrome_driver() + + elif self._driver_type == SeleniumDriver.EDGE: + self.driver = self.edge_driver() + + elif self._driver_type == SeleniumDriver.PHANTOMJS: + self.driver = self.phantomjs_driver() + + elif self._driver_type == SeleniumDriver.FIREFOX: + self.driver = self.firefox_driver() + + else: + raise TypeError( + "dirver_type must be one of CHROME or PHANTOMJS or FIREFOX, but received {}".format( + type(self._driver_type) + ) + ) + + # driver.get(url)一直不返回，但也不报错的问题，这时程序会卡住，设置超时选项能解决这个问题。 + self.driver.set_page_load_timeout(self._timeout) + # 设置10秒脚本超时时间 + self.driver.set_script_timeout(self._timeout) + self.url = None + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + if exc_val: + log.error(exc_val) + + self.quit() + return True + + def filter_kwargs(self, kwargs: dict, driver_attrs: set): + if not kwargs: + return {} + + data = {} + for key, value in kwargs.items(): + if key in driver_attrs: + data[key] = value + + return data + + def get_options(self, default_options, *option_keys): + for option_key in option_keys: + options = self._kwargs.get(option_key) + if options is not None: + return options + + return default_options + + def get_driver_kwargs(self): + return self.filter_kwargs(self._kwargs, self.__DRIVER_ATTRS__) + + def apply_capabilities(self, options, *capability_keys): + for capability_key in capability_keys: + capabilities = self._kwargs.get(capability_key) + if not capabilities: + continue + + for key, value in capabilities.items(): + options.set_capability(key, value) + + return options + + def build_service(self, service_cls, driver_manager_cls=None): + service = self._kwargs.get("service") + if service is not None: + return service + + service_kwargs = {} + service_args = self._kwargs.get("service_args") + if service_args is not None: + service_kwargs["service_args"] = service_args + + port = self._kwargs.get("port") + if port is not None: + service_kwargs["port"] = port + + log_path = self._kwargs.get("service_log_path") + if log_path is None: + log_path = self._kwargs.get("log_path") + if log_path is not None: + log_param = ( + "log_output" + if "log_output" in inspect.signature(service_cls).parameters + else "log_path" + ) + service_kwargs[log_param] = log_path + + if self._executable_path: + return service_cls(self._executable_path, **service_kwargs) + + if self._auto_install_driver and driver_manager_cls is not None: + return service_cls(driver_manager_cls().install(), **service_kwargs) + + if service_kwargs: + return service_cls(**service_kwargs) + + return None + + def create_driver(self, driver_cls, options, service): + kwargs = self.get_driver_kwargs() + if service is not None: + kwargs["service"] = service + + return driver_cls(options=options, **kwargs) + + def get_proxy(self): + return self._proxy() if callable(self._proxy) else self._proxy + + def get_user_agent(self): + return self._user_agent() if callable(self._user_agent) else self._user_agent + + def get_driver(self): + return self.driver + + def firefox_driver(self): + from selenium.webdriver.firefox.service import Service + + firefox_options = self.get_options( + webdriver.FirefoxOptions(), "options", "firefox_options" + ) + firefox_profile = self._kwargs.get("firefox_profile") + if firefox_profile is not None: + firefox_options.profile = firefox_profile + firefox_binary = self._kwargs.get("firefox_binary") + if firefox_binary is not None: + firefox_options.binary_location = ( + getattr(firefox_binary, "path", None) + or getattr(firefox_binary, "_start_cmd", None) + or firefox_binary + ) + + self.apply_capabilities(firefox_options, "desired_capabilities", "capabilities") + if self._proxy: + proxy = self.get_proxy() + firefox_options.set_capability( + "proxy", + { + "proxyType": "MANUAL", + "httpProxy": proxy, + "ftpProxy": proxy, + "sslProxy": proxy, + }, + ) + + if self._user_agent: + firefox_options.set_preference( + "general.useragent.override", self.get_user_agent() + ) + + if not self._load_images: + firefox_options.set_preference("permissions.default.image", 2) + + if self._headless: + firefox_options.add_argument("--headless") + firefox_options.add_argument("--disable-gpu") + + # 添加自定义的配置参数 + if self._custom_argument: + for arg in self._custom_argument: + firefox_options.add_argument(arg) + + service = self.build_service(Service, GeckoDriverManager) + driver = self.create_driver(webdriver.Firefox, firefox_options, service) + + if self._window_size: + driver.set_window_size(*self._window_size) + + return driver + + def chrome_driver(self): + chrome_options = self.get_options( + webdriver.ChromeOptions(), "options", "chrome_options" + ) + # 此步骤很重要，设置为开发者模式，防止被各大网站识别出来使用了Selenium + chrome_options.add_experimental_option("excludeSwitches", ["enable-automation"]) + chrome_options.add_experimental_option("useAutomationExtension", False) + # docker 里运行需要 + chrome_options.add_argument("--no-sandbox") + from selenium.webdriver.chrome.service import Service + + self.apply_capabilities(chrome_options, "desired_capabilities") + + if self._proxy: + chrome_options.add_argument("--proxy-server={}".format(self.get_proxy())) + if self._user_agent: + chrome_options.add_argument("user-agent={}".format(self.get_user_agent())) + if not self._load_images: + chrome_options.add_experimental_option( + "prefs", {"profile.managed_default_content_settings.images": 2} + ) + + if self._headless: + chrome_options.add_argument("--headless") + chrome_options.add_argument("--disable-gpu") + + if self._window_size: + chrome_options.add_argument( + "--window-size={},{}".format(self._window_size[0], self._window_size[1]) + ) + + if self._download_path: + os.makedirs(self._download_path, exist_ok=True) + prefs = { + "download.prompt_for_download": False, + "download.default_directory": self._download_path, + } + chrome_options.add_experimental_option("prefs", prefs) + + # 添加自定义的配置参数 + if self._custom_argument: + for arg in self._custom_argument: + chrome_options.add_argument(arg) + + service = self.build_service(Service, ChromeDriverManager) + driver = self.create_driver(webdriver.Chrome, chrome_options, service) + + # 隐藏浏览器特征 + if self._use_stealth_js: + with open( + os.path.join(os.path.dirname(__file__), "../js/stealth.min.js") + ) as f: + js = f.read() + driver.execute_cdp_cmd( + "Page.addScriptToEvaluateOnNewDocument", {"source": js} + ) + + if self._xhr_url_regexes: + assert isinstance(self._xhr_url_regexes, list) + with open( + os.path.join(os.path.dirname(__file__), "../js/intercept.js") + ) as f: + js = f.read() + driver.execute_cdp_cmd( + "Page.addScriptToEvaluateOnNewDocument", {"source": js} + ) + js = f"window.__urlRegexes = {self._xhr_url_regexes}" + driver.execute_cdp_cmd( + "Page.addScriptToEvaluateOnNewDocument", {"source": js} + ) + + if self._download_path: + driver.execute_cdp_cmd( + "Page.setDownloadBehavior", + {"behavior": "allow", "downloadPath": self._download_path}, + ) + + return driver + + def edge_driver(self): + edge_options = self.get_options( + webdriver.EdgeOptions(), "options", "edge_options" + ) + # 此步骤很重要，设置为开发者模式，防止被各大网站识别出来使用了Selenium + edge_options.add_experimental_option("excludeSwitches", ["enable-automation"]) + edge_options.add_experimental_option("useAutomationExtension", False) + # docker 里运行需要 + edge_options.add_argument("--no-sandbox") + from selenium.webdriver.edge.service import Service + + self.apply_capabilities(edge_options, "desired_capabilities") + + if self._proxy: + edge_options.add_argument("--proxy-server={}".format(self.get_proxy())) + if self._user_agent: + edge_options.add_argument("user-agent={}".format(self.get_user_agent())) + if not self._load_images: + edge_options.add_experimental_option( + "prefs", {"profile.managed_default_content_settings.images": 2} + ) + + if self._headless: + edge_options.add_argument("--headless") + edge_options.add_argument("--disable-gpu") + + if self._window_size: + edge_options.add_argument( + "--window-size={},{}".format(self._window_size[0], self._window_size[1]) + ) + + if self._download_path: + os.makedirs(self._download_path, exist_ok=True) + prefs = { + "download.prompt_for_download": False, + "download.default_directory": self._download_path, + } + edge_options.add_experimental_option("prefs", prefs) + + # 添加自定义的配置参数 + if self._custom_argument: + for arg in self._custom_argument: + edge_options.add_argument(arg) + + service = self.build_service(Service) + driver = self.create_driver(webdriver.Edge, edge_options, service) + + # 隐藏浏览器特征 + if self._use_stealth_js: + with open( + os.path.join(os.path.dirname(__file__), "../js/stealth.min.js") + ) as f: + js = f.read() + driver.execute_cdp_cmd( + "Page.addScriptToEvaluateOnNewDocument", {"source": js} + ) + + if self._xhr_url_regexes: + assert isinstance(self._xhr_url_regexes, list) + with open( + os.path.join(os.path.dirname(__file__), "../js/intercept.js") + ) as f: + js = f.read() + driver.execute_cdp_cmd( + "Page.addScriptToEvaluateOnNewDocument", {"source": js} + ) + js = f"window.__urlRegexes = {self._xhr_url_regexes}" + driver.execute_cdp_cmd( + "Page.addScriptToEvaluateOnNewDocument", {"source": js} + ) + + if self._download_path: + driver.execute_cdp_cmd( + "Page.setDownloadBehavior", + {"behavior": "allow", "downloadPath": self._download_path}, + ) + + return driver + + def phantomjs_driver(self): + raise NotImplementedError( + "PhantomJS is not supported by Selenium 4. " + "Please use CHROME, EDGE, or FIREFOX." + ) + + @property + def domain(self): + return tools.get_domain(self.url or self.driver.current_url) + + @property + def cookies(self): + cookies_json = {} + for cookie in self.driver.get_cookies(): + cookies_json[cookie["name"]] = cookie["value"] + + return cookies_json + + @cookies.setter + def cookies(self, val: Union[dict, List[dict]]): + """ + 设置cookie + Args: + val: {"key":"value", "key2":"value2"} + + Returns: + + """ + if isinstance(val, list): + for cookie in val: + # "path", "domain", "secure", "expiry" + _cookie = { + "name": cookie.get("name"), + "value": cookie.get("value"), + "domain": cookie.get("domain"), + "path": cookie.get("path"), + "expires": cookie.get("expires"), + "secure": cookie.get("secure"), + } + self.driver.add_cookie(_cookie) + else: + for key, value in val.items(): + self.driver.add_cookie({"name": key, "value": value}) + + @property + def user_agent(self): + return self.driver.execute_script("return navigator.userAgent;") + + def xhr_response(self, xhr_url_regex) -> Optional[InterceptResponse]: + data = self.driver.execute_script( + f'return window.__ajaxData["{xhr_url_regex}"];' + ) + if not data: + return None + + request = InterceptRequest(**data["request"]) + response = InterceptResponse(request, **data["response"]) + return response + + def xhr_data(self, xhr_url_regex) -> Union[str, dict, None]: + response = self.xhr_response(xhr_url_regex) + if not response: + return None + return response.content + + def xhr_text(self, xhr_url_regex) -> Optional[str]: + response = self.xhr_response(xhr_url_regex) + if not response: + return None + if isinstance(response.content, dict): + return json.dumps(response.content, ensure_ascii=False) + return response.content + + def xhr_json(self, xhr_url_regex) -> Optional[dict]: + text = self.xhr_text(xhr_url_regex) + return json.loads(text) + + def __getattr__(self, name): + if self.driver: + return getattr(self.driver, name) + else: + raise AttributeError + + # def __del__(self): + # self.quit() diff --git a/feapder/utils/webdriver/webdirver.py b/feapder/utils/webdriver/webdirver.py new file mode 100644 index 00000000..8fa2a34e --- /dev/null +++ b/feapder/utils/webdriver/webdirver.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- +""" +Created on 2022/9/7 4:27 PM +--------- +@summary: +--------- +@author: Boris +@email: boris_liu@foxmail.com +""" +import abc + +from feapder import setting + + +class InterceptRequest: + def __init__(self, url, data, headers): + self.url = url + self.data = data + self.headers = headers + + +class InterceptResponse: + def __init__(self, request: InterceptRequest, url, headers, content, status_code): + self.request = request + self.url = url + self.headers = headers + self.content = content + self.status_code = status_code + + +class WebDriver: + def __init__( + self, + load_images=True, + user_agent=None, + proxy=None, + headless=False, + driver_type=None, + timeout=16, + window_size=(1024, 800), + executable_path=None, + custom_argument=None, + download_path=None, + auto_install_driver=True, + use_stealth_js=True, + **kwargs, + ): + """ + webdirver 封装，支持chrome、phantomjs 和 firefox + Args: + load_images: 是否加载图片 + user_agent: 字符串 或 无参函数，返回值为user_agent + proxy: xxx.xxx.xxx.xxx:xxxx 或 无参函数，返回值为代理地址 + headless: 是否启用无头模式 + driver_type: CHROME,EDGE 或 PHANTOMJS,FIREFOX + timeout: 请求超时时间 + window_size: # 窗口大小 + executable_path: 浏览器路径，默认为默认路径 + custom_argument: 自定义参数 用于webdriver.Chrome(options=chrome_options, **kwargs) + download_path: 文件下载保存路径；如果指定，不再出现“保留”“放弃”提示，仅对Chrome有效 + auto_install_driver: 自动下载浏览器驱动 支持chrome 和 firefox + use_stealth_js: 使用stealth.min.js隐藏浏览器特征 + **kwargs: + """ + self._load_images = load_images + self._user_agent = user_agent or setting.DEFAULT_USERAGENT + self._proxy = proxy + self._headless = headless + self._timeout = timeout + self._window_size = window_size + self._executable_path = executable_path + self._custom_argument = custom_argument + self._download_path = download_path + self._auto_install_driver = auto_install_driver + self._use_stealth_js = use_stealth_js + self._driver_type = driver_type + self._kwargs = kwargs + + @abc.abstractmethod + def quit(self): + pass diff --git a/feapder/utils/webdriver/webdriver_pool.py b/feapder/utils/webdriver/webdriver_pool.py new file mode 100644 index 00000000..c9ecc5a9 --- /dev/null +++ b/feapder/utils/webdriver/webdriver_pool.py @@ -0,0 +1,115 @@ +# -*- coding: utf-8 -*- +""" +Created on 2021/3/18 4:59 下午 +--------- +@summary: +--------- +@author: Boris +@email: boris_liu@foxmail.com +""" + +import queue +import threading + +from feapder.utils.log import log +from feapder.utils.tools import Singleton +from feapder.utils.webdriver.selenium_driver import SeleniumDriver + + +@Singleton +class WebDriverPool: + def __init__( + self, pool_size=5, driver_cls=SeleniumDriver, thread_safe=False, **kwargs + ): + """ + + Args: + pool_size: driver池的大小 + driver: 驱动类型 + thread_safe: 是否线程安全 + 是则每个线程拥有一个driver，pool_size无效，driver数量为线程数 + 否则每个线程从池中获取driver + **kwargs: + """ + self.pool_size = pool_size + self.driver_cls = driver_cls + self.thread_safe = thread_safe + self.kwargs = kwargs + + self.queue = queue.Queue(maxsize=pool_size) + self.lock = threading.RLock() + self.driver_count = 0 + self.ctx = threading.local() + + @property + def driver(self): + if not hasattr(self.ctx, "driver"): + self.ctx.driver = None + return self.ctx.driver + + @driver.setter + def driver(self, driver): + self.ctx.driver = driver + + @property + def is_full(self): + return self.driver_count >= self.pool_size + + def create_driver(self, user_agent: str = None, proxy: str = None): + kwargs = self.kwargs.copy() + if user_agent: + kwargs["user_agent"] = user_agent + if proxy: + kwargs["proxy"] = proxy + return self.driver_cls(**kwargs) + + def get(self, user_agent: str = None, proxy: str = None): + """ + 获取webdriver + 当webdriver为新实例时会使用 user_agen, proxy, cookie参数来创建 + Args: + user_agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36 + proxy: xxx.xxx.xxx.xxx + Returns: + + """ + if not self.is_full and not self.thread_safe: + with self.lock: + if not self.is_full: + driver = self.create_driver(user_agent, proxy) + self.queue.put(driver) + self.driver_count += 1 + elif self.thread_safe: + if not self.driver: + driver = self.create_driver(user_agent, proxy) + self.driver = driver + self.driver_count += 1 + + if self.thread_safe: + driver = self.driver + else: + driver = self.queue.get() + + return driver + + def put(self, driver): + if not self.thread_safe: + self.queue.put(driver) + + def remove(self, driver): + if self.thread_safe: + if self.driver: + self.driver.quit() + self.driver = None + else: + driver.quit() + self.driver_count -= 1 + + def close(self): + if self.thread_safe: + log.info("暂不支持关闭需线程安全的driver") + + while not self.queue.empty(): + driver = self.queue.get() + driver.quit() + self.driver_count -= 1 diff --git a/setup.py b/setup.py index 814b90c1..14542b97 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ @summary: --------- @author: Boris -@email: boris@bzkj.tech +@email: boris_liu@foxmail.com """ from os.path import dirname, join @@ -13,13 +13,13 @@ import setuptools -if version_info < (3, 6, 0): - raise SystemExit("Sorry! feapder requires python 3.6.0 or later.") +if version_info < (3, 9, 0): + raise SystemExit("Sorry! feapder requires python 3.9.0 or later.") -with open(join(dirname(__file__), "feapder/VERSION"), "rb") as f: - version = f.read().decode("ascii").strip() +with open(join(dirname(__file__), "feapder/VERSION"), "rb") as fh: + version = fh.read().decode("ascii").strip() -with open("README.md", "r") as fh: +with open("README.md", "r", encoding="utf8") as fh: long_description = fh.read() packages = setuptools.find_packages() @@ -33,30 +33,47 @@ ] ) +requires = [ + "better-exceptions>=0.2.2", + "DBUtils>=3.0", + "parsel>=1.8.1", + "PyMySQL>=1.1.0", + "redis>=5.0.0,<9.0.0", + "requests>=2.31.0", + "beautifulsoup4>=4.12.0", + "ipython>=8.0.0", + "cryptography>=41.0.0", + "urllib3>=2.0.0,<3.0.0", + "loguru>=0.5.3", + "influxdb>=5.3.1", + "pyperclip>=1.8.2", + "terminal-layout>=2.1.3", +] + +render_requires = [ + "webdriver-manager>=4.0.0", + "playwright>=1.40.0", + "selenium>=4.10.0", +] + +all_requires = [ + "bitarray>=2.8.0", + "PyExecJS>=1.5.1", + "pymongo>=4.0.0", +] + render_requires + setuptools.setup( name="feapder", version=version, author="Boris", license="MIT", author_email="feapder@qq.com", - python_requires='>=3.6', - description="feapder是一款支持分布式、批次采集、任务防丢、报警丰富的python爬虫框架", + python_requires=">=3.9", + description="feapder是一款支持分布式、批次采集、数据防丢、报警丰富的python爬虫框架", long_description=long_description, long_description_content_type="text/markdown", - install_requires=[ # 工具包的依赖包 - "better-exceptions>=0.2.2", - "DBUtils>=2.0", - "parsel>=1.5.2", - "PyExecJS>=1.5.1", - "PyMySQL>=0.9.3", - "redis>=2.10.6", - "requests>=2.22.0", - "bs4>=0.0.1", - "ipython>=7.14.0", - "bitarray>=1.5.3", - "redis-py-cluster>=1.3.4", - "cryptography>=3.3.2" - ], + install_requires=requires, + extras_require={"all": all_requires, "render": render_requires}, entry_points={"console_scripts": ["feapder = feapder.commands.cmdline:execute"]}, url="https://github.com/Boris-code/feapder.git", packages=packages, diff --git a/tests/air-spider/test_air_spider.py b/tests/air-spider/test_air_spider.py index 547dc4fb..597bfe48 100644 --- a/tests/air-spider/test_air_spider.py +++ b/tests/air-spider/test_air_spider.py @@ -5,19 +5,46 @@ @summary: --------- @author: Boris -@email: boris@bzkj.tech +@email: boris_liu@foxmail.com """ import feapder class TestAirSpider(feapder.AirSpider): + __custom_setting__ = dict( + USE_SESSION=True, + TASK_MAX_CACHED_SIZE=10, + ) + + def start_callback(self): + print("爬虫开始") + + def end_callback(self): + print("爬虫结束") + def start_requests(self, *args, **kws): - yield feapder.Request("https://www.baidu.com") + for i in range(1): + print(i) + yield feapder.Request("https://www.baidu.com") + + def download_midware(self, request): + # request.headers = {'User-Agent': ""} + # request.proxies = {"https":"https://12.12.12.12:6666"} + # request.cookies = {} + return request + + def validate(self, request, response): + if response.status_code != 200: + raise Exception("response code not 200") # 重试 + + # if "哈哈" not in response.text: + # return False # 抛弃当前请求 def parse(self, request, response): + print(response.bs4().title) print(response.xpath("//title").extract_first()) if __name__ == "__main__": - TestAirSpider().start() + TestAirSpider(thread_count=1).start() diff --git a/tests/air-spider/test_air_spider_filter.py b/tests/air-spider/test_air_spider_filter.py new file mode 100644 index 00000000..a57065d2 --- /dev/null +++ b/tests/air-spider/test_air_spider_filter.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- +""" +Created on 2020/4/22 10:41 PM +--------- +@summary: +--------- +@author: Boris +@email: boris_liu@foxmail.com +""" + +import feapder + + +class TestAirSpider(feapder.AirSpider): + __custom_setting__ = dict( + REQUEST_FILTER_ENABLE=True, # request 去重 + # REQUEST_FILTER_SETTING=dict( + # filter_type=3, # 永久去重（BloomFilter） = 1 、内存去重（MemoryFilter） = 2、 临时去重（ExpireFilter）= 3、 轻量去重（LiteFilter）= 4 + # expire_time=2592000, # 过期时间1个月 + # ), + REQUEST_FILTER_SETTING=dict( + filter_type=4, # 永久去重（BloomFilter） = 1 、内存去重（MemoryFilter） = 2、 临时去重（ExpireFilter）= 3、 轻量去重（LiteFilter）= 4 + ), + ) + + def start_requests(self, *args, **kws): + for i in range(200): + yield feapder.Request("https://www.baidu.com") + + def parse(self, request, response): + print(response.bs4().title) + + +if __name__ == "__main__": + TestAirSpider(thread_count=1).start() diff --git a/tests/air-spider/test_air_spider_item.py b/tests/air-spider/test_air_spider_item.py new file mode 100644 index 00000000..cd61ed6e --- /dev/null +++ b/tests/air-spider/test_air_spider_item.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- +""" +Created on 2021-03-30 10:27:21 +--------- +@summary: +--------- +@author: Boris +""" + +import feapder +from feapder import Item + + +class TestAirSpiderItem(feapder.AirSpider): + __custom_setting__ = dict( + MYSQL_IP="localhost", + MYSQL_PORT=3306, + MYSQL_DB="feapder", + MYSQL_USER_NAME="feapder", + MYSQL_USER_PASS="feapder123", + ITEM_FILTER_ENABLE=True, # item 去重 + ITEM_FILTER_SETTING = dict( + filter_type=4 # 永久去重（BloomFilter） = 1 、内存去重（MemoryFilter） = 2、 临时去重（ExpireFilter）= 3、轻量去重（LiteFilter）= 4 + ) + ) + + def start_requests(self): + yield feapder.Request("https://www.baidu.com") + + def parse(self, request, response): + title = response.xpath("string(//title)").extract_first() + for i in range(3): + item = Item() + item.table_name = "spider_data" + item.url = request.url + item.title = title + yield item + + +if __name__ == "__main__": + TestAirSpiderItem().start() diff --git a/tests/air-spider/test_render_spider.py b/tests/air-spider/test_render_spider.py new file mode 100644 index 00000000..3067a443 --- /dev/null +++ b/tests/air-spider/test_render_spider.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +""" +Created on 2020/4/22 10:41 PM +--------- +@summary: +--------- +@author: Boris +@email: boris_liu@foxmail.com +""" + +import feapder + + +class TestAirSpider(feapder.AirSpider): + def start_requests(self, *args, **kws): + yield feapder.Request("https://www.baidu.com", render=True) + + # def download_midware(self, request): + # request.proxies = { + # "http": "http://xxx.xxx.xxx.xxx:8888", + # "https": "http://xxx.xxx.xxx.xxx:8888", + # } + + def parse(self, request, response): + print(response.bs4().title) + + +if __name__ == "__main__": + TestAirSpider(thread_count=1).start() diff --git a/tests/batch-spider-integration/main.py b/tests/batch-spider-integration/main.py index 6a4d55b3..24035428 100644 --- a/tests/batch-spider-integration/main.py +++ b/tests/batch-spider-integration/main.py @@ -4,7 +4,7 @@ --------- @summary: 爬虫入口 --------- -@author: liubo +@author: Boris """ from feapder import ArgumentParser diff --git a/tests/batch-spider-integration/setting.py b/tests/batch-spider-integration/setting.py index c3055785..390f0dd1 100644 --- a/tests/batch-spider-integration/setting.py +++ b/tests/batch-spider-integration/setting.py @@ -29,24 +29,20 @@ # # 重新尝试失败的requests 当requests重试次数超过允许的最大重试次数算失败 # RETRY_FAILED_REQUESTS = False # # request 超时时间，超过这个时间重新做（不是网络请求的超时时间）单位秒 -# REQUEST_TIME_OUT = 600 # 10分钟 +# REQUEST_LOST_TIMEOUT = 600 # 10分钟 # # 保存失败的request # SAVE_FAILED_REQUEST = True # # # 下载缓存 利用redis缓存，由于内存小，所以仅供测试时使用 # RESPONSE_CACHED_ENABLE = False # 是否启用下载缓存 成本高的数据或容易变需求的数据，建议设置为True # RESPONSE_CACHED_EXPIRE_TIME = 3600 # 缓存时间 秒 -# RESPONSE_CACHED_USED = False # 是否使用缓存 补才数据时可设置为True +# RESPONSE_CACHED_USED = False # 是否使用缓存 补采数据时可设置为True # # WARNING_FAILED_COUNT = 1000 # 任务失败数 超过WARNING_FAILED_COUNT则报警 # # # 爬虫初始化工作 -# # 爬虫做完request后是否自动结束或者等待任务 -# AUTO_STOP_WHEN_SPIDER_DONE = True -# # 是否将item添加到 mysql 支持列表 指定添加的item 可模糊指定 -# ADD_ITEM_TO_MYSQL = True -# # 是否将item添加到 redis 支持列表 指定添加的item 可模糊指定 -# ADD_ITEM_TO_REDIS = False +# # 爬虫是否常驻 +# KEEP_ALIVE = False # # # # 设置代理 diff --git a/tests/batch-spider-integration/spiders/sina_news_parser.py b/tests/batch-spider-integration/spiders/sina_news_parser.py index 9b55f05e..ccfb2157 100644 --- a/tests/batch-spider-integration/spiders/sina_news_parser.py +++ b/tests/batch-spider-integration/spiders/sina_news_parser.py @@ -4,7 +4,7 @@ --------- @summary: --------- -@author: liubo +@author: Boris """ import feapder diff --git a/tests/batch-spider-integration/spiders/tencent_news_parser.py b/tests/batch-spider-integration/spiders/tencent_news_parser.py index 0303f209..136eccd6 100644 --- a/tests/batch-spider-integration/spiders/tencent_news_parser.py +++ b/tests/batch-spider-integration/spiders/tencent_news_parser.py @@ -4,7 +4,7 @@ --------- @summary: --------- -@author: liubo +@author: Boris """ import feapder diff --git a/tests/batch-spider/items/spider_data_item.py b/tests/batch-spider/items/spider_data_item.py index 81d1d078..3072d9a5 100644 --- a/tests/batch-spider/items/spider_data_item.py +++ b/tests/batch-spider/items/spider_data_item.py @@ -4,7 +4,7 @@ --------- @summary: --------- -@author: liubo +@author: Boris """ from feapder import Item diff --git a/tests/batch-spider/main.py b/tests/batch-spider/main.py index 78c23056..cf7e858e 100644 --- a/tests/batch-spider/main.py +++ b/tests/batch-spider/main.py @@ -13,7 +13,7 @@ def crawl_test(args): spider = test_spider.TestSpider( - redis_key="feapder:test_batch_spider", # redis中存放任务等信息的根key + redis_key="feapder:test_batch_spider", # 分布式爬虫调度信息存储位置 task_table="batch_spider_task", # mysql中的任务表 task_keys=["id", "url"], # 需要获取任务表里的字段名，可添加多个 task_state="state", # mysql中任务状态字段 @@ -30,7 +30,7 @@ def crawl_test(args): def test_debug(): spider = test_spider.TestSpider.to_DebugBatchSpider( task_id=1, - redis_key="feapder:test_batch_spider", # redis中存放任务等信息的根key + redis_key="feapder:test_batch_spider", # 分布式爬虫调度信息存储位置 task_table="batch_spider_task", # mysql中的任务表 task_keys=["id", "url"], # 需要获取任务表里的字段名，可添加多个 task_state="state", # mysql中任务状态字段 diff --git a/tests/batch-spider/setting.py b/tests/batch-spider/setting.py index fb3cfdff..a6b9c98d 100644 --- a/tests/batch-spider/setting.py +++ b/tests/batch-spider/setting.py @@ -30,24 +30,20 @@ # # 重新尝试失败的requests 当requests重试次数超过允许的最大重试次数算失败 # RETRY_FAILED_REQUESTS = False # # request 超时时间，超过这个时间重新做（不是网络请求的超时时间）单位秒 -# REQUEST_TIME_OUT = 600 # 10分钟 +# REQUEST_LOST_TIMEOUT = 600 # 10分钟 # # 保存失败的request # SAVE_FAILED_REQUEST = True # # # 下载缓存 利用redis缓存，由于内存小，所以仅供测试时使用 # RESPONSE_CACHED_ENABLE = False # 是否启用下载缓存 成本高的数据或容易变需求的数据，建议设置为True # RESPONSE_CACHED_EXPIRE_TIME = 3600 # 缓存时间 秒 -# RESPONSE_CACHED_USED = False # 是否使用缓存 补才数据时可设置为True +# RESPONSE_CACHED_USED = False # 是否使用缓存 补采数据时可设置为True # # WARNING_FAILED_COUNT = 1000 # 任务失败数 超过WARNING_FAILED_COUNT则报警 # # # 爬虫初始化工作 -# # 爬虫做完request后是否自动结束或者等待任务 -# AUTO_STOP_WHEN_SPIDER_DONE = True -# # 是否将item添加到 mysql 支持列表 指定添加的item 可模糊指定 -# ADD_ITEM_TO_MYSQL = True -# # 是否将item添加到 redis 支持列表 指定添加的item 可模糊指定 -# ADD_ITEM_TO_REDIS = False +# # 爬虫是否常驻 +# KEEP_ALIVE = False # # # # 设置代理 diff --git a/tests/batch-spider/spiders/test_spider.py b/tests/batch-spider/spiders/test_spider.py index e99a8d46..684961bb 100644 --- a/tests/batch-spider/spiders/test_spider.py +++ b/tests/batch-spider/spiders/test_spider.py @@ -18,7 +18,7 @@ class TestSpider(feapder.BatchSpider): def start_requests(self, task): # task 为在任务表中取出的每一条任务 id, url = task # id， url为所取的字段，main函数中指定的 - yield feapder.Request(url, task_id=id) + yield feapder.Request(url, task_id=id, render=True) # task_id为任务id，用于更新任务状态 def parse(self, request, response): title = response.xpath('//title/text()').extract_first() # 取标题 @@ -49,6 +49,4 @@ def failed_request(self, request, response): """ yield request - yield self.update_task_batch(request.task_id, -1) # 更新任务状态为-1 - - + yield self.update_task_batch(request.task_id, -1) # 更新任务状态为-1 diff --git a/tests/jd_spider.py b/tests/jd_spider.py index d5523002..fff316de 100644 --- a/tests/jd_spider.py +++ b/tests/jd_spider.py @@ -4,7 +4,7 @@ --------- @summary: --------- -@author: liubo +@author: Boris """ import feapder diff --git a/tests/mongo_spider.py b/tests/mongo_spider.py new file mode 100644 index 00000000..b5bc5085 --- /dev/null +++ b/tests/mongo_spider.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- +""" +Created on 2021-02-08 16:06:12 +--------- +@summary: +--------- +@author: Boris +""" + +import feapder +from feapder import Item, UpdateItem + + +class TestMongo(feapder.AirSpider): + __custom_setting__ = dict( + ITEM_PIPELINES=["feapder.pipelines.mongo_pipeline.MongoPipeline"], + MONGO_IP="localhost", + MONGO_PORT=27017, + MONGO_DB="feapder", + MONGO_USER_NAME="", + MONGO_USER_PASS="", + ) + + def start_requests(self): + yield feapder.Request("https://www.baidu.com") + + def parse(self, request, response): + title = response.xpath("//title/text()").extract_first() # 取标题 + for i in range(10): + item = Item() # 声明一个item + item.table_name = "test_mongo" + item.title = title + str(666) # 给item属性赋值 + item.i = i + 5 + item.c = "777" + yield item # 返回item， item会自动批量入库 + + +if __name__ == "__main__": + TestMongo().start() diff --git a/tests/spider-integration/main.py b/tests/spider-integration/main.py index 74765f92..84b1663b 100644 --- a/tests/spider-integration/main.py +++ b/tests/spider-integration/main.py @@ -4,7 +4,7 @@ --------- @summary: 爬虫入口 --------- -@author: liubo +@author: Boris """ from feapder import Spider diff --git a/tests/spider-integration/setting.py b/tests/spider-integration/setting.py index c3055785..390f0dd1 100644 --- a/tests/spider-integration/setting.py +++ b/tests/spider-integration/setting.py @@ -29,24 +29,20 @@ # # 重新尝试失败的requests 当requests重试次数超过允许的最大重试次数算失败 # RETRY_FAILED_REQUESTS = False # # request 超时时间，超过这个时间重新做（不是网络请求的超时时间）单位秒 -# REQUEST_TIME_OUT = 600 # 10分钟 +# REQUEST_LOST_TIMEOUT = 600 # 10分钟 # # 保存失败的request # SAVE_FAILED_REQUEST = True # # # 下载缓存 利用redis缓存，由于内存小，所以仅供测试时使用 # RESPONSE_CACHED_ENABLE = False # 是否启用下载缓存 成本高的数据或容易变需求的数据，建议设置为True # RESPONSE_CACHED_EXPIRE_TIME = 3600 # 缓存时间 秒 -# RESPONSE_CACHED_USED = False # 是否使用缓存 补才数据时可设置为True +# RESPONSE_CACHED_USED = False # 是否使用缓存 补采数据时可设置为True # # WARNING_FAILED_COUNT = 1000 # 任务失败数 超过WARNING_FAILED_COUNT则报警 # # # 爬虫初始化工作 -# # 爬虫做完request后是否自动结束或者等待任务 -# AUTO_STOP_WHEN_SPIDER_DONE = True -# # 是否将item添加到 mysql 支持列表 指定添加的item 可模糊指定 -# ADD_ITEM_TO_MYSQL = True -# # 是否将item添加到 redis 支持列表 指定添加的item 可模糊指定 -# ADD_ITEM_TO_REDIS = False +# # 爬虫是否常驻 +# KEEP_ALIVE = False # # # # 设置代理 diff --git a/tests/spider-integration/spiders/sina_news_parser.py b/tests/spider-integration/spiders/sina_news_parser.py index 59b235db..63d4e1ca 100644 --- a/tests/spider-integration/spiders/sina_news_parser.py +++ b/tests/spider-integration/spiders/sina_news_parser.py @@ -4,7 +4,7 @@ --------- @summary: --------- -@author: liubo +@author: Boris """ import feapder diff --git a/tests/spider-integration/spiders/tencent_news_parser.py b/tests/spider-integration/spiders/tencent_news_parser.py index 8ee2c421..0a72b8d1 100644 --- a/tests/spider-integration/spiders/tencent_news_parser.py +++ b/tests/spider-integration/spiders/tencent_news_parser.py @@ -4,7 +4,7 @@ --------- @summary: --------- -@author: liubo +@author: Boris """ import feapder diff --git a/tests/spider/items/spider_data_item.py b/tests/spider/items/spider_data_item.py index 59bf18e3..81731d67 100644 --- a/tests/spider/items/spider_data_item.py +++ b/tests/spider/items/spider_data_item.py @@ -1,10 +1,10 @@ # -*- coding: utf-8 -*- """ -Created on 2021-02-08 16:40:12 +Created on 2021-03-10 17:28:36 --------- @summary: --------- -@author: liubo +@author: Boris """ from feapder import Item @@ -17,5 +17,5 @@ class SpiderDataItem(Item): """ def __init__(self, *args, **kwargs): - # self.id = None # type : int(10) unsigned | allow_null : NO | key : PRI | default_value : None | extra : auto_increment | column_comment : - self.title = None # type : varchar(255) | allow_null : YES | key : | default_value : None | extra : | column_comment : + # self.id = None + self.title = None diff --git a/tests/spider/main.py b/tests/spider/main.py index 3ee9c1d9..80bbe762 100644 --- a/tests/spider/main.py +++ b/tests/spider/main.py @@ -4,46 +4,11 @@ --------- @summary: 爬虫入口 --------- -@author: liubo +@author: Boris """ from spiders import * -from feapder import Request -from feapder import ArgumentParser - - -def spider_test(): - spider = test_spider.TestSpider(redis_key="feapder:test_spider") - spider.start() - - -def spider2_test(): - spider = test_spider.TestSpider2(redis_key="feapder:test_spider2") - spider.start() - - -def debug_spider_test(): - # debug爬虫 - spider = test_spider.TestSpider.to_DebugSpider( - redis_key="feapder:test_spider", request=Request("http://www.baidu.com") - ) - spider.start() - if __name__ == "__main__": - parser = ArgumentParser(description="Spider测试") - - parser.add_argument( - "--spider_test", action="store_true", help="测试Spider", function=spider_test - ) - parser.add_argument( - "--spider2_test", action="store_true", help="测试Spider2", function=spider2_test - ) - parser.add_argument( - "--test_debug_spider", - action="store_true", - help="测试DebugSpider", - function=debug_spider_test, - ) - - parser.start() + spider = test_spider.TestSpider(redis_key="feapder3:test_spider", thread_count=100, keep_alive=False) + spider.start() \ No newline at end of file diff --git a/tests/spider/setting.py b/tests/spider/setting.py index fb3cfdff..75470361 100644 --- a/tests/spider/setting.py +++ b/tests/spider/setting.py @@ -14,41 +14,35 @@ # IP:PORT REDISDB_IP_PORTS = "localhost:6379" REDISDB_USER_PASS = "" -# 默认 0 到 15 共16个数据库 REDISDB_DB = 0 # # 爬虫相关 # # COLLECTOR -# COLLECTOR_SLEEP_TIME = 1 # 从任务队列中获取任务到内存队列的间隔 -# COLLECTOR_TASK_COUNT = 100 # 每次获取任务数量 +COLLECTOR_SLEEP_TIME = 1 # 从任务队列中获取任务到内存队列的间隔 +COLLECTOR_TASK_COUNT = 100 # 每次获取任务数量 # # # SPIDER -# SPIDER_THREAD_COUNT = 10 # 爬虫并发数 -# SPIDER_SLEEP_TIME = 0 # 下载时间间隔（解析完一个response后休眠时间） +SPIDER_THREAD_COUNT = 100 # 爬虫并发数 +SPIDER_SLEEP_TIME = 0 # 下载时间间隔（解析完一个response后休眠时间） # SPIDER_MAX_RETRY_TIMES = 100 # 每个请求最大重试次数 # # 重新尝试失败的requests 当requests重试次数超过允许的最大重试次数算失败 # RETRY_FAILED_REQUESTS = False # # request 超时时间，超过这个时间重新做（不是网络请求的超时时间）单位秒 -# REQUEST_TIME_OUT = 600 # 10分钟 +# REQUEST_LOST_TIMEOUT = 600 # 10分钟 # # 保存失败的request # SAVE_FAILED_REQUEST = True # # # 下载缓存 利用redis缓存，由于内存小，所以仅供测试时使用 # RESPONSE_CACHED_ENABLE = False # 是否启用下载缓存 成本高的数据或容易变需求的数据，建议设置为True # RESPONSE_CACHED_EXPIRE_TIME = 3600 # 缓存时间 秒 -# RESPONSE_CACHED_USED = False # 是否使用缓存 补才数据时可设置为True +# RESPONSE_CACHED_USED = False # 是否使用缓存 补采数据时可设置为True # # WARNING_FAILED_COUNT = 1000 # 任务失败数 超过WARNING_FAILED_COUNT则报警 # # # 爬虫初始化工作 -# # 爬虫做完request后是否自动结束或者等待任务 -# AUTO_STOP_WHEN_SPIDER_DONE = True -# # 是否将item添加到 mysql 支持列表 指定添加的item 可模糊指定 -# ADD_ITEM_TO_MYSQL = True -# # 是否将item添加到 redis 支持列表 指定添加的item 可模糊指定 -# ADD_ITEM_TO_REDIS = False -# +# # 爬虫是否常驻 +# KEEP_ALIVE = True # # # 设置代理 # PROXY_EXTRACT_API = None # 代理提取API ，返回的代理分割符为\r\n @@ -73,3 +67,11 @@ # LOG_LEVEL = "DEBUG" # LOG_IS_WRITE_TO_FILE = False # OTHERS_LOG_LEVAL = "ERROR" # 第三方库的log等级 +REQUEST_FILTER_ENABLE=True # request 去重 +# REQUEST_FILTER_SETTING=dict( +# filter_type=3, # 永久去重（BloomFilter） = 1 、内存去重（MemoryFilter） = 2、 临时去重（ExpireFilter）= 3、 轻量去重（LiteFilter）= 4 +# expire_time=2592000, # 过期时间1个月 +# ), +REQUEST_FILTER_SETTING=dict( + filter_type=4, # 永久去重（BloomFilter） = 1 、内存去重（MemoryFilter） = 2、 临时去重（ExpireFilter）= 3、 轻量去重（LiteFilter）= 4 +) \ No newline at end of file diff --git a/tests/spider/spiders/test_spider.py b/tests/spider/spiders/test_spider.py index d28ac5f2..af70e082 100644 --- a/tests/spider/spiders/test_spider.py +++ b/tests/spider/spiders/test_spider.py @@ -8,19 +8,28 @@ """ import feapder -from feapder.dedup import Dedup from items import * class TestSpider(feapder.Spider): - def __init__(self, *args, **kwargs): - self.dedup = Dedup() - def start_requests(self): - yield feapder.Request("https://www.baidu.com") + for i in range(1): + yield feapder.Request(f"https://www.baidu.com#{i}", callback=self.parse) + + def validate(self, request, response): + if response.status_code != 200: + raise Exception("response code not 200") # 重试 + + # if "哈哈" not in response.text: + # return False # 抛弃当前请求 def parse(self, request, response): title = response.xpath("//title/text()").extract_first() # 取标题 item = spider_data_item.SpiderDataItem() # 声明一个item item.title = title # 给item属性赋值 yield item # 返回item， item会自动批量入库 + + +if __name__ == '__main__': + spider = TestSpider(redis_key="feapder3:test_spider", thread_count=100) + spider.start() \ No newline at end of file diff --git a/tests/spider/spiders/test_spider2.py b/tests/spider/spiders/test_spider2.py index 8fb70404..9fc25cc1 100644 --- a/tests/spider/spiders/test_spider2.py +++ b/tests/spider/spiders/test_spider2.py @@ -13,7 +13,8 @@ class TestSpider2(feapder.Spider): def start_requests(self): - yield feapder.Request("https://www.baidu.com") + for i in range(100): + yield feapder.Request("https://www.baidu.com#{}".format(i)) def parse(self, request, response): title = response.xpath("//title/text()").extract_first() # 取标题 diff --git a/tests/task-spider/test_task_spider.py b/tests/task-spider/test_task_spider.py new file mode 100644 index 00000000..3a361633 --- /dev/null +++ b/tests/task-spider/test_task_spider.py @@ -0,0 +1,80 @@ +# -*- coding: utf-8 -*- +""" +Created on 2022-06-10 14:30:54 +--------- +@summary: +--------- +@author: Boris +""" + +import feapder +from feapder import ArgumentParser + + +class TestTaskSpider(feapder.TaskSpider): + def add_task(self): + # 加种子任务 框架会调用这个函数，方便往redis里塞任务，但不能写成死循环。实际业务中可以自己写个脚本往redis里塞任务 + self._redisdb.zadd(self._task_table, {"id": 1, "url": "https://www.baidu.com"}) + + def start_requests(self, task): + task_id, url = task + yield feapder.Request(url, task_id=task_id) + + def parse(self, request, response): + # 提取网站title + print(response.xpath("//title/text()").extract_first()) + # 提取网站描述 + print(response.xpath("//meta[@name='description']/@content").extract_first()) + print("网站地址: ", response.url) + + # mysql 需要更新任务状态为做完 即 state=1 + # yield self.update_task_batch(request.task_id) + + +def start(args): + """ + 用mysql做种子表 + """ + spider = TestTaskSpider( + task_table="spider_task", + task_keys=["id", "url"], + redis_key="test:task_spider", + keep_alive=True, + ) + if args == 1: + spider.start_monitor_task() + else: + spider.start() + + +def start2(args): + """ + 用redis做种子表 + """ + spider = TestTaskSpider( + task_table="spider_task2", + task_table_type="redis", + redis_key="test:task_spider", + keep_alive=True, + use_mysql=False, + ) + if args == 1: + spider.start_monitor_task() + else: + spider.start() + + +if __name__ == "__main__": + parser = ArgumentParser(description="测试TaskSpider") + + parser.add_argument( + "--start", type=int, nargs=1, help="用mysql做种子表 (1|2）", function=start + ) + parser.add_argument( + "--start2", type=int, nargs=1, help="用redis做种子表 (1|2）", function=start2 + ) + + parser.start() + + # 下发任务 python3 test_task_spider.py --start 1 + # 采集 python3 test_task_spider.py --start 2 diff --git a/tests/test-debugger/README.md b/tests/test-debugger/README.md new file mode 100644 index 00000000..c160ae2c --- /dev/null +++ b/tests/test-debugger/README.md @@ -0,0 +1,8 @@ +# xxx爬虫文档 +## 调研 + +## 数据库设计 + +## 爬虫逻辑 + +## 项目架构 \ No newline at end of file diff --git a/tests/test-debugger/items/__init__.py b/tests/test-debugger/items/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test-debugger/main.py b/tests/test-debugger/main.py new file mode 100644 index 00000000..929f347b --- /dev/null +++ b/tests/test-debugger/main.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +""" +Created on 2023-06-09 20:26:29 +--------- +@summary: 爬虫入口 +--------- +@author: Boris +""" + +import feapder + +from spiders import * + + +if __name__ == "__main__": + test_debugger.TestDebugger.to_DebugSpider( + request=feapder.Request("https://spidertools.cn", render=True), + redis_key="test:xxx", + ).start() diff --git a/tests/test-debugger/setting.py b/tests/test-debugger/setting.py new file mode 100644 index 00000000..2191f57c --- /dev/null +++ b/tests/test-debugger/setting.py @@ -0,0 +1,185 @@ +# -*- coding: utf-8 -*- +"""爬虫配置文件""" +# import os +# import sys +# +# # MYSQL +# MYSQL_IP = "localhost" +# MYSQL_PORT = 3306 +# MYSQL_DB = "" +# MYSQL_USER_NAME = "" +# MYSQL_USER_PASS = "" +# +# # MONGODB +# MONGO_IP = "localhost" +# MONGO_PORT = 27017 +# MONGO_DB = "" +# MONGO_USER_NAME = "" +# MONGO_USER_PASS = "" +# +# # REDIS +# # ip:port 多个可写为列表或者逗号隔开 如 ip1:port1,ip2:port2 或 ["ip1:port1", "ip2:port2"] +# REDISDB_IP_PORTS = "localhost:6379" +# REDISDB_USER_PASS = "" +# REDISDB_DB = 0 +# # 连接redis时携带的其他参数，如ssl=True +# REDISDB_KWARGS = dict() +# # 适用于redis哨兵模式 +# REDISDB_SERVICE_NAME = "" +# +# # 数据入库的pipeline，可自定义，默认MysqlPipeline +# ITEM_PIPELINES = [ +# "feapder.pipelines.mysql_pipeline.MysqlPipeline", +# # "feapder.pipelines.mongo_pipeline.MongoPipeline", +# # "feapder.pipelines.console_pipeline.ConsolePipeline", +# ] +# EXPORT_DATA_MAX_FAILED_TIMES = 10 # 导出数据时最大的失败次数，包括保存和更新，超过这个次数报警 +# EXPORT_DATA_MAX_RETRY_TIMES = 10 # 导出数据时最大的重试次数，包括保存和更新，超过这个次数则放弃重试 +# +# # 爬虫相关 +# # COLLECTOR +# COLLECTOR_TASK_COUNT = 32 # 每次获取任务数量，追求速度推荐32 +# +# # SPIDER +# SPIDER_THREAD_COUNT = 1 # 爬虫并发数，追求速度推荐32 +# # 下载时间间隔 单位秒。 支持随机 如 SPIDER_SLEEP_TIME = [2, 5] 则间隔为 2~5秒之间的随机数，包含2和5 +# SPIDER_SLEEP_TIME = 0 +# SPIDER_MAX_RETRY_TIMES = 10 # 每个请求最大重试次数 +# KEEP_ALIVE = False # 爬虫是否常驻 + +# 下载 +# DOWNLOADER = "feapder.network.downloader.RequestsDownloader" +# SESSION_DOWNLOADER = "feapder.network.downloader.RequestsSessionDownloader" +# RENDER_DOWNLOADER = "feapder.network.downloader.SeleniumDownloader" +# # RENDER_DOWNLOADER="feapder.network.downloader.PlaywrightDownloader" +# MAKE_ABSOLUTE_LINKS = True # 自动转成绝对连接 + +# # 浏览器渲染 +WEBDRIVER = dict( + pool_size=1, # 浏览器的数量 + load_images=True, # 是否加载图片 + user_agent=None, # 字符串 或 无参函数，返回值为user_agent + proxy=None, # xxx.xxx.xxx.xxx:xxxx 或 无参函数，返回值为代理地址 + headless=False, # 是否为无头浏览器 + driver_type="CHROME", # CHROME、EDGE、PHANTOMJS、FIREFOX + timeout=30, # 请求超时时间 + window_size=(1024, 800), # 窗口大小 + executable_path=None, # 浏览器路径，默认为默认路径 + render_time=0, # 渲染时长，即打开网页等待指定时间后再获取源码 + custom_argument=[ + "--ignore-certificate-errors", + "--disable-blink-features=AutomationControlled", + ], # 自定义浏览器渲染参数 + xhr_url_regexes=None, # 拦截xhr接口，支持正则，数组类型 + auto_install_driver=True, # 自动下载浏览器驱动 支持chrome 和 firefox + download_path=None, # 下载文件的路径 + use_stealth_js=False, # 使用stealth.min.js隐藏浏览器特征 +) + +# PLAYWRIGHT = dict( +# user_agent=None, # 字符串 或 无参函数，返回值为user_agent +# proxy=None, # xxx.xxx.xxx.xxx:xxxx 或 无参函数，返回值为代理地址 +# headless=False, # 是否为无头浏览器 +# driver_type="chromium", # chromium、firefox、webkit +# timeout=30, # 请求超时时间 +# window_size=(1024, 800), # 窗口大小 +# executable_path=None, # 浏览器路径，默认为默认路径 +# download_path=None, # 下载文件的路径 +# render_time=0, # 渲染时长，即打开网页等待指定时间后再获取源码 +# wait_until="networkidle", # 等待页面加载完成的事件,可选值："commit", "domcontentloaded", "load", "networkidle" +# use_stealth_js=False, # 使用stealth.min.js隐藏浏览器特征 +# page_on_event_callback=None, # page.on() 事件的回调 如 page_on_event_callback={"dialog": lambda dialog: dialog.accept()} +# storage_state_path=None, # 保存浏览器状态的路径 +# url_regexes=None, # 拦截接口，支持正则，数组类型 +# save_all=False, # 是否保存所有拦截的接口, 配合url_regexes使用，为False时只保存最后一次拦截的接口 +# ) +# +# # 爬虫启动时，重新抓取失败的requests +# RETRY_FAILED_REQUESTS = False +# # 爬虫启动时，重新入库失败的item +# RETRY_FAILED_ITEMS = False +# # 保存失败的request +# SAVE_FAILED_REQUEST = True +# # request防丢机制。（指定的REQUEST_LOST_TIMEOUT时间内request还没做完，会重新下发 重做） +# REQUEST_LOST_TIMEOUT = 600 # 10分钟 +# # request网络请求超时时间 +# REQUEST_TIMEOUT = 22 # 等待服务器响应的超时时间，浮点数，或(connect timeout, read timeout)元组 +# # item在内存队列中最大缓存数量 +# ITEM_MAX_CACHED_COUNT = 5000 +# # item每批入库的最大数量 +# ITEM_UPLOAD_BATCH_MAX_SIZE = 1000 +# # item入库时间间隔 +# ITEM_UPLOAD_INTERVAL = 1 +# # 内存任务队列最大缓存的任务数，默认不限制；仅对AirSpider有效。 +# TASK_MAX_CACHED_SIZE = 0 +# +# # 下载缓存 利用redis缓存，但由于内存大小限制，所以建议仅供开发调试代码时使用，防止每次debug都需要网络请求 +# RESPONSE_CACHED_ENABLE = False # 是否启用下载缓存 成本高的数据或容易变需求的数据，建议设置为True +# RESPONSE_CACHED_EXPIRE_TIME = 3600 # 缓存时间 秒 +# RESPONSE_CACHED_USED = False # 是否使用缓存 补采数据时可设置为True +# +# # 设置代理 +# PROXY_EXTRACT_API = None # 代理提取API ，返回的代理分割符为\r\n +# PROXY_ENABLE = True +# +# # 随机headers +# RANDOM_HEADERS = True +# # UserAgent类型 支持 'chrome', 'opera', 'firefox', 'internetexplorer', 'safari'，'mobile' 若不指定则随机类型 +# USER_AGENT_TYPE = "chrome" +# # 默认使用的浏览器头 +# DEFAULT_USERAGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36" +# # requests 使用session +# USE_SESSION = False +# +# # 去重 +# ITEM_FILTER_ENABLE = False # item 去重 +# REQUEST_FILTER_ENABLE = False # request 去重 +# ITEM_FILTER_SETTING = dict( +# filter_type=1 # 永久去重（BloomFilter） = 1 、内存去重（MemoryFilter） = 2、 临时去重（ExpireFilter）= 3、轻量去重（LiteFilter）= 4 +# ) +# REQUEST_FILTER_SETTING = dict( +# filter_type=3, # 永久去重（BloomFilter） = 1 、内存去重（MemoryFilter） = 2、 临时去重（ExpireFilter）= 3、 轻量去重（LiteFilter）= 4 +# expire_time=2592000, # 过期时间1个月 +# ) +# +# # 报警 支持钉钉、飞书、企业微信、邮件 +# # 钉钉报警 +# DINGDING_WARNING_URL = "" # 钉钉机器人api +# DINGDING_WARNING_PHONE = "" # 报警人 支持列表，可指定多个 +# DINGDING_WARNING_ALL = False # 是否提示所有人， 默认为False +# # 飞书报警 +# # https://open.feishu.cn/document/ukTMukTMukTM/ucTM5YjL3ETO24yNxkjN#e1cdee9f +# FEISHU_WARNING_URL = "" # 飞书机器人api +# FEISHU_WARNING_USER = None # 报警人 {"open_id":"ou_xxxxx", "name":"xxxx"} 或 [{"open_id":"ou_xxxxx", "name":"xxxx"}] +# FEISHU_WARNING_ALL = False # 是否提示所有人， 默认为False +# # 邮件报警 +# EMAIL_SENDER = "" # 发件人 +# EMAIL_PASSWORD = "" # 授权码 +# EMAIL_RECEIVER = "" # 收件人 支持列表，可指定多个 +# EMAIL_SMTPSERVER = "smtp.163.com" # 邮件服务器 默认为163邮箱 +# # 企业微信报警 +# WECHAT_WARNING_URL = "" # 企业微信机器人api +# WECHAT_WARNING_PHONE = "" # 报警人 将会在群内@此人, 支持列表，可指定多人 +# WECHAT_WARNING_ALL = False # 是否提示所有人， 默认为False +# # 时间间隔 +# WARNING_INTERVAL = 3600 # 相同报警的报警时间间隔，防止刷屏; 0表示不去重 +# WARNING_LEVEL = "DEBUG" # 报警级别， DEBUG / INFO / ERROR +# WARNING_FAILED_COUNT = 1000 # 任务失败数 超过WARNING_FAILED_COUNT则报警 +# +# LOG_NAME = os.path.basename(os.getcwd()) +# LOG_PATH = "log/%s.log" % LOG_NAME # log存储路径 +# LOG_LEVEL = "DEBUG" +# LOG_COLOR = True # 是否带有颜色 +# LOG_IS_WRITE_TO_CONSOLE = True # 是否打印到控制台 +# LOG_IS_WRITE_TO_FILE = False # 是否写文件 +# LOG_MODE = "w" # 写文件的模式 +# LOG_MAX_BYTES = 10 * 1024 * 1024 # 每个日志文件的最大字节数 +# LOG_BACKUP_COUNT = 20 # 日志文件保留数量 +# LOG_ENCODING = "utf8" # 日志文件编码 +# OTHERS_LOG_LEVAL = "ERROR" # 第三方库的log等级 +# +# # 切换工作路径为当前项目路径 +# project_path = os.path.abspath(os.path.dirname(__file__)) +# os.chdir(project_path) # 切换工作路经 +# sys.path.insert(0, project_path) +# print("当前工作路径为 " + os.getcwd()) diff --git a/tests/test-debugger/spiders/__init__.py b/tests/test-debugger/spiders/__init__.py new file mode 100644 index 00000000..4243fbe2 --- /dev/null +++ b/tests/test-debugger/spiders/__init__.py @@ -0,0 +1,3 @@ +__all__ = [ + "test_debugger" +] \ No newline at end of file diff --git a/tests/test-debugger/spiders/test_debugger.py b/tests/test-debugger/spiders/test_debugger.py new file mode 100644 index 00000000..2ef73f56 --- /dev/null +++ b/tests/test-debugger/spiders/test_debugger.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- +""" +Created on 2023-06-09 20:26:47 +--------- +@summary: +--------- +@author: Boris +""" + +import feapder + + +class TestDebugger(feapder.Spider): + def start_requests(self): + yield feapder.Request("https://spidertools.cn", render=True) + + def parse(self, request, response): + # 提取网站title + print(response.xpath("//title/text()").extract_first()) + # 提取网站描述 + print(response.xpath("//meta[@name='description']/@content").extract_first()) + print("网站地址: ", response.url) + + +if __name__ == "__main__": + TestDebugger.to_DebugSpider( + request=feapder.Request("https://spidertools.cn", render=True), redis_key="test:xxx" + ).start() diff --git a/tests/test-pipeline/items/__init__.py b/tests/test-pipeline/items/__init__.py new file mode 100644 index 00000000..ee6b05eb --- /dev/null +++ b/tests/test-pipeline/items/__init__.py @@ -0,0 +1,3 @@ +__all__ = [ + "spider_data_item" +] \ No newline at end of file diff --git a/tests/test-pipeline/items/spider_data_item.py b/tests/test-pipeline/items/spider_data_item.py new file mode 100644 index 00000000..1960649a --- /dev/null +++ b/tests/test-pipeline/items/spider_data_item.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- +""" +Created on 2021-02-08 16:39:27 +--------- +@summary: +--------- +@author: Boris +""" + +from feapder import Item +from feapder.pipelines.csv_pipeline import CsvPipeline + + +class SpiderDataItem(Item): + """ + This class was generated by feapder. + command: feapder create -i spider_data. + """ + __pipelines__ = [CsvPipeline()] + + def __init__(self, *args, **kwargs): + # self.id = None # type : int(10) unsigned | allow_null : NO | key : PRI | default_value : None | extra : auto_increment | column_comment : + self.title = None # type : varchar(255) | allow_null : YES | key : | default_value : None | extra : | column_comment : diff --git a/tests/test-pipeline/main.py b/tests/test-pipeline/main.py new file mode 100644 index 00000000..c6454dd9 --- /dev/null +++ b/tests/test-pipeline/main.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- +""" +Created on 2021-02-08 16:02:02 +--------- +@summary: 爬虫入口 +--------- +@author: Boris +""" + +from spiders import * +from feapder import ArgumentParser + + +def crawl_test(args): + spider = test_spider.TestSpider( + redis_key="feapder:test_batch_spider", # 分布式爬虫调度信息存储位置 + task_table="batch_spider_task", # mysql中的任务表 + task_keys=["id", "url"], # 需要获取任务表里的字段名，可添加多个 + task_state="state", # mysql中任务状态字段 + batch_record_table="batch_spider_batch_record", # mysql中的批次记录表 + batch_name="批次爬虫测试(周全)", # 批次名字 + batch_interval=7, # 批次周期 天为单位 若为小时 可写 1 / 24 + ) + + if args == 1: + spider.start_monitor_task() # 下发及监控任务 + else: + spider.start() # 采集 + + +if __name__ == "__main__": + + parser = ArgumentParser(description="批次爬虫测试") + + parser.add_argument( + "--crawl_test", type=int, nargs=1, help="(1|2）", function=crawl_test + ) + + parser.start() + + # 运行 + # 下发任务及监控进度 python3 main.py --crawl_test 1 + # 采集 python3 main.py --crawl_test 2 \ No newline at end of file diff --git a/tests/test-pipeline/pipeline.py b/tests/test-pipeline/pipeline.py new file mode 100644 index 00000000..00ccdf18 --- /dev/null +++ b/tests/test-pipeline/pipeline.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +""" +Created on 2021/3/18 12:39 上午 +--------- +@summary: +--------- +@author: Boris +@email: boris_liu@foxmail.com +""" + +from feapder.pipelines import BasePipeline +from typing import Dict, List, Tuple + + +class Pipeline(BasePipeline): + """ + pipeline 是单线程的，批量保存数据的操作，不建议在这里写网络请求代码，如下载图片等 + """ + + def save_items(self, table, items: List[Dict]) -> bool: + """ + 保存数据 + Args: + table: 表名 + items: 数据，[{},{},...] + + Returns: 是否保存成功 True / False + 若False，不会将本批数据入到去重库，以便再次入库 + + """ + + print("自定义pipeline， 保存数据 >>>>", table, items) + + return True + + def update_items(self, table, items: List[Dict], update_keys=Tuple) -> bool: + """ + 更新数据, 与UpdateItem配合使用，若爬虫中没使用UpdateItem，则可不实现此接口 + Args: + table: 表名 + items: 数据，[{},{},...] + update_keys: 更新的字段, 如 ("title", "publish_time") + + Returns: 是否更新成功 True / False + 若False，不会将本批数据入到去重库，以便再次入库 + + """ + + print("自定义pipeline， 更新数据 >>>>", table, items, update_keys) + + return True diff --git a/tests/test-pipeline/setting.py b/tests/test-pipeline/setting.py new file mode 100644 index 00000000..ba985f09 --- /dev/null +++ b/tests/test-pipeline/setting.py @@ -0,0 +1,77 @@ +# -*- coding: utf-8 -*- +"""爬虫配置文件""" +import os + + +# MYSQL +MYSQL_IP = "localhost" +MYSQL_PORT = 3306 +MYSQL_DB = "feapder" +MYSQL_USER_NAME = "feapder" +MYSQL_USER_PASS = "feapder123" + +# REDIS +# IP:PORT +REDISDB_IP_PORTS = "localhost:6379" +REDISDB_USER_PASS = "" +# 默认 0 到 15 共16个数据库 +REDISDB_DB = 0 + +# 数据入库的pipeline，可自定义，默认MysqlPipeline +ITEM_PIPELINES = [ + "pipeline.Pipeline", + # "feapder.pipelines.csv_pipeline.CsvPipeline" +] + +# # 爬虫相关 +# # COLLECTOR +# COLLECTOR_SLEEP_TIME = 1 # 从任务队列中获取任务到内存队列的间隔 +# COLLECTOR_TASK_COUNT = 100 # 每次获取任务数量 +# +# # SPIDER +# SPIDER_THREAD_COUNT = 10 # 爬虫并发数 +# SPIDER_SLEEP_TIME = 0 # 下载时间间隔（解析完一个response后休眠时间） +# SPIDER_MAX_RETRY_TIMES = 100 # 每个请求最大重试次数 + +# # 重新尝试失败的requests 当requests重试次数超过允许的最大重试次数算失败 +# RETRY_FAILED_REQUESTS = False +# # request 超时时间，超过这个时间重新做（不是网络请求的超时时间）单位秒 +# REQUEST_LOST_TIMEOUT = 600 # 10分钟 +# # 保存失败的request +# SAVE_FAILED_REQUEST = True +# +# # 下载缓存 利用redis缓存，由于内存小，所以仅供测试时使用 +# RESPONSE_CACHED_ENABLE = False # 是否启用下载缓存 成本高的数据或容易变需求的数据，建议设置为True +# RESPONSE_CACHED_EXPIRE_TIME = 3600 # 缓存时间 秒 +# RESPONSE_CACHED_USED = False # 是否使用缓存 补采数据时可设置为True +# +# WARNING_FAILED_COUNT = 1000 # 任务失败数 超过WARNING_FAILED_COUNT则报警 +# +# # 爬虫初始化工作 +# # 爬虫是否常驻 +# KEEP_ALIVE = False +# +# +# # 设置代理 +# PROXY_EXTRACT_API = None # 代理提取API ，返回的代理分割符为\r\n +# PROXY_ENABLE = True +# +# # 随机headers +# RANDOM_HEADERS = True +# # requests 使用session +# USE_SESSION = False +# +# # 去重 +# ITEM_FILTER_ENABLE = False # item 去重 +# REQUEST_FILTER_ENABLE = False # request 去重 +# +# # 报警 +# DINGDING_WARNING_URL = "" # 钉钉机器人api +# DINGDING_WARNING_PHONE = "" # 报警人 +# LINGXI_TOKEN = "" # 灵犀报警token +# +# LOG_NAME = os.path.basename(os.getcwd()) +# LOG_PATH = "log/%s.log" % LOG_NAME # log存储路径 +# LOG_LEVEL = "DEBUG" +# LOG_IS_WRITE_TO_FILE = False +# OTHERS_LOG_LEVAL = "ERROR" # 第三方库的log等级 diff --git a/tests/test-pipeline/spiders/__init__.py b/tests/test-pipeline/spiders/__init__.py new file mode 100644 index 00000000..5a149c44 --- /dev/null +++ b/tests/test-pipeline/spiders/__init__.py @@ -0,0 +1,3 @@ +__all__ = [ + "test_spider" +] \ No newline at end of file diff --git a/tests/test-pipeline/spiders/test_csv_pipeline_spider.py b/tests/test-pipeline/spiders/test_csv_pipeline_spider.py new file mode 100644 index 00000000..83d4b842 --- /dev/null +++ b/tests/test-pipeline/spiders/test_csv_pipeline_spider.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- +""" +Created on 2025-12-16 14:52:29 +--------- +@summary: +--------- +@author: Boris +""" + +import feapder +from items import * + + +class TestCsvPipelineSpider(feapder.AirSpider): + def start_requests(self): + for i in range(100): + yield feapder.Request("https://baidu.com", page=i) + + def parse(self, request, response): + # 提取网站title + title = response.xpath("//title/text()").extract_first() + item = spider_data_item.SpiderDataItem() # 声明一个item + item.title = title # 给item属性赋值 + yield item # 返回item， item会自动批量入库 + + +if __name__ == "__main__": + TestCsvPipelineSpider().start() diff --git a/tests/test-pipeline/spiders/test_spider.py b/tests/test-pipeline/spiders/test_spider.py new file mode 100644 index 00000000..e99a8d46 --- /dev/null +++ b/tests/test-pipeline/spiders/test_spider.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- +""" +Created on 2021-02-08 16:09:47 +--------- +@summary: +--------- +@author: Boris +""" + +import feapder +from items import * + + +class TestSpider(feapder.BatchSpider): + # def init_task(self): + # pass + + def start_requests(self, task): + # task 为在任务表中取出的每一条任务 + id, url = task # id， url为所取的字段，main函数中指定的 + yield feapder.Request(url, task_id=id) + + def parse(self, request, response): + title = response.xpath('//title/text()').extract_first() # 取标题 + item = spider_data_item.SpiderDataItem() # 声明一个item + item.title = title # 给item属性赋值 + yield item # 返回item， item会自动批量入库 + yield self.update_task_batch(request.task_id, 1) # 更新任务状态为1 + + def exception_request(self, request, response): + """ + @summary: 请求或者parser里解析出异常的request + --------- + @param request: + @param response: + --------- + @result: request / callback / None (返回值必须可迭代) + """ + + pass + + def failed_request(self, request, response): + """ + @summary: 超过最大重试次数的request + --------- + @param request: + --------- + @result: request / item / callback / None (返回值必须可迭代) + """ + + yield request + yield self.update_task_batch(request.task_id, -1) # 更新任务状态为-1 + + diff --git a/tests/test-pipeline/table.sql b/tests/test-pipeline/table.sql new file mode 100644 index 00000000..172c43c8 --- /dev/null +++ b/tests/test-pipeline/table.sql @@ -0,0 +1,23 @@ +-- ---------------------------- +-- Table structure for spider_data +-- ---------------------------- +CREATE TABLE `spider_data` ( + `id` int(10) unsigned NOT NULL AUTO_INCREMENT, + `title` varchar(255) DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- ---------------------------- +-- Table structure for batch_spider_task +-- ---------------------------- +CREATE TABLE `batch_spider_task` ( + `id` int(10) unsigned NOT NULL AUTO_INCREMENT, + `url` varchar(255) DEFAULT NULL, + `state` int(11) DEFAULT '0', + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- ---------------------------- +-- Records of batch_spider_task +-- ---------------------------- +INSERT INTO `batch_spider_task` VALUES (1, 'https://www.baidu.com', 0); diff --git a/tests/test_csv_pipeline/test_functionality.py b/tests/test_csv_pipeline/test_functionality.py new file mode 100644 index 00000000..190c9137 --- /dev/null +++ b/tests/test_csv_pipeline/test_functionality.py @@ -0,0 +1,454 @@ +# -*- coding: utf-8 -*- +""" +CSV Pipeline 功能测试 + +测试内容： +1. 基础功能测试 +2. 异常处理测试 +3. 边界条件测试 +4. 兼容性测试 + +Created on 2025-10-16 +@author: 道长 +@email: ctrlf4@yeah.net +""" + +import csv +import os +import sys +import shutil +from pathlib import Path + +# 添加项目路径 +sys.path.insert(0, str(Path(__file__).parent.parent.parent)) + +from feapder.pipelines.csv_pipeline import CsvPipeline + + +class FunctionalityTester: + """CSV Pipeline 功能测试器""" + + def __init__(self, test_dir="test_output"): + """初始化测试器""" + self.test_dir = test_dir + self.pipeline = None + self.passed = 0 + self.failed = 0 + + def setup(self): + """测试前准备""" + if os.path.exists(self.test_dir): + shutil.rmtree(self.test_dir) + + os.makedirs(self.test_dir, exist_ok=True) + + csv_dir = os.path.join(self.test_dir, "csv") + self.pipeline = CsvPipeline(csv_dir=csv_dir) + + print(f"✅ 测试环境准备完成") + + def teardown(self): + """测试后清理""" + if self.pipeline: + self.pipeline.close() + + def assert_true(self, condition, message): + """断言真""" + if condition: + print(f" ✅ {message}") + self.passed += 1 + else: + print(f" ❌ {message}") + self.failed += 1 + + def assert_false(self, condition, message): + """断言假""" + self.assert_true(not condition, message) + + def assert_equal(self, actual, expected, message): + """断言相等""" + if actual == expected: + print(f" ✅ {message}") + self.passed += 1 + else: + print(f" ❌ {message} (期望: {expected}, 实际: {actual})") + self.failed += 1 + + def test_basic_save(self): + """测试基础保存功能""" + print("\n" + "=" * 80) + print("测试 1: 基础保存功能") + print("=" * 80) + + # 测试保存单条数据 + item = {"id": 1, "name": "Test Product", "price": 99.99} + result = self.pipeline.save_items("product", [item]) + self.assert_true(result, "保存单条数据") + + # 检查文件是否创建 + csv_file = os.path.join(self.pipeline.csv_dir, "product.csv") + self.assert_true(os.path.exists(csv_file), "CSV 文件已创建") + + # 检查数据是否正确 + with open(csv_file, 'r', encoding='utf-8', newline='') as f: + reader = csv.DictReader(f) + rows = list(reader) + self.assert_equal(len(rows), 1, "文件中有 1 条数据") + if rows: + self.assert_equal(rows[0]["id"], "1", "数据 ID 正确") + self.assert_equal(rows[0]["name"], "Test Product", "数据名称正确") + + def test_batch_save(self): + """测试批量保存""" + print("\n" + "=" * 80) + print("测试 2: 批量保存功能") + print("=" * 80) + + # 生成测试数据 + items = [] + for i in range(10): + items.append({ + "id": i + 1, + "name": f"Product_{i + 1}", + "price": 100 + i, + }) + + result = self.pipeline.save_items("batch_test", items) + self.assert_true(result, "批量保存 10 条数据") + + # 检查数据行数 + csv_file = os.path.join(self.pipeline.csv_dir, "batch_test.csv") + with open(csv_file, 'r', encoding='utf-8', newline='') as f: + reader = csv.DictReader(f) + rows = list(reader) + self.assert_equal(len(rows), 10, "批量保存数据行数正确") + + def test_empty_items(self): + """测试空数据处理""" + print("\n" + "=" * 80) + print("测试 3: 空数据处理") + print("=" * 80) + + result = self.pipeline.save_items("empty_test", []) + self.assert_true(result, "空数据列表返回 True") + + def test_special_characters(self): + """测试特殊字符处理""" + print("\n" + "=" * 80) + print("测试 4: 特殊字符处理") + print("=" * 80) + + items = [ + { + "id": 1, + "name": "产品名称", + "description": 'Contains "quotes" and, commas', + "emoji": "😀🎉🚀", + "newline": "Line1\nLine2", + } + ] + + result = self.pipeline.save_items("special_chars", items) + self.assert_true(result, "保存包含特殊字符的数据") + + # 读取并检查 + csv_file = os.path.join(self.pipeline.csv_dir, "special_chars.csv") + with open(csv_file, 'r', encoding='utf-8', newline='') as f: + reader = csv.DictReader(f) + rows = list(reader) + if rows: + self.assert_equal(rows[0]["name"], "产品名称", "中文字符正确") + self.assert_equal( + rows[0].get("emoji", ""), + "😀🎉🚀", + "Emoji 正确" + ) + + def test_multiple_tables(self): + """测试多表存储""" + print("\n" + "=" * 80) + print("测试 5: 多表存储") + print("=" * 80) + + tables = ["product", "user", "order"] + for table in tables: + item = {"id": 1, "name": f"Test {table}"} + result = self.pipeline.save_items(table, [item]) + self.assert_true(result, f"保存到表 {table}") + + # 检查所有文件 + for table in tables: + csv_file = os.path.join(self.pipeline.csv_dir, f"{table}.csv") + self.assert_true(os.path.exists(csv_file), f"表 {table} 的 CSV 文件存在") + + def test_header_only_once(self): + """测试表头只写一次""" + print("\n" + "=" * 80) + print("测试 6: 表头只写一次") + print("=" * 80) + + table = "header_test" + + # 第一次写入 + items1 = [{"id": 1, "name": "Product 1"}] + self.pipeline.save_items(table, items1) + + # 第二次写入 + items2 = [{"id": 2, "name": "Product 2"}] + self.pipeline.save_items(table, items2) + + # 检查表头行数 + csv_file = os.path.join(self.pipeline.csv_dir, f"{table}.csv") + with open(csv_file, 'r', encoding='utf-8', newline='') as f: + lines = f.readlines() + # 应该是：1 个表头 + 2 条数据 + self.assert_equal(len(lines), 3, "文件中只有 1 行表头和 2 行数据") + + def test_numeric_values(self): + """测试数值类型""" + print("\n" + "=" * 80) + print("测试 7: 数值类型处理") + print("=" * 80) + + items = [ + { + "id": 1, + "price": 99.99, + "stock": 100, + "rating": 4.5, + "active": True, + } + ] + + result = self.pipeline.save_items("numeric_test", items) + self.assert_true(result, "保存包含各类数值的数据") + + # 读取并检查 + csv_file = os.path.join(self.pipeline.csv_dir, "numeric_test.csv") + with open(csv_file, 'r', encoding='utf-8', newline='') as f: + reader = csv.DictReader(f) + rows = list(reader) + if rows: + self.assert_equal(rows[0]["price"], "99.99", "浮点数正确") + self.assert_equal(rows[0]["stock"], "100", "整数正确") + self.assert_equal(rows[0]["rating"], "4.5", "小数正确") + + def test_large_values(self): + """测试大值处理""" + print("\n" + "=" * 80) + print("测试 8: 大值处理") + print("=" * 80) + + large_text = "x" * 10000 # 10KB 的文本 + items = [ + { + "id": 1, + "name": "Large Content", + "content": large_text, + } + ] + + result = self.pipeline.save_items("large_test", items) + self.assert_true(result, "保存大内容数据") + + # 检查数据完整性 + csv_file = os.path.join(self.pipeline.csv_dir, "large_test.csv") + with open(csv_file, 'r', encoding='utf-8', newline='') as f: + reader = csv.DictReader(f) + rows = list(reader) + if rows: + self.assert_equal( + len(rows[0]["content"]), + len(large_text), + "大内容数据完整" + ) + + def test_update_items_fallback(self): + """测试 update_items 降级为 save""" + print("\n" + "=" * 80) + print("测试 9: update_items 降级为 save") + print("=" * 80) + + items = [{"id": 1, "name": "Product 1", "price": 100}] + result = self.pipeline.update_items("update_test", items, ("price",)) + self.assert_true(result, "update_items 返回 True") + + # 检查数据是否存在 + csv_file = os.path.join(self.pipeline.csv_dir, "update_test.csv") + self.assert_true(os.path.exists(csv_file), "update_items 创建了 CSV 文件") + + def test_file_operations(self): + """测试文件操作""" + print("\n" + "=" * 80) + print("测试 10: 文件操作") + print("=" * 80) + + items = [{"id": 1, "name": "Test"}] + table = "file_test" + + result = self.pipeline.save_items(table, items) + self.assert_true(result, "保存数据") + + csv_file = os.path.join(self.pipeline.csv_dir, f"{table}.csv") + + # 检查文件是否可读 + try: + with open(csv_file, 'r', encoding='utf-8') as f: + f.read() + self.assert_true(True, "CSV 文件可读") + except Exception as e: + self.assert_true(False, f"CSV 文件可读 ({e})") + + # 检查文件大小 + file_size = os.path.getsize(csv_file) + self.assert_true(file_size > 0, f"CSV 文件大小 > 0 ({file_size} 字节)") + + def test_concurrent_same_table(self): + """测试同表并发写入""" + print("\n" + "=" * 80) + print("测试 11: 同表并发写入（Per-Table Lock）") + print("=" * 80) + + import threading + + table = "concurrent_same_table" + errors = [] + + def write_data(thread_id): + try: + items = [{"id": thread_id, "name": f"Item_{thread_id}"}] + result = self.pipeline.save_items(table, items) + if not result: + errors.append(f"线程{thread_id}写入失败") + except Exception as e: + errors.append(f"线程{thread_id}异常: {e}") + + # 创建多个线程 + threads = [] + for i in range(5): + t = threading.Thread(target=write_data, args=(i,)) + t.start() + threads.append(t) + + # 等待所有线程完成 + for t in threads: + t.join() + + self.assert_equal(len(errors), 0, "并发写入无错误") + + # 检查数据完整性 + csv_file = os.path.join(self.pipeline.csv_dir, f"{table}.csv") + with open(csv_file, 'r', encoding='utf-8', newline='') as f: + reader = csv.DictReader(f) + rows = list(reader) + self.assert_true(len(rows) > 0, "并发写入产生了数据") + + def test_directory_creation(self): + """测试目录自动创建""" + print("\n" + "=" * 80) + print("测试 12: 目录自动创建") + print("=" * 80) + + # 创建新的 pipeline 实例，指定不存在的目录 + new_csv_dir = os.path.join(self.test_dir, "new_csv_dir") + self.assert_false(os.path.exists(new_csv_dir), "新目录不存在") + + new_pipeline = CsvPipeline(csv_dir=new_csv_dir) + self.assert_true(os.path.exists(new_csv_dir), "目录自动创建") + + new_pipeline.close() + + def test_none_values(self): + """测试 None 值处理""" + print("\n" + "=" * 80) + print("测试 13: None 值处理") + print("=" * 80) + + items = [ + { + "id": 1, + "name": "Product", + "description": None, + "optional_field": "", + } + ] + + result = self.pipeline.save_items("none_test", items) + self.assert_true(result, "保存包含 None 值的数据") + + # 检查文件 + csv_file = os.path.join(self.pipeline.csv_dir, "none_test.csv") + with open(csv_file, 'r', encoding='utf-8', newline='') as f: + reader = csv.DictReader(f) + rows = list(reader) + if rows: + # None 会被转换为字符串 "None" + self.assert_true("None" in rows[0]["description"], + "None 值被正确处理") + + def run_all_tests(self): + """运行所有测试""" + print("\n") + print("╔" + "═" * 78 + "╗") + print("║" + " CSV Pipeline 功能测试 ".center(78) + "║") + print("║" + " 作者: 道长 | 日期: 2025-10-16 ".center(78) + "║") + print("╚" + "═" * 78 + "╝") + + try: + self.setup() + + # 运行所有测试 + self.test_basic_save() + self.test_batch_save() + self.test_empty_items() + self.test_special_characters() + self.test_multiple_tables() + self.test_header_only_once() + self.test_numeric_values() + self.test_large_values() + self.test_update_items_fallback() + self.test_file_operations() + self.test_concurrent_same_table() + self.test_directory_creation() + self.test_none_values() + + # 打印总结 + self.print_summary() + + return self.failed == 0 + + except Exception as e: + print(f"\n❌ 测试过程中出错: {e}") + import traceback + traceback.print_exc() + return False + + finally: + self.teardown() + + def print_summary(self): + """打印测试总结""" + print("\n" + "=" * 80) + print("测试总结") + print("=" * 80) + print(f"✅ 通过: {self.passed}") + print(f"❌ 失败: {self.failed}") + print(f"总计: {self.passed + self.failed}") + + if self.failed == 0: + print("\n🎉 所有测试通过！") + else: + print(f"\n⚠️ 有 {self.failed} 个测试失败") + + print("=" * 80) + + +def main(): + """主函数""" + tester = FunctionalityTester(test_dir="tests/test_csv_pipeline/test_output_func") + success = tester.run_all_tests() + return 0 if success else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_csv_pipeline/test_performance.py b/tests/test_csv_pipeline/test_performance.py new file mode 100644 index 00000000..94eb64a7 --- /dev/null +++ b/tests/test_csv_pipeline/test_performance.py @@ -0,0 +1,537 @@ +# -*- coding: utf-8 -*- +""" +CSV Pipeline 性能测试 + +测试内容： +1. 批量写入性能 +2. 并发写入性能 +3. 内存占用情况 +4. 文件大小和数据完整性 + +Created on 2025-10-16 +@author: 道长 +@email: ctrlf4@yeah.net +""" + +import csv +import os +import sys +import time +import shutil +import threading +import psutil +from pathlib import Path +from typing import List, Dict + +# 添加项目路径 +sys.path.insert(0, str(Path(__file__).parent.parent.parent)) + +from feapder.pipelines.csv_pipeline import CsvPipeline + + +class PerformanceTester: + """CSV Pipeline 性能测试器""" + + def __init__(self, test_dir="test_output"): + """初始化测试器""" + self.test_dir = test_dir + self.pipeline = None + self.process = psutil.Process() + self.test_results = {} + + def setup(self): + """测试前准备""" + # 清理历史测试目录 + if os.path.exists(self.test_dir): + shutil.rmtree(self.test_dir) + + # 创建测试输出目录 + os.makedirs(self.test_dir, exist_ok=True) + + # 初始化 Pipeline + csv_dir = os.path.join(self.test_dir, "csv") + self.pipeline = CsvPipeline(csv_dir=csv_dir) + + print(f"✅ 测试环境准备完成，输出目录: {self.test_dir}") + + def teardown(self): + """测试后清理""" + if self.pipeline: + self.pipeline.close() + + def generate_test_data(self, count: int) -> List[Dict]: + """生成测试数据""" + data = [] + for i in range(count): + data.append({ + "id": i + 1, + "name": f"Product_{i + 1}", + "price": 99.99 + i * 0.1, + "category": "Electronics", + "url": f"https://example.com/product/{i + 1}", + "stock": 100 - (i % 50), + "rating": 4.5 + (i % 5) * 0.1, + "description": f"Description for product {i + 1}" * 3, + }) + return data + + def test_single_batch_performance(self): + """测试单批写入性能""" + print("\n" + "=" * 80) + print("测试 1: 单批写入性能") + print("=" * 80) + + batch_sizes = [100, 500, 1000, 5000] + results = {} + + for batch_size in batch_sizes: + data = self.generate_test_data(batch_size) + + # 测试写入时间 + start_time = time.time() + success = self.pipeline.save_items("product", data) + elapsed = time.time() - start_time + + # 测试结果 + results[batch_size] = { + "success": success, + "elapsed_time": elapsed, + "throughput": batch_size / elapsed if elapsed > 0 else 0, + } + + print(f"批量大小: {batch_size:5d} | " + f"耗时: {elapsed:.4f}s | " + f"吞吐量: {results[batch_size]['throughput']:.0f} 条/秒 | " + f"状态: {'✅' if success else '❌'}") + + self.test_results["single_batch"] = results + return results + + def test_concurrent_write_performance(self): + """测试并发写入性能""" + print("\n" + "=" * 80) + print("测试 2: 并发写入性能（模拟多爬虫线程）") + print("=" * 80) + + thread_counts = [1, 2, 4, 8] + results = {} + + for thread_count in thread_counts: + # 每个线程写入的数据条数 + items_per_thread = 100 + total_items = thread_count * items_per_thread + + def write_thread(thread_id): + """线程工作函数""" + data = self.generate_test_data(items_per_thread) + # 为了模拟不同表，使用不同的表名 + table_name = f"product_thread_{thread_id}" + return self.pipeline.save_items(table_name, data) + + # 记录初始内存 + mem_before = self.process.memory_info().rss / 1024 / 1024 + + # 并发执行 + start_time = time.time() + threads = [] + for i in range(thread_count): + t = threading.Thread(target=write_thread, args=(i,)) + t.start() + threads.append(t) + + # 等待所有线程完成 + for t in threads: + t.join() + + elapsed = time.time() - start_time + mem_after = self.process.memory_info().rss / 1024 / 1024 + mem_delta = mem_after - mem_before + + results[thread_count] = { + "total_items": total_items, + "elapsed_time": elapsed, + "throughput": total_items / elapsed if elapsed > 0 else 0, + "memory_delta_mb": mem_delta, + } + + print(f"线程数: {thread_count} | " + f"总数据: {total_items:5d} | " + f"耗时: {elapsed:.4f}s | " + f"吞吐量: {results[thread_count]['throughput']:.0f} 条/秒 | " + f"内存增长: {mem_delta:.2f}MB") + + self.test_results["concurrent_write"] = results + return results + + def test_memory_usage(self): + """测试内存占用""" + print("\n" + "=" * 80) + print("测试 3: 内存占用情况") + print("=" * 80) + + # 测试不同数量的数据对内存的影响 + test_counts = [1000, 5000, 10000, 50000] + results = {} + + for count in test_counts: + data = self.generate_test_data(count) + + # 记录内存 + mem_before = self.process.memory_info().rss / 1024 / 1024 + + # 执行写入 + start_time = time.time() + self.pipeline.save_items("product_memory", data) + elapsed = time.time() - start_time + + mem_after = self.process.memory_info().rss / 1024 / 1024 + mem_used = mem_after - mem_before + mem_per_item = mem_used / count if count > 0 else 0 + + results[count] = { + "memory_before_mb": mem_before, + "memory_after_mb": mem_after, + "memory_used_mb": mem_used, + "memory_per_item_kb": mem_per_item * 1024, + "elapsed_time": elapsed, + } + + print(f"数据条数: {count:6d} | " + f"内存占用: {mem_used:6.2f}MB | " + f"每条数据: {mem_per_item * 1024:.2f}KB | " + f"耗时: {elapsed:.4f}s") + + self.test_results["memory_usage"] = results + return results + + def test_file_integrity(self): + """测试文件完整性""" + print("\n" + "=" * 80) + print("测试 4: 文件完整性检查") + print("=" * 80) + + # 写入测试数据 + test_data = self.generate_test_data(1000) + table_name = "product_integrity" + + success = self.pipeline.save_items(table_name, test_data) + + if not success: + print("❌ 写入失败") + return {"status": "failed"} + + # 检查文件是否存在 + csv_file = os.path.join(self.pipeline.csv_dir, f"{table_name}.csv") + if not os.path.exists(csv_file): + print("❌ CSV 文件不存在") + return {"status": "file_not_found"} + + # 读取 CSV 文件并检查数据完整性 + read_data = [] + with open(csv_file, 'r', encoding='utf-8', newline='') as f: + reader = csv.DictReader(f) + for row in reader: + read_data.append(row) + + # 对比数据 + if len(read_data) != len(test_data): + print(f"❌ 数据条数不符: 写入{len(test_data)}条，读取{len(read_data)}条") + return { + "status": "count_mismatch", + "written": len(test_data), + "read": len(read_data), + } + + # 检查字段是否完整 + expected_fields = set(test_data[0].keys()) + actual_fields = set(read_data[0].keys()) + if expected_fields != actual_fields: + print(f"❌ 字段不符\n期望: {expected_fields}\n实际: {actual_fields}") + return { + "status": "field_mismatch", + "expected": list(expected_fields), + "actual": list(actual_fields), + } + + # 检查数据值是否正确（抽样检查） + sample_indices = [0, len(test_data) // 2, len(test_data) - 1] + for idx in sample_indices: + original = test_data[idx] + read = read_data[idx] + + for key in original.keys(): + if str(original[key]) != read.get(key, ""): + print(f"❌ 数据不符 (第{idx}行, 字段{key})\n" + f"期望: {original[key]}\n" + f"实际: {read.get(key)}") + return {"status": "data_mismatch", "index": idx, "field": key} + + print(f"✅ 文件完整性检查通过") + print(f" 总条数: {len(read_data)}") + print(f" 字段数: {len(actual_fields)}") + print(f" 文件大小: {os.path.getsize(csv_file) / 1024:.2f}KB") + + return { + "status": "passed", + "total_rows": len(read_data), + "total_fields": len(actual_fields), + "file_size_kb": os.path.getsize(csv_file) / 1024, + } + + def test_append_mode(self): + """测试追加模式（断点续爬）""" + print("\n" + "=" * 80) + print("测试 5: 追加模式（断点续爬）") + print("=" * 80) + + table_name = "product_append" + + # 第一次写入 + data1 = self.generate_test_data(100) + self.pipeline.save_items(table_name, data1) + + csv_file = os.path.join(self.pipeline.csv_dir, f"{table_name}.csv") + size_after_first = os.path.getsize(csv_file) if os.path.exists(csv_file) else 0 + + # 第二次写入（追加） + data2 = self.generate_test_data(100) + self.pipeline.save_items(table_name, data2) + + size_after_second = os.path.getsize(csv_file) if os.path.exists(csv_file) else 0 + + # 读取文件检查数据 + read_data = [] + with open(csv_file, 'r', encoding='utf-8', newline='') as f: + reader = csv.DictReader(f) + for row in reader: + read_data.append(row) + + # 检查是否正确追加 + if len(read_data) == len(data1) + len(data2): + print(f"✅ 追加模式正常") + print(f" 第一次写入: {len(data1)} 条") + print(f" 第二次写入: {len(data2)} 条") + print(f" 最终总数: {len(read_data)} 条") + print(f" 第一次后大小: {size_after_first / 1024:.2f}KB") + print(f" 第二次后大小: {size_after_second / 1024:.2f}KB") + + return { + "status": "passed", + "first_write": len(data1), + "second_write": len(data2), + "total": len(read_data), + "size_growth_kb": (size_after_second - size_after_first) / 1024, + } + else: + print(f"❌ 追加模式异常: 期望{len(data1) + len(data2)}条，实际{len(read_data)}条") + return { + "status": "failed", + "expected": len(data1) + len(data2), + "actual": len(read_data), + } + + def test_concurrent_safety(self): + """测试并发安全性（Per-Table Lock）""" + print("\n" + "=" * 80) + print("测试 6: 并发安全性（Per-Table Lock）") + print("=" * 80) + + table_name = "product_concurrent_safety" + thread_count = 4 + items_per_thread = 250 + + errors = [] + lock = threading.Lock() + + def write_thread(thread_id): + """线程工作函数""" + try: + data = self.generate_test_data(items_per_thread) + success = self.pipeline.save_items(table_name, data) + if not success: + with lock: + errors.append(f"线程{thread_id}写入失败") + except Exception as e: + with lock: + errors.append(f"线程{thread_id}异常: {e}") + + # 并发执行 + threads = [] + start_time = time.time() + for i in range(thread_count): + t = threading.Thread(target=write_thread, args=(i,)) + t.start() + threads.append(t) + + for t in threads: + t.join() + + elapsed = time.time() - start_time + + # 检查文件 + csv_file = os.path.join(self.pipeline.csv_dir, f"{table_name}.csv") + read_data = [] + with open(csv_file, 'r', encoding='utf-8', newline='') as f: + reader = csv.DictReader(f) + for row in reader: + read_data.append(row) + + expected_total = thread_count * items_per_thread + + if len(errors) == 0 and len(read_data) == expected_total: + print(f"✅ 并发安全性测试通过") + print(f" 线程数: {thread_count}") + print(f" 每线程数据: {items_per_thread}") + print(f" 期望总数: {expected_total}") + print(f" 实际总数: {len(read_data)}") + print(f" 耗时: {elapsed:.4f}s") + print(f" 吞吐量: {expected_total / elapsed:.0f} 条/秒") + + return { + "status": "passed", + "thread_count": thread_count, + "items_per_thread": items_per_thread, + "expected_total": expected_total, + "actual_total": len(read_data), + "elapsed_time": elapsed, + "throughput": expected_total / elapsed, + } + else: + print(f"❌ 并发安全性测试失败") + if errors: + for error in errors: + print(f" {error}") + if len(read_data) != expected_total: + print(f" 数据条数不符: 期望{expected_total}条，实际{len(read_data)}条") + + return { + "status": "failed", + "errors": errors, + "expected_total": expected_total, + "actual_total": len(read_data), + } + + def test_multiple_tables(self): + """测试多表存储""" + print("\n" + "=" * 80) + print("测试 7: 多表存储") + print("=" * 80) + + tables = ["product", "user", "order"] + rows_per_table = 500 + results = {} + + start_time = time.time() + + for table in tables: + data = self.generate_test_data(rows_per_table) + success = self.pipeline.save_items(table, data) + + csv_file = os.path.join(self.pipeline.csv_dir, f"{table}.csv") + file_size = os.path.getsize(csv_file) / 1024 if os.path.exists(csv_file) else 0 + + results[table] = { + "success": success, + "file_size_kb": file_size, + } + + print(f"表: {table:10s} | 状态: {'✅' if success else '❌'} | " + f"文件大小: {file_size:.2f}KB") + + elapsed = time.time() - start_time + + # 检查所有文件 + csv_dir = self.pipeline.csv_dir + files = [f for f in os.listdir(csv_dir) if f.endswith('.csv')] + + print(f"\n✅ 多表存储测试完成") + print(f" 表数: {len(tables)}") + print(f" 每表行数: {rows_per_table}") + print(f" 生成的 CSV 文件: {len(files)}") + print(f" 耗时: {elapsed:.4f}s") + + return { + "status": "passed", + "tables": results, + "file_count": len(files), + "elapsed_time": elapsed, + } + + def run_all_tests(self): + """运行所有测试""" + print("\n") + print("╔" + "═" * 78 + "╗") + print("║" + " CSV Pipeline 性能和功能测试 ".center(78) + "║") + print("║" + " 作者: 道长 | 日期: 2025-10-16 ".center(78) + "║") + print("╚" + "═" * 78 + "╝") + + try: + self.setup() + + # 运行所有测试 + self.test_single_batch_performance() + self.test_concurrent_write_performance() + self.test_memory_usage() + self.test_file_integrity() + self.test_append_mode() + self.test_concurrent_safety() + self.test_multiple_tables() + + # 打印总结 + self.print_summary() + + return True + + except Exception as e: + print(f"\n❌ 测试过程中出错: {e}") + import traceback + traceback.print_exc() + return False + + finally: + self.teardown() + + def print_summary(self): + """打印测试总结""" + print("\n" + "=" * 80) + print("测试总结") + print("=" * 80) + + # 单批性能总结 + if "single_batch" in self.test_results: + print("\n1. 单批写入性能:") + results = self.test_results["single_batch"] + for batch_size, data in results.items(): + print(f" {batch_size:5d} 条: {data['throughput']:.0f} 条/秒, " + f"耗时 {data['elapsed_time']:.4f}s") + + # 并发性能总结 + if "concurrent_write" in self.test_results: + print("\n2. 并发写入性能:") + results = self.test_results["concurrent_write"] + for thread_count, data in results.items(): + print(f" {thread_count} 线程: {data['throughput']:.0f} 条/秒, " + f"内存增长 {data['memory_delta_mb']:.2f}MB") + + # 内存占用总结 + if "memory_usage" in self.test_results: + print("\n3. 内存占用情况:") + results = self.test_results["memory_usage"] + for count, data in results.items(): + print(f" {count:6d} 条: {data['memory_used_mb']:.2f}MB, " + f"每条 {data['memory_per_item_kb']:.2f}KB") + + print("\n" + "=" * 80) + print("✅ 所有测试完成！") + print("=" * 80) + + +def main(): + """主函数""" + tester = PerformanceTester(test_dir="tests/test_csv_pipeline/test_output") + success = tester.run_all_tests() + return 0 if success else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_dedup.py b/tests/test_dedup.py index 943afd1a..84d4131f 100644 --- a/tests/test_dedup.py +++ b/tests/test_dedup.py @@ -1,56 +1,104 @@ -from feapder.dedup import Dedup - -data = {"xxx": 123, "xxxx": "xxxx"} - -datas = ["xxx", "bbb"] - - -def test_MemoryFilter(): - dedup = Dedup(Dedup.MemoryFilter) # 表名为test 历史数据3秒有效期 - - # 逐条去重 - assert dedup.add(data) == 1 - assert dedup.get(data) == 1 - - # 批量去重 - assert dedup.add(datas) == [1, 1] - assert dedup.get(datas) == [1, 1] - +import unittest -def test_ExpireFilter(): - dedup = Dedup( - Dedup.ExpireFilter, expire_time=10, redis_url="redis://@localhost:6379/0" - ) +from redis import Redis - # 逐条去重 - assert dedup.add(data) == 1 - assert dedup.get(data) == 1 - - # 批量去重 - assert dedup.add(datas) == [1, 1] - assert dedup.get(datas) == [1, 1] - - -def test_BloomFilter(): - dedup = Dedup(Dedup.BloomFilter, redis_url="redis://@localhost:6379/0") - - # 逐条去重 - assert dedup.add(data) == 1 - assert dedup.get(data) == 1 - - # 批量去重 - assert dedup.add(datas) == [1, 1] - assert dedup.get(datas) == [1, 1] - - -def test_filter(): - dedup = Dedup(Dedup.BloomFilter, redis_url="redis://@localhost:6379/0") +from feapder.dedup import Dedup - # 制造已存在数据 - datas = ["xxx", "bbb"] - dedup.add(datas) - # 过滤掉已存在数据 "xxx", "bbb" - datas = ["xxx", "bbb", "ccc"] - dedup.filter_exist_data(datas) - assert datas == ["ccc"] +class TestDedup(unittest.TestCase): + def clear(self): + self.absolute_name = "test_dedup" + redis = Redis.from_url("redis://@localhost:6379/0", decode_responses=True) + keys = redis.keys(self.absolute_name + "*") + if keys: + redis.delete(*keys) + + def setUp(self) -> None: + self.clear() + self.mock_data() + + def tearDown(self) -> None: + self.clear() + + def mock_data(self): + self.data = {"xxx": 123, "xxxx": "xxxx"} + self.datas = ["xxx", "bbb", "xxx"] + + def test_MemoryFilter(self): + dedup = Dedup( + Dedup.MemoryFilter, absolute_name=self.absolute_name + ) # 表名为test 历史数据3秒有效期 + + # 逐条去重 + self.assertEqual(dedup.add(self.data), 1) + self.assertEqual(dedup.get(self.data), 1) + + # 批量去重 + self.assertEqual(dedup.get(self.datas), [0, 0, 1]) + self.assertEqual(dedup.add(self.datas), [1, 1, 0]) + self.assertEqual(dedup.get(self.datas), [1, 1, 1]) + + def test_ExpireFilter(self): + dedup = Dedup( + Dedup.ExpireFilter, + expire_time=10, + redis_url="redis://@localhost:6379/0", + absolute_name=self.absolute_name, + ) + + # 逐条去重 + self.assertEqual(dedup.add(self.data), 1) + self.assertEqual(dedup.get(self.data), 1) + + # 批量去重 + self.assertEqual(dedup.get(self.datas), [0, 0, 1]) + self.assertEqual(dedup.add(self.datas), [1, 1, 0]) + self.assertEqual(dedup.get(self.datas), [1, 1, 1]) + + def test_BloomFilter(self): + dedup = Dedup( + Dedup.BloomFilter, + redis_url="redis://@localhost:6379/0", + absolute_name=self.absolute_name, + ) + + # 逐条去重 + self.assertEqual(dedup.add(self.data), 1) + self.assertEqual(dedup.get(self.data), 1) + + # 批量去重 + self.assertEqual(dedup.get(self.datas), [0, 0, 1]) + self.assertEqual(dedup.add(self.datas), [1, 1, 0]) + self.assertEqual(dedup.get(self.datas), [1, 1, 1]) + + def test_LiteFilter(self): + dedup = Dedup( + Dedup.LiteFilter, + ) + + # 逐条去重 + self.assertEqual(dedup.add(self.data), 1) + self.assertEqual(dedup.get(self.data), 1) + + # 批量去重 + self.assertEqual(dedup.get(self.datas), [0, 0, 1]) + self.assertEqual(dedup.add(self.datas), [1, 1, 0]) + self.assertEqual(dedup.get(self.datas), [1, 1, 1]) + + def test_filter(self): + dedup = Dedup( + Dedup.BloomFilter, + redis_url="redis://@localhost:6379/0", + to_md5=True, + absolute_name=self.absolute_name, + ) + + # 制造已存在数据 + self.datas = ["xxx", "bbb"] + result = dedup.add(self.datas) + self.assertEqual(result, [1, 1]) + + # 过滤掉已存在数据 "xxx", "bbb" + self.datas = ["xxx", "bbb", "ccc"] + dedup.filter_exist_data(self.datas) + self.assertEqual(self.datas, ["ccc"]) diff --git a/tests/test_dependency_modernization.py b/tests/test_dependency_modernization.py new file mode 100644 index 00000000..e9a73b2a --- /dev/null +++ b/tests/test_dependency_modernization.py @@ -0,0 +1,115 @@ +import inspect + +from feapder.db.redisdb import RedisDB +from feapder.utils.webdriver.selenium_driver import SeleniumDriver + + +class OldStyleService: + def __init__( + self, executable_path=None, port=0, service_args=None, log_path=None, **kwargs + ): + self.executable_path = executable_path + self.port = port + self.service_args = service_args + self.log_path = log_path + + +class NewStyleService: + def __init__( + self, executable_path=None, port=0, service_args=None, log_output=None, **kwargs + ): + self.executable_path = executable_path + self.port = port + self.service_args = service_args + self.log_output = log_output + + +class FakeBrowser: + def set_window_size(self, *args): + self.window_size = args + + +def make_selenium_driver(**kwargs): + driver = object.__new__(SeleniumDriver) + driver._kwargs = kwargs + driver._executable_path = kwargs.pop("executable_path", "/tmp/driver") + driver._auto_install_driver = False + driver._proxy = None + driver._user_agent = None + driver._load_images = True + driver._headless = False + driver._custom_argument = None + driver._window_size = None + return driver + + +def test_selenium_service_log_path_supports_old_and_new_service_api(): + driver = make_selenium_driver( + service_log_path="/tmp/webdriver.log", service_args=["--verbose"], port=1234 + ) + + old_service = driver.build_service(OldStyleService) + new_service = driver.build_service(NewStyleService) + + assert old_service.executable_path == "/tmp/driver" + assert old_service.service_args == ["--verbose"] + assert old_service.port == 1234 + assert old_service.log_path == "/tmp/webdriver.log" + assert new_service.executable_path == "/tmp/driver" + assert new_service.service_args == ["--verbose"] + assert new_service.port == 1234 + assert new_service.log_output == "/tmp/webdriver.log" + + +def test_selenium_firefox_binary_maps_to_options_binary_location(): + driver = make_selenium_driver(firefox_binary="/tmp/firefox") + captured = {} + + def create_driver(driver_cls, options, service): + captured["options"] = options + captured["service"] = service + return FakeBrowser() + + driver.create_driver = create_driver + driver.build_service = lambda *args, **kwargs: None + + assert driver.firefox_driver() is not None + assert captured["options"].binary_location == "/tmp/firefox" + assert captured["service"] is None + + +def test_selenium_driver_kwargs_keep_public_constructor_args_internal(): + driver = make_selenium_driver( + keep_alive=False, + executable_path="/tmp/driver", + desired_capabilities={"acceptInsecureCerts": True}, + service_args=["--verbose"], + ) + + assert driver.get_driver_kwargs() == {"keep_alive": False} + + +def test_redisdb_public_api_signatures_are_preserved(): + expected = { + "__init__": [ + "self", + "ip_ports", + "db", + "user_pass", + "url", + "decode_responses", + "service_name", + "max_connections", + "kwargs", + ], + "from_url": ["url"], + "zadd": ["self", "table", "values", "prioritys"], + "zget": ["self", "table", "count", "is_pop"], + "zexists": ["self", "table", "values"], + "setbit": ["self", "table", "offsets", "values"], + "getbit": ["self", "table", "offsets"], + } + + for method_name, parameters in expected.items(): + signature = inspect.signature(getattr(RedisDB, method_name)) + assert list(signature.parameters) == parameters diff --git a/tests/test_download_midware.py b/tests/test_download_midware.py new file mode 100644 index 00000000..1accbaf7 --- /dev/null +++ b/tests/test_download_midware.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- +""" +Created on 2023/9/21 13:59 +--------- +@summary: +--------- +@author: Boris +@email: boris_liu@foxmail.com +""" + +import feapder + + +def download_midware(request): + print("outter download_midware") + return request + + +class TestAirSpider(feapder.AirSpider): + def start_requests(self): + yield feapder.Request( + "https://www.baidu.com", download_midware=download_midware + ) + + def parse(self, request, response): + print(request, response) + + +class TestSpiderSpider(feapder.Spider): + def start_requests(self): + yield feapder.Request( + "https://www.baidu.com", download_midware=[download_midware, self.download_midware] + ) + + def download_midware(self, request): + print("class download_midware") + return request + + def parse(self, request, response): + print(request, response) + + +if __name__ == "__main__": + # TestAirSpider().start() + TestSpiderSpider(redis_key="test").start() diff --git a/tests/test_lock.py b/tests/test_lock.py new file mode 100644 index 00000000..b31360bc --- /dev/null +++ b/tests/test_lock.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +""" +Created on 2021/7/15 5:00 下午 +--------- +@summary: +--------- +@author: Boris +@email: boris_liu@foxmail.com +""" + +from feapder.utils.redis_lock import RedisLock +from feapder.db.redisdb import RedisDB +import time + +def test_lock(): + with RedisLock(key="test", redis_cli=RedisDB().get_redis_obj(), wait_timeout=10) as _lock: + if _lock.locked: + print(1) + time.sleep(100) + +if __name__ == '__main__': + test_lock() \ No newline at end of file diff --git a/tests/test_log.py b/tests/test_log.py new file mode 100644 index 00000000..c044a238 --- /dev/null +++ b/tests/test_log.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +""" +Created on 2021/6/18 10:36 上午 +--------- +@summary: +--------- +@author: Boris +@email: boris_liu@foxmail.com +""" + +from feapder.utils.log import log + +log.debug("debug") +log.info("info") +log.success("success") +log.warning("warning") +log.error("error") +log.critical("critical") +log.exception("exception") \ No newline at end of file diff --git a/tests/test_metrics.py b/tests/test_metrics.py new file mode 100644 index 00000000..308c2711 --- /dev/null +++ b/tests/test_metrics.py @@ -0,0 +1,52 @@ +import asyncio + +from feapder.utils import metrics + +# 初始化打点系统 +metrics.init( + influxdb_host="localhost", + influxdb_port="8086", + influxdb_udp_port="8089", + influxdb_database="feapder", + influxdb_user="***", + influxdb_password="***", + influxdb_measurement="test_metrics", + debug=True, +) + + +async def test_counter_async(): + for i in range(100): + await metrics.aemit_counter("total count", count=100, classify="test5") + for j in range(100): + await metrics.aemit_counter("key", count=1, classify="test5") + + +def test_counter(): + for i in range(100): + metrics.emit_counter("total count", count=100, classify="test5") + for j in range(100): + metrics.emit_counter("key", count=1, classify="test5") + + +def test_store(): + metrics.emit_store("total", 100, classify="cookie_count") + + +def test_time(): + metrics.emit_timer("total", 100, classify="time") + + +def test_any(): + metrics.emit_any( + tags={"_key": "total", "_type": "any"}, fields={"_value": 100}, classify="time" + ) + + +if __name__ == "__main__": + asyncio.run(test_counter_async()) + test_counter_async() + test_store() + test_time() + test_any() + metrics.close() diff --git a/tests/test_mongodb.py b/tests/test_mongodb.py new file mode 100644 index 00000000..2c006506 --- /dev/null +++ b/tests/test_mongodb.py @@ -0,0 +1,141 @@ +import unittest + +from feapder.db.mongodb import MongoDB + + +class TestMongoDB(unittest.TestCase): + coll_name = "test" + + def setUp(self) -> None: + # self.db = MongoDB(ip="localhost", port=27017, db="feapder") + self.db = MongoDB.from_url("mongodb://localhost:27017/feapder") + + def test_create_index(self): + self.db.drop_collection(coll_name=self.coll_name) + self.db.create_index(self.coll_name, ["a", "b"]) + + def test_get_indexx(self): + index = self.db.get_index(self.coll_name) + print(index) + + def test_find(self): + """ + 查询数据 + @return: + """ + r = self.db.find( + coll_name=self.coll_name, limit=2, condition={"a": 1}, projection={"_id": 0} + ) + print(r) + + def test_insert(self): + """ + 插入单条数据 + """ + r = self.db.add(coll_name=self.coll_name, data={"a": 1, "b": "你好", "c": "哈哈"}) + print(r) + self.assertEqual(r, 1) + + def test_insert_replace(self): + """ + 插入单条数据，冲突时自动更新，即将重复数据替换为最新数据 + """ + r = self.db.add( + coll_name=self.coll_name, data={"a": 1, "b": "你好", "c": "啦啦"}, replace=True + ) + self.assertEqual(r, 1) + + def test_insert_columns(self): + """ + 插入单条数据，发生冲突时，更新指定字段 + """ + r = self.db.add( + coll_name=self.coll_name, + data={"a": 1, "b": "你好", "c": "666"}, + update_columns=("c",), + ) + self.assertEqual(r, 1) + + def test_insert_update_columns_value(self): + """ + 插入单条数据，发生冲突时，用指定的值更新指定字段 + """ + r = self.db.add( + coll_name=self.coll_name, + data={"a": 1, "b": "你好", "c": "666"}, + update_columns=("c",), + update_columns_value=("888",), + ) + self.assertEqual(r, 1) + + def test_batch_insert(self): + """ + 测试批量数据插入，冲突时忽略 + @return: + """ + items = [{"a": 1, "b": "你好", "c": "666"}, {"a": 2, "b": "他好", "c": "888"}] + add_count = self.db.add_batch(self.coll_name, items) + datas_size = len(items) + print("共导出 %s 条数据 到 %s, 重复 %s 条" % (datas_size, "test", datas_size - add_count)) + + def test_batch_insert_replace(self): + """ + 测试批量插入重复数据，重复时覆盖 + @return: + """ + items = [{"a": 1, "b": "你好", "c": "xixixi"}, {"a": 2, "b": "他好", "c": "777"}] + add_count = self.db.add_batch(self.coll_name, items, replace=True) + datas_size = len(items) + print("共导出 %s 条数据 到 %s, 重复 %s 条" % (datas_size, "test", datas_size - add_count)) + self.assertEqual(add_count, 0) + + def test_batch_insert_update_columns(self): + """ + 当数据冲突时，更新指定字段 + """ + items = [{"a": 1, "b": "你好", "c": "88"}, {"a": 2, "b": "他好", "c": "888"}] + add_count = self.db.add_batch(self.coll_name, items, update_columns=("c",)) + datas_size = len(items) + print("共导出 %s 条数据 到 %s, 重复 %s 条" % (datas_size, "test", datas_size - add_count)) + self.assertEqual(add_count, 0) + + def test_batch_insert_update_columns_value(self): + """ + 指定columns及columns_value + 当数据重复时, 用指定的值更新指定字段 + """ + items = [{"a": 1, "b": "你好", "c": "88"}, {"a": 2, "b": "他好", "c": "888"}] + add_count = self.db.add_batch( + self.coll_name, items, update_columns=("c",), update_columns_value=("haha",) + ) + datas_size = len(items) + print("共导出 %s 条数据 到 %s, 重复 %s 条" % (datas_size, "test", datas_size - add_count)) + self.assertEqual(add_count, 0) + + def test_update(self): + """ + 测试单条数据更新 + """ + data = {"a": 1, "b": "你好", "c": "666"} + r = self.db.update(self.coll_name, data, {"a": 1}) + self.assertEqual(r, True) + + def test_delete(self): + r = self.db.delete(self.coll_name, {"a": 1}) + self.assertEqual(r, True) + + def test_run_command(self): + """ + 测试运行指令 + @return: + """ + r = self.db.run_command({"find": self.coll_name, "filter": {}}) + print(r) + + def test_drop_collection(self): + r = self.db.drop_collection(self.coll_name) + print(r) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_mysqldb.py b/tests/test_mysqldb.py new file mode 100644 index 00000000..1fdd9c09 --- /dev/null +++ b/tests/test_mysqldb.py @@ -0,0 +1,11 @@ +from feapder.db.mysqldb import MysqlDB + + +db = MysqlDB( + ip="localhost", port=3306, db="feapder", user_name="feapder", user_pass="feapder123", set_session=["SET time_zone='+08:00'"] +) + +MysqlDB.from_url("mysql://feapder:feapder123@localhost:3306/feapder?charset=utf8mb4") + +result = db.find("SELECT @@global.time_zone, @@session.time_zone, date_format(NOW(), '%Y-%m-%d %H:%i:%s')") +print(f"Database timezone info: {result}") \ No newline at end of file diff --git a/tests/test_playwright.py b/tests/test_playwright.py new file mode 100644 index 00000000..91668c9e --- /dev/null +++ b/tests/test_playwright.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- +""" +Created on 2022/9/15 8:47 PM +--------- +@summary: +--------- +@author: Boris +@email: boris_liu@foxmail.com +""" + +import time + +from playwright.sync_api import Page + +import feapder +from feapder.utils.webdriver import PlaywrightDriver + + +class TestPlaywright(feapder.AirSpider): + __custom_setting__ = dict( + RENDER_DOWNLOADER="feapder.network.downloader.PlaywrightDownloader", + ) + + def start_requests(self): + yield feapder.Request("https://www.baidu.com", render=True) + + def parse(self, reqeust, response): + driver: PlaywrightDriver = response.driver + page: Page = driver.page + + page.type("#kw", "feapder") + page.click("#su") + page.wait_for_load_state("networkidle") + time.sleep(1) + + html = page.content() + response.text = html # 使response加载最新的页面 + for data_container in response.xpath("//div[@class='c-container']"): + print(data_container.xpath("string(.//h3)").extract_first()) + + +if __name__ == "__main__": + TestPlaywright(thread_count=1).run() diff --git a/tests/test_playwright2.py b/tests/test_playwright2.py new file mode 100644 index 00000000..fefeb897 --- /dev/null +++ b/tests/test_playwright2.py @@ -0,0 +1,92 @@ +# -*- coding: utf-8 -*- +""" +Created on 2022/9/15 8:47 PM +--------- +@summary: +--------- +@author: Boris +@email: boris_liu@foxmail.com +""" + +from playwright.sync_api import Response +from feapder.utils.webdriver import ( + PlaywrightDriver, + InterceptResponse, + InterceptRequest, +) + +import feapder + + +def on_response(response: Response): + print(response.url) + + +class TestPlaywright(feapder.AirSpider): + __custom_setting__ = dict( + RENDER_DOWNLOADER="feapder.network.downloader.PlaywrightDownloader", + PLAYWRIGHT=dict( + user_agent=None, # 字符串 或 无参函数，返回值为user_agent + proxy=None, # xxx.xxx.xxx.xxx:xxxx 或 无参函数，返回值为代理地址 + headless=False, # 是否为无头浏览器 + driver_type="chromium", # chromium、firefox、webkit + timeout=30, # 请求超时时间 + window_size=(1024, 800), # 窗口大小 + executable_path=None, # 浏览器路径，默认为默认路径 + download_path=None, # 下载文件的路径 + render_time=0, # 渲染时长，即打开网页等待指定时间后再获取源码 + wait_until="networkidle", # 等待页面加载完成的事件,可选值："commit", "domcontentloaded", "load", "networkidle" + use_stealth_js=False, # 使用stealth.min.js隐藏浏览器特征 + # page_on_event_callback=dict(response=on_response), # 监听response事件 + # page.on() 事件的回调 如 page_on_event_callback={"dialog": lambda dialog: dialog.accept()} + storage_state_path=None, # 保存浏览器状态的路径 + url_regexes=["wallpaper/list"], # 拦截接口，支持正则，数组类型 + save_all=True, # 是否保存所有拦截的接口 + ), + ) + + def start_requests(self): + yield feapder.Request( + "http://www.soutushenqi.com/image/search/?searchWord=%E6%A0%91%E5%8F%B6", + render=True, + ) + + def parse(self, reqeust, response): + driver: PlaywrightDriver = response.driver + + intercept_response: InterceptResponse = driver.get_response("wallpaper/list") + intercept_request: InterceptRequest = intercept_response.request + + req_url = intercept_request.url + req_header = intercept_request.headers + req_data = intercept_request.data + print("请求url", req_url) + print("请求header", req_header) + print("请求data", req_data) + + data = driver.get_json("wallpaper/list") + print("接口返回的数据", data) + + print("------ 测试save_all=True ------- ") + + # 测试save_all=True + all_intercept_response: list = driver.get_all_response("wallpaper/list") + for intercept_response in all_intercept_response: + intercept_request: InterceptRequest = intercept_response.request + req_url = intercept_request.url + req_header = intercept_request.headers + req_data = intercept_request.data + print("请求url", req_url) + print("请求header", req_header) + print("请求data", req_data) + + all_intercept_json = driver.get_all_json("wallpaper/list") + for intercept_json in all_intercept_json: + print("接口返回的数据", intercept_json) + + # 千万别忘了 + driver.clear_cache() + + +if __name__ == "__main__": + TestPlaywright(thread_count=1).run() diff --git a/tests/test_rander.py b/tests/test_rander.py new file mode 100644 index 00000000..12bdab09 --- /dev/null +++ b/tests/test_rander.py @@ -0,0 +1,22 @@ +import feapder + + +class XueQiuSpider(feapder.AirSpider): + def start_requests(self): + for i in range(10): + yield feapder.Request("https://baidu.com/#{}".format(i), render=True) + + def parse(self, request, response): + print(response.cookies.get_dict()) + print(response.headers) + print(response.browser) + print("response.url ", response.url) + + # article_list = response.xpath('//div[@class="detail"]') + # for article in article_list: + # title = article.xpath("string(.//a)").extract_first() + # print(title) + + +if __name__ == "__main__": + XueQiuSpider(thread_count=1).start() diff --git a/tests/test_rander2.py b/tests/test_rander2.py new file mode 100644 index 00000000..b65bb62b --- /dev/null +++ b/tests/test_rander2.py @@ -0,0 +1,26 @@ +import feapder + + +class XueQiuSpider(feapder.Spider): + __custom_setting__ = dict( + REDISDB_IP_PORTS="localhost:6379", REDISDB_USER_PASS="", REDISDB_DB=0 + ) + + def start_requests(self): + for i in range(10): + yield feapder.Request("https://news.qq.com/#{}".format(i), render=True) + + def parse(self, request, response): + print(response.cookies.get_dict()) + print("response.url ", response.url) + + article_list = response.xpath('//div[@class="detail"]') + for article in article_list: + title = article.xpath("string(.//a)").extract_first() + print(title) + + +if __name__ == "__main__": + XueQiuSpider( + thread_count=10, redis_key="feapter:test_rander", delete_keys=True + ).start() diff --git a/tests/test_rander3.py b/tests/test_rander3.py new file mode 100644 index 00000000..8851f598 --- /dev/null +++ b/tests/test_rander3.py @@ -0,0 +1,20 @@ +import time + +import feapder +from feapder.utils.webdriver import WebDriver + + +class TestRender(feapder.AirSpider): + def start_requests(self): + yield feapder.Request("http://www.baidu.com", render=True) + + def parse(self, request, response): + browser: WebDriver = response.browser + browser.find_element_by_id("kw").send_keys("feapder") + browser.find_element_by_id("su").click() + time.sleep(5) + print(browser.page_source) + + +if __name__ == "__main__": + TestRender().start() diff --git a/tests/test_rander_xhr.py b/tests/test_rander_xhr.py new file mode 100644 index 00000000..15fe2da8 --- /dev/null +++ b/tests/test_rander_xhr.py @@ -0,0 +1,52 @@ +import time + +import feapder +from feapder.utils.webdriver import WebDriver + + +class TestRender(feapder.AirSpider): + __custom_setting__ = dict( + WEBDRIVER=dict( + pool_size=1, # 浏览器的数量 + load_images=True, # 是否加载图片 + user_agent=None, # 字符串 或 无参函数，返回值为user_agent + proxy=None, # xxx.xxx.xxx.xxx:xxxx 或 无参函数，返回值为代理地址 + headless=False, # 是否为无头浏览器 + driver_type="CHROME", # CHROME、EDGE、PHANTOMJS、FIREFOX + timeout=30, # 请求超时时间 + window_size=(1024, 800), # 窗口大小 + executable_path=None, # 浏览器路径，默认为默认路径 + render_time=0, # 渲染时长，即打开网页等待指定时间后再获取源码 + custom_argument=["--ignore-certificate-errors"], # 自定义浏览器渲染参数 + xhr_url_regexes=[ + "/ad", + ], # 拦截 http://www.spidertools.cn/spidertools/ad 接口 + ) + ) + + def start_requests(self): + yield feapder.Request("http://www.spidertools.cn", render=True) + + def parse(self, request, response): + browser: WebDriver = response.browser + time.sleep(3) + + # 获取接口数据 文本类型 + ad = browser.xhr_text("/ad") + print(ad) + + # 获取接口数据 转成json，本例因为返回的接口是文本，所以不转了 + # browser.xhr_json("/ad") + + xhr_response = browser.xhr_response("/ad") + print("请求接口", xhr_response.request.url) + # 请求头目前获取的不完整 + print("请求头", xhr_response.request.headers) + print("请求体", xhr_response.request.data) + print("返回头", xhr_response.headers) + print("返回地址", xhr_response.url) + print("返回内容", xhr_response.content) + + +if __name__ == "__main__": + TestRender().start() diff --git a/tests/test_redisdb.py b/tests/test_redisdb.py new file mode 100644 index 00000000..758ab73e --- /dev/null +++ b/tests/test_redisdb.py @@ -0,0 +1,6 @@ +from feapder.db.redisdb import RedisDB +import time +db = RedisDB.from_url("redis://localhost:6379") + +# db.clear("test") +db.zincrby("test", 1.0, "a") diff --git a/tests/test_request.py b/tests/test_request.py index 58759932..15626457 100644 --- a/tests/test_request.py +++ b/tests/test_request.py @@ -8,15 +8,39 @@ @email: boris_liu@foxmail.com """ -from feapder import Request +from feapder import Request, Response -request = Request("https://www.baidu.com", data={}, params=None) -response = request.get_response() -print(response) -print(response.xpath("//a/@href")) -print(response.css("a::attr(href)")) -print(response.css("a::attr(href)").extract_first()) +def test_selector(): + request = Request("https://www.baidu.com?a=1&b=2", data={}, params=None) + response = request.get_response() + print(response) -content = response.re(" + + + + + + + + + """ + + resp = Response.from_text(text=text, url="http://feapder.com/#/README") + print(resp.text) + print(resp) + print(resp.xpath("//script")) + +def test_to_dict(): + request = Request("https://www.baidu.com?a=1&b=2", data={"a":1}, params="k=1", callback="test", task_id=1, cookies={"a":1}) + print(request.to_dict) \ No newline at end of file diff --git a/tests/test_spider_params.py b/tests/test_spider_params.py index ea97890b..341eacd1 100644 --- a/tests/test_spider_params.py +++ b/tests/test_spider_params.py @@ -4,7 +4,7 @@ --------- @summary: --------- -@author: liubo +@author: Boris """ import feapder @@ -17,15 +17,12 @@ class TestSpiderParams(feapder.Spider): ) def start_requests(self): - for i in range(100): - print(f"下发任务 {i}") - yield feapder.Request(f"https://www.baidu.com?p={i}") + yield feapder.Request(f"https://www.baidu.com") def parse(self, request, response): print(request.url) if __name__ == "__main__": - spider = TestSpiderParams(redis_key="feapder:test_spider_params", min_task_count=10) - spider.start_monitor_task() - # spider.start() + spider = TestSpiderParams(redis_key="feapder:test_spider_params") + spider.start() diff --git a/tests/test_task.py b/tests/test_task.py new file mode 100644 index 00000000..1b92c0af --- /dev/null +++ b/tests/test_task.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +""" +Created on 2021/4/8 1:06 下午 +--------- +@summary: +--------- +@author: Boris +@email: boris_liu@foxmail.com +""" + +from feapder.utils.perfect_dict import PerfectDict + + +task_key = ["id", "url"] +task = [1, "http://www.badu.com"] +task = PerfectDict(_dict=dict(zip(task_key, task)), _values=task) + +task = PerfectDict(id=1, url="http://www.badu.com") +task = PerfectDict({"id":"1", "url":"http://www.badu.com"}) + +print(task) +id, url = task +print(id, url) +print(task[0], task[1]) +print(task.id, task.url) +print(task["id"], task["url"]) +print(task.get("id"), task.get("url")) diff --git a/tests/test_template/test_spider.py b/tests/test_template/test_spider.py new file mode 100644 index 00000000..c46136d8 --- /dev/null +++ b/tests/test_template/test_spider.py @@ -0,0 +1,79 @@ +# -*- coding: utf-8 -*- +""" +Created on 2022-08-04 17:58:45 +--------- +@summary: +--------- +@author: Boris +""" + +import feapder +from feapder import ArgumentParser + + +class TestSpider(feapder.TaskSpider): + # 自定义数据库，若项目中有setting.py文件，此自定义可删除 + __custom_setting__ = dict( + REDISDB_IP_PORTS="localhost:6379", + REDISDB_USER_PASS="", + REDISDB_DB=0, + MYSQL_IP="localhost", + MYSQL_PORT=3306, + MYSQL_DB="", + MYSQL_USER_NAME="", + MYSQL_USER_PASS="", + ) + + def start_requests(self, task): + task_id = task.id + url = task.url + yield feapder.Request(url, task_id=task_id) + + def parse(self, request, response): + # 提取网站title + print(response.xpath("//title/text()").extract_first()) + # 提取网站描述 + print(response.xpath("//meta[@name='description']/@content").extract_first()) + print("网站地址: ", response.url) + + # mysql 需要更新任务状态为做完 即 state=1 + yield self.update_task_batch(request.task_id) + + +if __name__ == "__main__": + # 用mysql做任务表，需要先建好任务任务表 + spider = TestSpider( + redis_key="xxx:xxx", # 分布式爬虫调度信息存储位置 + task_table="", # mysql中的任务表 + task_keys=["id", "url"], # 需要获取任务表里的字段名，可添加多个 + task_state="state", # mysql中任务状态字段 + ) + + # 用redis做任务表 + # spider = TestSpider( + # redis_key="xxx:xxxx", # 分布式爬虫调度信息存储位置 + # task_table="", # 任务表名 + # task_table_type="redis", # 任务表类型为redis + # ) + + parser = ArgumentParser(description="TestSpider爬虫") + + parser.add_argument( + "--start_master", + action="store_true", + help="添加任务", + function=spider.start_monitor_task, + ) + parser.add_argument( + "--start_worker", action="store_true", help="启动爬虫", function=spider.start + ) + + parser.start() + + # 直接启动 + # spider.start() # 启动爬虫 + # spider.start_monitor_task() # 添加任务 + + # 通过命令行启动 + # python test_spider.py --start_master # 添加任务 + # python test_spider.py --start_worker # 启动爬虫 \ No newline at end of file diff --git a/tests/test_tools.py b/tests/test_tools.py new file mode 100644 index 00000000..ae4ec752 --- /dev/null +++ b/tests/test_tools.py @@ -0,0 +1,25 @@ +from feapder.utils import tools +from datetime import datetime + + +date = tools.format_time("昨天3:10") +print(date) + +print(tools.format_date("2017年4月17日 3时27分12秒")) + +date = tools.format_time("昨天") +print(date) + +date = tools.format_time("2021-11-05 14:18:10") +print(date) + +date = tools.format_time("1 年前") +print(date) + + +class C: + pass + + +data = {"date": datetime.now(), "c": C()} +print(tools.dumps_json(data)) diff --git a/tests/test_webdriver.py b/tests/test_webdriver.py new file mode 100644 index 00000000..f7b1c0a2 --- /dev/null +++ b/tests/test_webdriver.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- +""" +Created on 2021/3/18 7:05 下午 +--------- +@summary: +--------- +@author: Boris +@email: boris_liu@foxmail.com +""" +from feapder.utils.webdriver import WebDriverPool, WebDriver +import threading + + +def test_webdirver_pool(): + + webdriver_pool = WebDriverPool( + pool_size=2, load_images=False, driver_type=WebDriver.FIREFOX, timeout=30 + ) + + def request(): + try: + browser = webdriver_pool.get() + browser.get("https://baidu.com") + print(browser.title) + webdriver_pool.put(browser) + except: + print("失败") + + for i in range(5): + threading.Thread(target=request).start() + + +def test_webdriver(): + with WebDriver( + load_images=True, driver_type=WebDriver.CHROME, timeout=30 + ) as browser: + browser.get("https://httpbin.org/get") + html = browser.page_source + print(html) + print(browser.user_agent) + + import time + time.sleep(1000) + +test_webdriver() \ No newline at end of file diff --git a/tests/user_pool/test_gold_user_pool.py b/tests/user_pool/test_gold_user_pool.py new file mode 100644 index 00000000..0cf74bd2 --- /dev/null +++ b/tests/user_pool/test_gold_user_pool.py @@ -0,0 +1,95 @@ +# -*- coding: utf-8 -*- +""" +Created on 2021/9/13 2:33 下午 +--------- +@summary: +--------- +@author: Boris +@email: boris_liu@foxmail.com +""" + +import unittest + +from feapder.network.user_pool import GoldUser +from feapder.network.user_pool import GoldUserPool + + +class TestUserPool(unittest.TestCase): + def setUp(self) -> None: + users = [ + GoldUser( + username="zhangsan", + password="1234", + max_use_times=10, + use_interval=5, + ), + GoldUser( + username="lisi", + password="1234", + max_use_times=10, + use_interval=5, + login_interval=50, + ), + ] + + class CustomGoldUserPool(GoldUserPool): + def login(self, user: GoldUser) -> GoldUser: + # 此处为假数据，正常需通过登录网站获取cookie + username = user.username + password = user.password + + # 登录获取cookie + cookie = "zzzz" + user.cookies = cookie + + return user + + self.user_pool = CustomGoldUserPool( + "test:user_pool", + users=users, + keep_alive=True, + ) + + def test_run(self): + self.user_pool.run() + + def test_get_user(self): + user = self.user_pool.get_user() + print(user) + + user = self.user_pool.get_user(username="zhangsan") + print(user) + + def test_del_user(self): + self.user_pool.del_user("lisi") + + def test_delay_user(self): + user = self.user_pool.get_user(username="lisi") + print(user) + self.user_pool.delay_use("lisi", 60) + user = self.user_pool.get_user(username="lisi") + print(user) + + def test_exclusive(self): + """ + 测试独占 + """ + # 用户lisi被test_spider爬虫独占 + user = self.user_pool.get_user( + username="lisi", used_for_spider_name="test_spider" + ) + print(user) + + # test_spider爬虫可以正常使用 + user = self.user_pool.get_user( + username="lisi", used_for_spider_name="test_spider" + ) + print(user) + + # 其他的爬虫需要在独占的间隔后使用 + user = self.user_pool.get_user(username="lisi") + print(user) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/user_pool/test_guest_user_pool.py b/tests/user_pool/test_guest_user_pool.py new file mode 100644 index 00000000..acc14baf --- /dev/null +++ b/tests/user_pool/test_guest_user_pool.py @@ -0,0 +1,76 @@ +# -*- coding: utf-8 -*- +""" +Created on 2021/9/13 2:33 下午 +--------- +@summary: +--------- +@author: Boris +@email: boris_liu@foxmail.com +""" + +import time +import unittest +from typing import Optional + +from feapder.network.user_pool import GuestUser +from feapder.network.user_pool import GuestUserPool + + +class TestUserPool(unittest.TestCase): + def setUp(self) -> None: + # 默认的用户池，使用webdriver访问page_url生产cookie + self.user_pool = GuestUserPool( + "test:user_pool", page_url="https://www.baidu.com" + ) + + # 自定义生产cookie的方法 + class CustomGuestUserPool(GuestUserPool): + def login(self) -> Optional[GuestUser]: + # 此处为假数据，正常需通过网站获取cookie + user = GuestUser( + user_agent="xxx", + proxies="yyy", + cookies={"some_key": "some_value{}".format(time.time())}, + ) + return user + + self.custom_user_pool = CustomGuestUserPool( + "test:custom_user_pool", min_users=10, keep_alive=True + ) + + def test_get_user(self): + """ + 测试直接获取游客用户 + Returns: + + """ + user = self.custom_user_pool.get_user(block=True) + print("取到user：", user) + print("cookie：", user.cookies) + print("user_agent：", user.user_agent) + print("proxies：", user.proxies) + + def test_del_user(self): + user = GuestUser( + **{ + "user_id": "9f1654ba654e12adfea548eae89a8f6f", + "user_agent": "xxx", + "proxies": "yyy", + "cookies": {"some_key": "some_value1640006728.908013"}, + } + ) + print(user.user_id) + self.custom_user_pool.del_user(user.user_id) + + def test_keep_alive(self): + """ + 测试生产游客用户，面对需要大量cookie，需要单独起个进程维护cookie的场景 + Returns: + + """ + + self.custom_user_pool.run() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/user_pool/test_normal_user_pool.py b/tests/user_pool/test_normal_user_pool.py new file mode 100644 index 00000000..9ec44f61 --- /dev/null +++ b/tests/user_pool/test_normal_user_pool.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- +""" +Created on 2021/9/13 2:33 下午 +--------- +@summary: +--------- +@author: Boris +@email: boris_liu@foxmail.com +""" + +import unittest + +from feapder.network.user_pool import NormalUser +from feapder.network.user_pool import NormalUserPool + + +class TestUserPool(unittest.TestCase): + def setUp(self) -> None: + class CustomNormalUserPool(NormalUserPool): + def login(self, user: NormalUser) -> NormalUser: + # 此处为假数据，正常需通过登录网站获取cookie + username = user.username + password = user.password + + # 登录获取cookie + cookie = "xxx" + user.cookies = cookie + + return user + + self.user_pool = CustomNormalUserPool( + "test:user_pool", + table_userbase="test_userbase", + login_retry_times=0, + keep_alive=True, + ) + + def test_get_user(self): + user = self.user_pool.get_user() + print("取到user：", user) + print("cookie：", user.cookies) + print("user_agent：", user.user_agent) + print("proxies：", user.proxies) + + def test_del_user(self): + self.user_pool.del_user(1) + + def test_tag_user_locked(self): + self.user_pool.tag_user_locked(2) + + def test_keep_alive(self): + self.user_pool.run() + + +if __name__ == "__main__": + unittest.main()