diff --git a/.gitignore b/.gitignore index e0b469d..020ee65 100644 --- a/.gitignore +++ b/.gitignore @@ -1,315 +1,320 @@ -.idea -### JetBrains template -# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider -# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 -weibo/settings.py -# User-specific stuff -.idea/**/workspace.xml -.idea/**/tasks.xml -.idea/**/usage.statistics.xml -.idea/**/dictionaries -.idea/**/shelf - -# AWS User-specific -.idea/**/aws.xml - -# Generated files -.idea/**/contentModel.xml - -# Sensitive or high-churn files -.idea/**/dataSources/ -.idea/**/dataSources.ids -.idea/**/dataSources.local.xml -.idea/**/sqlDataSources.xml -.idea/**/dynamic.xml -.idea/**/uiDesigner.xml -.idea/**/dbnavigator.xml - -# Gradle -.idea/**/gradle.xml -.idea/**/libraries - -# Gradle and Maven with auto-import -# When using Gradle or Maven with auto-import, you should exclude module files, -# since they will be recreated, and may cause churn. Uncomment if using -# auto-import. -# .idea/artifacts -# .idea/compiler.xml -# .idea/jarRepositories.xml -# .idea/modules.xml -# .idea/*.iml -# .idea/modules -# *.iml -# *.ipr - -# CMake -cmake-build-*/ - -# Mongo Explorer plugin -.idea/**/mongoSettings.xml - -# File-based project format -*.iws - -# IntelliJ -out/ - -# mpeltonen/sbt-idea plugin -.idea_modules/ - -# JIRA plugin -atlassian-ide-plugin.xml - -# Cursive Clojure plugin -.idea/replstate.xml - -# SonarLint plugin -.idea/sonarlint/ - -# Crashlytics plugin (for Android Studio and IntelliJ) -com_crashlytics_export_strings.xml -crashlytics.properties -crashlytics-build.properties -fabric.properties - -# Editor-based Rest Client -.idea/httpRequests - -# Android studio 3.1+ serialized cache file -.idea/caches/build_file_checksums.ser - -### Linux template -*~ - -# temporary files which can be created if a process still has a handle open of a deleted file -.fuse_hidden* - -# KDE directory preferences -.directory - -# Linux trash folder which might appear on any partition or disk -.Trash-* - -# .nfs files are created when an open file is removed but is still being accessed -.nfs* - -### Windows template -# Windows thumbnail cache files -Thumbs.db -Thumbs.db:encryptable -ehthumbs.db -ehthumbs_vista.db - -# Dump file -*.stackdump - -# Folder config file -[Dd]esktop.ini - -# Recycle Bin used on file shares -$RECYCLE.BIN/ - -# Windows Installer files -*.cab -*.msi -*.msix -*.msm -*.msp - -# Windows shortcuts -*.lnk - -### macOS template -# General -.DS_Store -.AppleDouble -.LSOverride - -# Icon must end with two \r -Icon - -# Thumbnails -._* - -# Files that might appear in the root of a volume -.DocumentRevisions-V100 -.fseventsd -.Spotlight-V100 -.TemporaryItems -.Trashes -.VolumeIcon.icns -.com.apple.timemachine.donotpresent - -# Directories potentially created on remote AFP share -.AppleDB -.AppleDesktop -Network Trash Folder -Temporary Items -.apdisk - -### Python template -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ -cover/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy -crawls/ -结果文件/ - -# Sphinx documentation -docs/_build/ - -# PyBuilder -.pybuilder/ -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -# For a library or package, you might want to ignore these files since the code is -# intended to run in multiple environments; otherwise, check them in: -# .python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# poetry -# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. -# This is especially recommended for binary packages to ensure reproducibility, and is more -# commonly ignored for libraries. -# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control -#poetry.lock - -# pdm -# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. -#pdm.lock -# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it -# in version control. -# https://pdm.fming.dev/#use-with-ide -.pdm.toml - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ - -# pytype static type analyzer -.pytype/ - -# Cython debug symbols -cython_debug/ - -# PyCharm -# JetBrains specific template is maintained in a separate JetBrains.gitignore that can -# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore -# and can be added to the global gitignore or merged into this file. For a more nuclear -# option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ - -# - +.idea +### JetBrains template +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 +weibo/settings.py +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf + +# AWS User-specific +.idea/**/aws.xml + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# Gradle and Maven with auto-import +# When using Gradle or Maven with auto-import, you should exclude module files, +# since they will be recreated, and may cause churn. Uncomment if using +# auto-import. +# .idea/artifacts +# .idea/compiler.xml +# .idea/jarRepositories.xml +# .idea/modules.xml +# .idea/*.iml +# .idea/modules +# *.iml +# *.ipr + +# CMake +cmake-build-*/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# SonarLint plugin +.idea/sonarlint/ + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + +# Android studio 3.1+ serialized cache file +.idea/caches/build_file_checksums.ser + +### Linux template +*~ + +# temporary files which can be created if a process still has a handle open of a deleted file +.fuse_hidden* + +# KDE directory preferences +.directory + +# Linux trash folder which might appear on any partition or disk +.Trash-* + +# .nfs files are created when an open file is removed but is still being accessed +.nfs* + +### Windows template +# Windows thumbnail cache files +Thumbs.db +Thumbs.db:encryptable +ehthumbs.db +ehthumbs_vista.db + +# Dump file +*.stackdump + +# Folder config file +[Dd]esktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ + +# Windows Installer files +*.cab +*.msi +*.msix +*.msm +*.msp + +# Windows shortcuts +*.lnk + +### macOS template +# General +.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +### Python template +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy +crawls/ +结果文件/ + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +# + + +# 项目本地数据/垃圾/记忆(本地保留,不入库) +1113/ +.workbuddy/ +NUL diff --git a/README.md b/README.md index 8b2e035..4d20a06 100644 --- a/README.md +++ b/README.md @@ -1,129 +1,165 @@ -## 功能 -连续获取一个或多个**微博关键词搜索**结果，并将结果写入文件（可选）、数据库（可选）等。所谓微博关键词搜索即：**搜索正文中包含指定关键词的微博**，可以指定搜索的时间范围。
-举个栗子，比如你可以搜索包含关键词“迪丽热巴”且发布日期在2020-03-01和2020-03-16之间的微博。搜索结果数量巨大，对于非常热门的关键词，在一天的指定时间范围，可以获得**1000万**以上的搜索结果。注意这里的一天指的是时间筛选范围，具体多长时间将这1000万微博下载到本地还要看获取的速度。1000万只是一天时间范围可获取的微博数量，如果想获取更多微博，可以加大时间范围，比如10天，最多可以获得1000万X10=1亿条搜索结果，当然你也可以再加大时间范围。对于大多数关键词，微博一天产生的相关搜索结果应该低于1000万，因此可以说**本程序可以获取指定关键词的全部或近似全部的搜索结果**。本程序可以获得几乎全部的微博信息，如微博正文、发布者等，详情见[输出](#输出)部分。支持输出多种文件类型，具体如下： -- 写入**csv文件**（默认） -- 写入**MySQL数据库**（可选） -- 写入**MongoDB数据库**（可选） -- 写入**Sqlite数据库**（可选，无需外部安装，相比MySQL和MongoDB更方便） -- 下载微博中的**图片**（可选） -- 下载微博中的**视频**（可选） - -## 输出 -- 微博id：微博的id，为一串数字形式 -- 微博bid：微博的bid -- 微博内容：微博正文 -- 头条文章url：微博中头条文章的url，若某微博中不存在头条文章，则该值为'' -- 原始图片url：原创微博图片和转发微博转发理由中图片的url，若某条微博存在多张图片，则每个url以英文逗号分隔，若没有图片则值为'' -- 视频url: 微博中的视频url和Live Photo中的视频url，若某条微博存在多个视频，则每个url以英文分号分隔，若没有视频则值为'' -- 微博发布位置：位置微博中的发布位置 -- 微博发布时间：微博发布时的时间，精确到天 -- 点赞数：微博被赞的数量 -- 转发数：微博被转发的数量 -- 评论数：微博被评论的数量 -- 微博发布工具：微博的发布工具，如iPhone客户端、HUAWEI Mate 20 Pro等，若没有则值为'' -- 话题：微博话题，即两个#中的内容，若存在多个话题，每个url以英文逗号分隔，若没有则值为'' -- @用户：微博@的用户，若存在多个@用户，每个url以英文逗号分隔，若没有则值为'' -- 原始微博id：为转发微博所特有，是转发微博中那条被转发微博的id，那条被转发的微博也会存储，字段和原创微博一样，只是它的本字段为空 -- 结果文件：保存在当前目录“结果文件”文件夹下以关键词为名的文件夹里 -- 微博图片：微博中的图片，保存在以关键词为名的文件夹下的images文件夹里 -- 微博视频：微博中的视频，保存在以关键词为名的文件夹下的videos文件夹里 -- user_authentication：微博用户类型，值分别是`蓝v`，`黄v`，`红v`，`金v`和`普通用户` - -## 使用说明 -本程序的所有配置都在setting.py文件中完成，该文件位于“weibo-search\weibo\settings.py”。 -### 1.下载脚本 -```bash -$ git clone https://github.com/dataabc/weibo-search.git -``` -### 2.安装Scrapy -本程序依赖Scrapy，要想运行程序，需要安装Scrapy。如果系统中没有安装Scrapy，请根据自己的系统安装Scrapy，以Ubuntu为例，可以使用如下命令： -```bash -$ pip install scrapy -``` -### 3.安装依赖 -```bash -$ pip install -r requirements.txt -``` - -### 4.设置cookie -DEFAULT_REQUEST_HEADERS中的cookie是我们需要填的值，如何获取cookie详见[如何获取cookie](#如何获取cookie)，获取后将"your cookie"替换成真实的cookie即可。 -### 5.设置搜索关键词 -修改setting.py文件夹中的KEYWORD_LIST参数。 -如果你想搜索一个关键词，如“迪丽热巴”： -``` -KEYWORD_LIST = ['迪丽热巴'] -``` -如果你想分别搜索多个关键词，如想要分别获得“迪丽热巴”和“杨幂”的搜索结果： -``` -KEYWORD_LIST = ['迪丽热巴', '杨幂'] -``` -如果你想搜索同时包含多个关键词的微博，如同时包含“迪丽热巴”和“杨幂”微博的搜索结果： -``` -KEYWORD_LIST = ['迪丽热巴 杨幂'] -``` -如果你想搜索微博话题，即包含#的内容，如“#迪丽热巴#”： -``` -KEYWORD_LIST = ['#迪丽热巴#'] -``` -也可以把关键词写进txt文件里，然后将txt文件路径赋值给KEYWORD_LIST，如： -``` -KEYWORD_LIST = 'keyword_list.txt' -``` -txt文件中每个关键词占一行。 -### 6.设置搜索时间范围 -START_DATE代表搜索的起始日期，END_DATE代表搜索的结束日期，值为“yyyy-mm-dd”形式，程序会搜索包含关键词且发布时间在起始日期和结束日期之间的微博（包含边界）。比如我想筛选发布时间在2020-06-01到2020-06-02这两天的微博： -``` -START_DATE = '2020-06-01' -END_DATE = '2020-06-02' -``` -### 7.设置FURTHER_THRESHOLD（可选） -FURTHER_THRESHOLD是程序是否进一步搜索的阈值。一般情况下，如果在某个搜索条件下，搜索结果很多，则搜索结果应该有50页微博，多于50页不显示。当总页数等于50时，程序认为搜索结果可能没有显示完全，所以会继续细分。比如，若当前是按天搜索的，程序会把当前的1个搜索分成24个搜索，每个搜索条件粒度是小时。这样就能获取在天粒度下无法获取完全的微博。同理，如果小时粒度下总页数仍然是50，会继续细分，以此类推。然而，有一些关键词，搜索结果即便很多，也只显示40多页。所以此时如果FURTHER_THRESHOLD是50，程序会认为只有这么多微博，不再继续细分，导致很多微博没有获取。因此为了获取更多微博，FURTHER_THRESHOLD应该是小于50的数字。但是如果设置的特别小，如1，这样即便结果真的只有几页，程序也会细分，这些没有必要的细分会使程序速度降低。因此，建议**FURTHER_THRESHOLD的值设置在40与46之间**： -``` -FURTHER_THRESHOLD = 46 -``` -### 8.设置结果保存类型（可选） -ITEM_PIPELINES是我们可选的结果保存类型，第一个代表去重，第二个代表写入csv文件，第三个代表写入MySQL数据库，第四个代表写入MongDB数据库，第五个代表下载图片，第六个代表下载视频。后面的数字代表执行的顺序，数字越小优先级越高。如果你只要写入部分类型，可以把不需要的类型用“#”注释掉，以节省资源；如果你想写入数据库，需要在setting.py填写相关数据库的配置。 -### 9.设置等待时间（可选） -DOWNLOAD_DELAY代表访问完一个页面再访问下一个时需要等待的时间，默认为10秒。如我想设置等待15秒左右，可以修改setting.py文件的DOWNLOAD_DELAY参数： -``` -DOWNLOAD_DELAY = 15 -``` -### 10.设置微博类型（可选） -WEIBO_TYPE筛选要搜索的微博类型，0代表搜索全部微博，1代表搜索全部原创微博，2代表热门微博，3代表关注人微博，4代表认证用户微博，5代表媒体微博，6代表观点微博。比如我想要搜索全部原创微博，修改setting.py文件的WEIBO_TYPE参数： -``` -WEIBO_TYPE = 1 -``` -### 11.设置包含内容（可选） -CONTAIN_TYPE筛选结果微博中必需包含的内容，0代表不筛选，获取全部微博，1代表搜索包含图片的微博，2代表包含视频的微博，3代表包含音乐的微博，4代表包含短链接的微博。比如我想筛选包含图片的微博，修改setting.py文件的CONTAIN_TYPE参数： -``` -CONTAIN_TYPE = 1 -``` -### 12.筛选微博发布地区（可选） -REGION筛选微博的发布地区，精确到省或直辖市，值不应包含“省”或“市”等字，如想筛选北京市的微博请用“北京”而不是“北京市”，想要筛选安徽省的微博请用“安徽”而不是“安徽省”，可以写多个地区，具体支持的地名见region.py文件，注意只支持省或直辖市的名字，省下面的市名及直辖市下面的区县名不支持，不筛选请用”全部“。比如我想要筛选发布地在山东省的微博： -``` -REGION = ['山东'] -``` -### 13.配置数据库（可选） -MONGO_URI是MongoDB数据库的配置；
-MYSQL开头的是MySQL数据库的配置。 -### 14.运行程序 -```bash -$ scrapy crawl search -s JOBDIR=crawls/search -``` -其实只运行“scrapy crawl search”也可以，只是上述方式在结束时可以保存进度，下次运行时会在程序上次的地方继续获取。注意，如果想要保存进度，请使用“Ctrl + C”**一次**，注意是**一次**。按下“Ctrl + C”一次后，程序会继续运行一会，主要用来保存获取的数据、保存进度等操作，请耐心等待。下次再运行时，只要再运行上面的指令就可以恢复上次的进度。如果再次运行没有结果，可能是进度没有正确保存，可以先删除crawls文件夹内的进度文件，再运行上述命令。 -## 如何获取cookie -1. 用Chrome打开 https://weibo.com/ -2. 点击"立即登录", 完成私信验证或手机验证码验证, 进入新版微博. 如下图所示: -... -3. 按F12打开开发者工具, 在开发者工具的 Network->Name->weibo.cn->Headers->Request Headers, 找到"Cookie:"后的值, 这就是我们要找的cookie值, 复制即可, 如图所示: -... - -> ## 兼容性说明: 获取旧版微博的Cookie -> 1.用Chrome打开；
-> 2.输入微博的用户名、密码，登录，如图所示： -> ![](https://github.com/dataabc/media/blob/master/weiboSpider/images/cookie1.png) -> 登录成功后会跳转到;
-> 3.按F12键打开Chrome开发者工具，在地址栏输入并跳转到，跳转后会显示如下类似界面: -> ![](https://github.com/dataabc/media/blob/master/weiboSpider/images/cookie2.png) -> 4.依此点击Chrome开发者工具中的Network->Name中的weibo.cn->Headers->Request Headers，"Cookie:"后的值即为我们要找的cookie值，复制即可，如图所示： -> ![](https://github.com/dataabc/media/blob/master/weiboSpider/images/cookie3.png) +## 功能 +连续获取一个或多个**微博关键词搜索**结果，并将结果写入文件（可选）、数据库（可选）等。所谓微博关键词搜索即：**搜索正文中包含指定关键词的微博**，可以指定搜索的时间范围。
+举个栗子，比如你可以搜索包含关键词“迪丽热巴”且发布日期在2020-03-01和2020-03-16之间的微博。搜索结果数量巨大，对于非常热门的关键词，在一天的指定时间范围，可以获得**1000万**以上的搜索结果。注意这里的一天指的是时间筛选范围，具体多长时间将这1000万微博下载到本地还要看获取的速度。1000万只是一天时间范围可获取的微博数量，如果想获取更多微博，可以加大时间范围，比如10天，最多可以获得1000万X10=1亿条搜索结果，当然你也可以再加大时间范围。对于大多数关键词，微博一天产生的相关搜索结果应该低于1000万，因此可以说**本程序可以获取指定关键词的全部或近似全部的搜索结果**。本程序可以获得几乎全部的微博信息，如微博正文、发布者等，详情见[输出](#输出)部分。支持输出多种文件类型，具体如下： +- 写入**csv文件**（默认） +- 写入**MySQL数据库**（可选） +- 写入**MongoDB数据库**（可选） +- 写入**Sqlite数据库**（可选，无需外部安装，相比MySQL和MongoDB更方便） +- 下载微博中的**图片**（可选） +- 下载微博中的**视频**（可选） + +## 输出 +- 微博id：微博的id，为一串数字形式 +- 微博bid：微博的bid +- 微博内容：微博正文 +- 头条文章url：微博中头条文章的url，若某微博中不存在头条文章，则该值为'' +- 原始图片url：原创微博图片和转发微博转发理由中图片的url，若某条微博存在多张图片，则每个url以英文逗号分隔，若没有图片则值为'' +- 视频url: 微博中的视频url和Live Photo中的视频url，若某条微博存在多个视频，则每个url以英文分号分隔，若没有视频则值为'' +- 微博发布位置：位置微博中的发布位置 +- 微博发布时间：微博发布时的时间，精确到天 +- 点赞数：微博被赞的数量 +- 转发数：微博被转发的数量 +- 评论数：微博被评论的数量 +- 微博发布工具：微博的发布工具，如iPhone客户端、HUAWEI Mate 20 Pro等，若没有则值为'' +- 话题：微博话题，即两个#中的内容，若存在多个话题，每个url以英文逗号分隔，若没有则值为'' +- @用户：微博@的用户，若存在多个@用户，每个url以英文逗号分隔，若没有则值为'' +- 原始微博id：为转发微博所特有，是转发微博中那条被转发微博的id，那条被转发的微博也会存储，字段和原创微博一样，只是它的本字段为空 +- 结果文件：保存在当前目录“结果文件”文件夹下以关键词为名的文件夹里 +- 微博图片：微博中的图片，保存在以关键词为名的文件夹下的images文件夹里 +- 微博视频：微博中的视频，保存在以关键词为名的文件夹下的videos文件夹里 +- user_authentication：微博用户类型，值分别是`蓝v`，`黄v`，`红v`，`金v`和`普通用户` +- vip_type：会员类型，值分别为`超级会员`、`会员`、`非会员` +- vip_level：会员等级，整数 +- ip：微博发布者的IP属地（可选，需设置 `FETCH_IP=1` 开启，默认不抓取） + +## 使用说明 +本程序的所有配置都在 settings.py 文件中完成，该文件位于"weibo-search\weibo\settings.py"。本程序还支持通过 `WEIBO_*` 环境变量覆盖几乎所有配置项，推荐用环境变量方式配置，无需修改源码。 +### 1.下载脚本 +```bash +$ git clone https://github.com/dataabc/weibo-search.git +``` +### 2.安装Scrapy +本程序依赖Scrapy，要想运行程序，需要安装Scrapy。如果系统中没有安装Scrapy，请根据自己的系统安装Scrapy，以Ubuntu为例，可以使用如下命令： +```bash +$ pip install scrapy +``` +### 3.安装依赖 +```bash +$ pip install -r requirements.txt +``` + +### 4.设置cookie +Cookie 有两种配置方式，任选其一：(1) 设置环境变量 `WEIBO_COOKIE`，例如 `WEIBO_COOKIE="你的cookie值"`；(2) 直接修改 `settings.py` 中 `DEFAULT_REQUEST_HEADERS` 的 `cookie` 字段。如何获取 cookie 详见[如何获取cookie](#如何获取cookie)。注意：Cookie 为空时程序启动会直接报错退出，请务必填入已登录的真实 cookie。 +### 5.设置搜索关键词 +修改 settings.py 文件中的 KEYWORD_LIST 参数（或通过环境变量 `WEIBO_KEYWORDS` 设置）。 +如果你想搜索一个关键词，如“迪丽热巴”： +``` +KEYWORD_LIST = ['迪丽热巴'] +``` +如果你想分别搜索多个关键词，如想要分别获得“迪丽热巴”和“杨幂”的搜索结果： +``` +KEYWORD_LIST = ['迪丽热巴', '杨幂'] +``` +如果你想搜索同时包含多个关键词的微博，如同时包含“迪丽热巴”和“杨幂”微博的搜索结果： +``` +KEYWORD_LIST = ['迪丽热巴 杨幂'] +``` +如果你想搜索微博话题，即包含#的内容，如“#迪丽热巴#”： +``` +KEYWORD_LIST = ['#迪丽热巴#'] +``` +也可以把关键词写进txt文件里，然后将txt文件路径赋值给KEYWORD_LIST，如： +``` +KEYWORD_LIST = 'keyword_list.txt' +``` +txt文件中每个关键词占一行。 +### 6.设置搜索时间范围 +START_DATE代表搜索的起始日期，END_DATE代表搜索的结束日期，值为“yyyy-mm-dd”形式，程序会搜索包含关键词且发布时间在起始日期和结束日期之间的微博（包含边界）。也可用环境变量 `WEIBO_START_DATE` / `WEIBO_END_DATE` 设置。比如我想筛选发布时间在2020-06-01到2020-06-02这两天的微博： +``` +START_DATE = '2026-07-20' +END_DATE = '2026-07-20' +``` +### 7.设置FURTHER_THRESHOLD（可选） +FURTHER_THRESHOLD是程序是否进一步搜索的阈值。一般情况下，如果在某个搜索条件下，搜索结果很多，则搜索结果应该有50页微博，多于50页不显示。当总页数等于50时，程序认为搜索结果可能没有显示完全，所以会继续细分。比如，若当前是按天搜索的，程序会把当前的1个搜索分成24个搜索，每个搜索条件粒度是小时。这样就能获取在天粒度下无法获取完全的微博。同理，如果小时粒度下总页数仍然是50，会继续细分，以此类推。然而，有一些关键词，搜索结果即便很多，也只显示40多页。所以此时如果FURTHER_THRESHOLD是50，程序会认为只有这么多微博，不再继续细分，导致很多微博没有获取。因此为了获取更多微博，FURTHER_THRESHOLD应该是小于50的数字。但是如果设置的特别小，如1，这样即便结果真的只有几页，程序也会细分，这些没有必要的细分会使程序速度降低。因此，建议**FURTHER_THRESHOLD的值设置在40与46之间**（也可用环境变量 `WEIBO_FURTHER_THRESHOLD` 设置）： +``` +FURTHER_THRESHOLD = 46 +``` +### 8.设置结果保存类型（可选） +ITEM_PIPELINES是我们可选的结果保存类型，按 Scrapy 优先级数字从低到高依次为：去重(300) → 写入csv文件(301) → 写入MySQL数据库(302) → 写入MongoDB数据库(303) → 下载图片(304) → 下载视频(305) → 写入SQLite数据库(306)。默认已开启去重和 csv 写入，其余类型以"#"注释，按需取消注释即可启用。如果你只要写入部分类型，可以把不需要的类型用“#”注释掉，以节省资源；如果你想写入数据库，需要在 settings.py 填写相关数据库的配置。 +### 9.设置等待时间（可选） +DOWNLOAD_DELAY代表访问完一个页面再访问下一个时需要等待的时间，默认为6.0秒（搜索结果量大时建议保持或增大，以免触发限流）。如我想设置等待15秒左右，可以修改 settings.py 文件的 DOWNLOAD_DELAY 参数，或通过环境变量 `WEIBO_DOWNLOAD_DELAY` 设置： +``` +DOWNLOAD_DELAY = 15 +``` +当前版本默认开启了随机等待、AutoThrottle、限流重试与 BAN 检测，以便在稳定性、速度和账号安全之间取得平衡。推荐先使用默认值运行；如果希望稍快，可以小幅调低环境变量 `WEIBO_DOWNLOAD_DELAY` 或小幅调高 `WEIBO_CONCURRENT_REQUESTS_PER_DOMAIN`，并观察是否出现 403、429、验证码或登录页。如果出现这些情况，应立即调慢速度或暂停任务。 + +本程序几乎所有配置都支持 `WEIBO_*` 环境变量覆盖，常用如下： + +| 环境变量 | 对应 settings.py 参数 | 默认值 | 说明 | +|------|------|------|------| +| `WEIBO_COOKIE` | DEFAULT_REQUEST_HEADERS.cookie | 空 | 登录 Cookie（必填） | +| `WEIBO_KEYWORDS` | KEYWORD_LIST | `['迪丽热巴']` | 搜索关键词 | +| `WEIBO_START_DATE` / `WEIBO_END_DATE` | START_DATE / END_DATE | `2020-03-01` | 时间范围 | +| `WEIBO_TYPE` | WEIBO_TYPE | `1` | 微博类型 | +| `WEIBO_CONTAIN_TYPE` | CONTAIN_TYPE | `0` | 包含内容类型 | +| `WEIBO_REGION` | REGION | `['全部']` | 发布地区 | +| `WEIBO_FURTHER_THRESHOLD` | FURTHER_THRESHOLD | `46` | 细分阈值 | +| `WEIBO_LIMIT_RESULT` | LIMIT_RESULT | `0` | 结果数量上限（0=不限） | +| `WEIBO_DOWNLOAD_DELAY` | DOWNLOAD_DELAY | `6.0` | 下载延迟（秒） | +| `WEIBO_CONCURRENT_REQUESTS` | CONCURRENT_REQUESTS | `4` | 全局并发 | +| `WEIBO_CONCURRENT_REQUESTS_PER_DOMAIN` | CONCURRENT_REQUESTS_PER_DOMAIN | `2` | 每域名并发 | +| `WEIBO_RETRY_TIMES` | RETRY_TIMES | `5` | 重试次数 | +| `WEIBO_AUTOTHROTTLE_*` | AUTOTHROTTLE_* | — | 自动限速参数 | +| `WEIBO_FETCH_IP` | FETCH_IP | `0` | 是否抓取 IP 属地（1=开启） | +| `WEIBO_IP_REQUEST_DELAY` | IP_REQUEST_DELAY | `6.0` | IP 属地接口独立请求间隔（秒） | + +环境变量优先级高于 settings.py 中的默认值，可通过 `scrapy crawl search -s KEY=VALUE` 或 shell 导出设置。 + +常用环境变量示例： +```bash +WEIBO_DOWNLOAD_DELAY=4 \ +WEIBO_CONCURRENT_REQUESTS_PER_DOMAIN=2 \ +WEIBO_AUTOTHROTTLE_TARGET_CONCURRENCY=1.0 \ +scrapy crawl search -s JOBDIR=crawls/search +``` +### 10.设置微博类型（可选） +WEIBO_TYPE筛选要搜索的微博类型，0代表搜索全部微博，1代表搜索全部原创微博，2代表热门微博，3代表关注人微博，4代表认证用户微博，5代表媒体微博，6代表观点微博。比如我想要搜索全部原创微博，修改 settings.py 文件的 WEIBO_TYPE 参数（或通过环境变量 `WEIBO_TYPE` 设置）： +``` +WEIBO_TYPE = 1 +``` +### 11.设置包含内容（可选） +CONTAIN_TYPE筛选结果微博中必需包含的内容，0代表不筛选，获取全部微博，1代表搜索包含图片的微博，2代表包含视频的微博，3代表包含音乐的微博，4代表包含短链接的微博。比如我想筛选包含图片的微博，修改 settings.py 文件的 CONTAIN_TYPE 参数（或通过环境变量 `WEIBO_CONTAIN_TYPE` 设置）： +``` +CONTAIN_TYPE = 1 +``` +### 12.筛选微博发布地区（可选） +REGION筛选微博的发布地区，精确到省或直辖市，值不应包含“省”或“市”等字，如想筛选北京市的微博请用“北京”而不是“北京市”，想要筛选安徽省的微博请用“安徽”而不是“安徽省”，可以写多个地区，具体支持的地名见region.py文件，注意只支持省或直辖市的名字，省下面的市名及直辖市下面的区县名不支持，不筛选请用”全部“。比如我想要筛选发布地在山东省的微博（或通过环境变量 `WEIBO_REGION` 设置，如 `WEIBO_REGION='["山东"]'`）： +``` +REGION = ['山东'] +``` +### 13.配置数据库（可选） +如需写入数据库，先取消 settings.py 中 ITEM_PIPELINES 对应行的注释，再填写配置： +- MongoDB：设置 `MONGO_URI`（如 'localhost'） +- MySQL：`MYSQL_HOST`、`MYSQL_PORT`、`MYSQL_USER`、`MYSQL_PASSWORD`、`MYSQL_DATABASE`（程序自动创建名为 weibo 的数据库） +- SQLite：设置 `SQLITE_DATABASE`（如 'weibo.db'），无需安装外部数据库，相比 MySQL/MongoDB 更方便 +### 14.运行程序 +```bash +$ scrapy crawl search -s JOBDIR=crawls/search +``` +其实只运行“scrapy crawl search”也可以，只是上述方式在结束时可以保存进度，下次运行时会在程序上次的地方继续获取。注意，如果想要保存进度，请使用“Ctrl + C”**一次**，注意是**一次**。按下“Ctrl + C”一次后，程序会继续运行一会，主要用来保存获取的数据、保存进度等操作，请耐心等待。下次再运行时，只要再运行上面的指令就可以恢复上次的进度。如果再次运行没有结果，可能是进度没有正确保存，可以先删除crawls文件夹内的进度文件，再运行上述命令。 +## 如何获取cookie +1. 用Chrome打开 https://weibo.com/ +2. 点击"立即登录", 完成私信验证或手机验证码验证, 进入新版微博. 如下图所示: +... +3. 按F12打开开发者工具, 在开发者工具的 Network->Name->weibo.cn->Headers->Request Headers, 找到"Cookie:"后的值, 这就是我们要找的cookie值, 复制即可, 如图所示: +... + +> ## 兼容性说明: 获取旧版微博的Cookie +> 1.用Chrome打开；
+> 2.输入微博的用户名、密码，登录，如图所示： +> ![](https://github.com/dataabc/media/blob/master/weiboSpider/images/cookie1.png) +> 登录成功后会跳转到;
+> 3.按F12键打开Chrome开发者工具，在地址栏输入并跳转到，跳转后会显示如下类似界面: +> ![](https://github.com/dataabc/media/blob/master/weiboSpider/images/cookie2.png) +> 4.依此点击Chrome开发者工具中的Network->Name中的weibo.cn->Headers->Request Headers，"Cookie:"后的值即为我们要找的cookie值，复制即可，如图所示： +> ![](https://github.com/dataabc/media/blob/master/weiboSpider/images/cookie3.png) diff --git a/requirements.txt b/requirements.txt index ad75bc8..8602bd5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -Scrapy>=2.11,<3 -requests>=2.31,<3 -Pillow>=8.1.1 -pytest>=8,<9 +Scrapy>=2.11,<3 +requests>=2.31,<3 +Pillow>=8.1.1 +pytest>=8,<9 diff --git a/scrapy.cfg b/scrapy.cfg index 6b02f75..9bfd59d 100644 --- a/scrapy.cfg +++ b/scrapy.cfg @@ -1,11 +1,11 @@ -# Automatically created by: scrapy startproject -# -# For more information about the [deploy] section see: -# https://scrapyd.readthedocs.io/en/latest/deploy.html - -[settings] -default = weibo.settings - -[deploy] -#url = http://localhost:6800/ -project = weibo +# Automatically created by: scrapy startproject +# +# For more information about the [deploy] section see: +# https://scrapyd.readthedocs.io/en/latest/deploy.html + +[settings] +default = weibo.settings + +[deploy] +#url = http://localhost:6800/ +project = weibo diff --git a/tests/test_search_ip.py b/tests/test_search_ip.py new file mode 100644 index 0000000..279d6d4 --- /dev/null +++ b/tests/test_search_ip.py @@ -0,0 +1,83 @@ +from unittest.mock import Mock, PropertyMock, patch + +from scrapy.settings import Settings + +from weibo.spiders.search import SearchSpider + + +def make_spider(): + settings = Settings({ + 'KEYWORD_LIST': ['广西洪灾'], + 'WEIBO_TYPE': 0, + 'CONTAIN_TYPE': 0, + 'REGION': ['全部'], + 'FETCH_IP': True, + 'IP_REQUEST_DELAY': 0, + 'DEFAULT_REQUEST_HEADERS': { + 'cookie': 'Cookie: test-cookie\r\n', + }, + 'USER_AGENT_LIST': ['test-browser-user-agent'], + }) + return SearchSpider(settings=settings) + + +def test_get_ip_uses_browser_headers_and_parses_region(): + spider = make_spider() + response = Mock(status_code=200) + response.json.return_value = {'region_name': '发布于 广西'} + spider.ip_session.get = Mock(return_value=response) + + assert spider.get_ip('R8wCeyBvq') == '广西' + + _, kwargs = spider.ip_session.get.call_args + assert kwargs['headers']['Cookie'] == 'test-cookie' + assert kwargs['headers']['User-Agent'] == 'test-browser-user-agent' + assert kwargs['headers']['Accept'] == 'application/json, text/plain, */*' + assert kwargs['headers']['Referer'] == ( + 'https://weibo.com/detail/R8wCeyBvq') + assert kwargs['headers']['X-Requested-With'] == 'XMLHttpRequest' + + +def test_get_ip_accepts_region_without_display_prefix(): + spider = make_spider() + response = Mock(status_code=200) + response.json.return_value = {'region_name': '广西'} + spider.ip_session.get = Mock(return_value=response) + + assert spider.get_ip('R8wCeyBvq') == '广西' + + +def test_get_ip_falls_back_to_mobile_endpoint_with_numeric_id(): + spider = make_spider() + web_response = Mock(status_code=200) + web_response.json.return_value = {'id': '123'} + mobile_response = Mock(status_code=200) + mobile_response.json.return_value = { + 'data': { + 'region_name': '发布于 山东', + }, + } + spider.ip_session.get = Mock( + side_effect=[web_response, mobile_response]) + + assert spider.get_ip('R8wCbiAEM', '5194600000000000') == '山东' + + calls = spider.ip_session.get.call_args_list + assert calls[0].args[0].startswith( + 'https://weibo.com/ajax/statuses/show?id=R8wCbiAEM') + assert calls[1].args[0] == ( + 'https://m.weibo.cn/statuses/show?id=5194600000000000') + assert calls[1].kwargs['headers']['MWeibo-Pwa'] == '1' + + +def test_get_ip_caches_http_failure_and_reports_it_once(): + spider = make_spider() + spider.ip_session.get = Mock(return_value=Mock(status_code=403)) + with patch.object(SearchSpider, 'logger', new_callable=PropertyMock) as logger: + logger.return_value.error = Mock() + assert spider.get_ip('first') == '' + assert spider.get_ip('second') == '' + logger.return_value.error.assert_called_once() + + spider.ip_session.get.assert_called_once() + assert spider.ip_failure_counts == {'网页接口HTTP 403': 1} diff --git a/weibo/items.py b/weibo/items.py index a2c6cd8..fd7207d 100644 --- a/weibo/items.py +++ b/weibo/items.py @@ -1,33 +1,33 @@ -# -*- coding: utf-8 -*- - -# Define here the models for your scraped items -# -# See documentation in: -# https://docs.scrapy.org/en/latest/topics/items.html - -import scrapy - - -class WeiboItem(scrapy.Item): - # define the fields for your item here like: - id = scrapy.Field() - bid = scrapy.Field() - user_id = scrapy.Field() - screen_name = scrapy.Field() - text = scrapy.Field() - article_url = scrapy.Field() - location = scrapy.Field() - at_users = scrapy.Field() - topics = scrapy.Field() - reposts_count = scrapy.Field() - comments_count = scrapy.Field() - attitudes_count = scrapy.Field() - created_at = scrapy.Field() - source = scrapy.Field() - pics = scrapy.Field() - video_url = scrapy.Field() - retweet_id = scrapy.Field() - ip = scrapy.Field() - user_authentication = scrapy.Field() - vip_type = scrapy.Field() - vip_level = scrapy.Field() +# -*- coding: utf-8 -*- + +# Define here the models for your scraped items +# +# See documentation in: +# https://docs.scrapy.org/en/latest/topics/items.html + +import scrapy + + +class WeiboItem(scrapy.Item): + # define the fields for your item here like: + id = scrapy.Field() + bid = scrapy.Field() + user_id = scrapy.Field() + screen_name = scrapy.Field() + text = scrapy.Field() + article_url = scrapy.Field() + location = scrapy.Field() + at_users = scrapy.Field() + topics = scrapy.Field() + reposts_count = scrapy.Field() + comments_count = scrapy.Field() + attitudes_count = scrapy.Field() + created_at = scrapy.Field() + source = scrapy.Field() + pics = scrapy.Field() + video_url = scrapy.Field() + retweet_id = scrapy.Field() + ip = scrapy.Field() + user_authentication = scrapy.Field() + vip_type = scrapy.Field() + vip_level = scrapy.Field() diff --git a/weibo/middlewares.py b/weibo/middlewares.py index 1b579ec..8265a46 100644 --- a/weibo/middlewares.py +++ b/weibo/middlewares.py @@ -1,103 +1,117 @@ -# -*- coding: utf-8 -*- - -# Define here the models for your spider middleware -# -# See documentation in: -# https://docs.scrapy.org/en/latest/topics/spider-middleware.html - -from scrapy import signals - - -class WeiboSpiderMiddleware(object): - # Not all methods need to be defined. If a method is not defined, - # scrapy acts as if the spider middleware does not modify the - # passed objects. - - @classmethod - def from_crawler(cls, crawler): - # This method is used by Scrapy to create your spiders. - s = cls() - crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) - return s - - def process_spider_input(self, response, spider): - # Called for each response that goes through the spider - # middleware and into the spider. - - # Should return None or raise an exception. - return None - - def process_spider_output(self, response, result, spider): - # Called with the results returned from the Spider, after - # it has processed the response. - - # Must return an iterable of Request, dict or Item objects. - for i in result: - yield i - - def process_spider_exception(self, response, exception, spider): - # Called when a spider or process_spider_input() method - # (from other spider middleware) raises an exception. - - # Should return either None or an iterable of Request, dict - # or Item objects. - pass - - def process_start_requests(self, start_requests, spider): - # Called with the start requests of the spider, and works - # similarly to the process_spider_output() method, except - # that it doesn’t have a response associated. - - # Must return only requests (not items). - for r in start_requests: - yield r - - def spider_opened(self, spider): - spider.logger.info('Spider opened: %s' % spider.name) - - -class WeiboDownloaderMiddleware(object): - # Not all methods need to be defined. If a method is not defined, - # scrapy acts as if the downloader middleware does not modify the - # passed objects. - - @classmethod - def from_crawler(cls, crawler): - # This method is used by Scrapy to create your spiders. - s = cls() - crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) - return s - - def process_request(self, request, spider): - # Called for each request that goes through the downloader - # middleware. - - # Must either: - # - return None: continue processing this request - # - or return a Response object - # - or return a Request object - # - or raise IgnoreRequest: process_exception() methods of - # installed downloader middleware will be called - return None - - def process_response(self, request, response, spider): - # Called with the response returned from the downloader. - - # Must either; - # - return a Response object - # - return a Request object - # - or raise IgnoreRequest - return response - - def process_exception(self, request, exception, spider): - # Called when a download handler or a process_request() - # (from other downloader middleware) raises an exception. - - # Must either: - # - return None: continue processing this exception - # - return a Response object: stops process_exception() chain - # - return a Request object: stops process_exception() chain - pass - - def spider_opened(self, spider): - spider.logger.info('Spider opened: %s' % spider.name) +# -*- coding: utf-8 -*- + +# Define here the models for your spider middleware +# +# See documentation in: +# https://docs.scrapy.org/en/latest/topics/spider-middleware.html + +import random + +from scrapy import signals +from scrapy.downloadermiddlewares.retry import get_retry_request + + +class WeiboSpiderMiddleware(object): + # Not all methods need to be defined. If a method is not defined, + # scrapy acts as if the spider middleware does not modify the + # passed objects. + + @classmethod + def from_crawler(cls, crawler): + # This method is used by Scrapy to create your spiders. + s = cls() + crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) + return s + + def process_spider_input(self, response, spider): + # Called for each response that goes through the spider + # middleware and into the spider. + + # Should return None or raise an exception. + return None + + def process_spider_output(self, response, result, spider): + # Called with the results returned from the Spider, after + # it has processed the response. + + # Must return an iterable of Request, dict or Item objects. + for i in result: + yield i + + def process_spider_exception(self, response, exception, spider): + # Called when a spider or process_spider_input() method + # (from other spider middleware) raises an exception. + + # Should return either None or an iterable of Request, dict + # or Item objects. + pass + + def process_start_requests(self, start_requests, spider): + # Called with the start requests of the spider, and works + # similarly to the process_spider_output() method, except + # that it doesn’t have a response associated. + + # Must return only requests (not items). + for r in start_requests: + yield r + + def spider_opened(self, spider): + spider.logger.info('Spider opened: %s' % spider.name) + + +class WeiboDownloaderMiddleware(object): + """Downloader safeguards for stable, account-friendly crawling.""" + + @classmethod + def from_crawler(cls, crawler): + s = cls(crawler) + crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) + return s + + def __init__(self, crawler): + self.crawler = crawler + settings = crawler.settings + self.user_agents = settings.getlist('USER_AGENT_LIST') or [ + settings.get('USER_AGENT', '') + ] + self.ban_status_codes = set(settings.getlist('BAN_STATUS_CODES')) + self.ban_keywords = [ + keyword for keyword in settings.getlist('BAN_KEYWORDS') if keyword + ] + + def process_request(self, request): + if self.user_agents: + request.headers.setdefault('User-Agent', random.choice(self.user_agents)) + request.headers.setdefault('Referer', 'https://s.weibo.com/') + request.headers.setdefault('Connection', 'keep-alive') + return None + + def process_response(self, request, response): + if self._looks_limited(response): + spider = self.crawler.spider + retry = get_retry_request( + request, + spider=spider, + reason='weibo_rate_limited_or_login_required', + ) + if retry: + retry.dont_filter = True + retry.priority = request.priority - 10 + spider.logger.warning('疑似被限流/需要验证，稍后重试: %s %s', + response.status, response.url) + return retry + return response + + def process_exception(self, request, exception): + return None + + def _looks_limited(self, response): + if response.status in self.ban_status_codes: + return True + if response.status != 200: + return False + body = response.text[:4096] + return any(keyword in body for keyword in self.ban_keywords) + + def spider_opened(self, spider): + spider.logger.info('Spider opened: %s' % spider.name) diff --git a/weibo/pipelines.py b/weibo/pipelines.py index 11287ed..e17469f 100644 --- a/weibo/pipelines.py +++ b/weibo/pipelines.py @@ -1,311 +1,311 @@ -# -*- coding: utf-8 -*- - -# Define your item pipelines here -# -# Don't forget to add your pipeline to the ITEM_PIPELINES setting -# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html - -import copy -import csv -import os - -import scrapy -from scrapy.exceptions import DropItem -from scrapy.pipelines.files import FilesPipeline -from scrapy.pipelines.images import ImagesPipeline -from scrapy.utils.project import get_project_settings - -settings = get_project_settings() - - -def normalize_pics(pics): - if not pics: - return '' - if isinstance(pics, str): - return pics - return ','.join(pics) - - -class CsvPipeline(object): - def process_item(self, item, spider): - base_dir = '结果文件' + os.sep + item['keyword'] - if not os.path.isdir(base_dir): - os.makedirs(base_dir) - file_path = base_dir + os.sep + item['keyword'] + '.csv' - if not os.path.isfile(file_path): - is_first_write = 1 - else: - is_first_write = 0 - - if item: - with open(file_path, 'a', encoding='utf-8-sig', newline='') as f: - writer = csv.writer(f) - if is_first_write: - header = [ - 'id', 'bid', 'user_id', '用户昵称', '微博正文', '头条文章url', - '发布位置', '艾特用户', '话题', '转发数', '评论数', '点赞数', '发布时间', - '发布工具', '微博图片url', '微博视频url', 'retweet_id', 'ip', 'user_authentication', - '会员类型', '会员等级' - ] - writer.writerow(header) - - writer.writerow([ - item['weibo'].get('id', ''), - item['weibo'].get('bid', ''), - item['weibo'].get('user_id', ''), - item['weibo'].get('screen_name', ''), - item['weibo'].get('text', ''), - item['weibo'].get('article_url', ''), - item['weibo'].get('location', ''), - item['weibo'].get('at_users', ''), - item['weibo'].get('topics', ''), - item['weibo'].get('reposts_count', ''), - item['weibo'].get('comments_count', ''), - item['weibo'].get('attitudes_count', ''), - item['weibo'].get('created_at', ''), - item['weibo'].get('source', ''), - normalize_pics(item['weibo'].get('pics', [])), - item['weibo'].get('video_url', ''), - item['weibo'].get('retweet_id', ''), - item['weibo'].get('ip', ''), - item['weibo'].get('user_authentication', ''), - item['weibo'].get('vip_type', ''), - item['weibo'].get('vip_level', 0) - ]) - return item - -class SQLitePipeline(object): - def open_spider(self, spider): - try: - import sqlite3 - # 在结果文件目录下创建SQLite数据库 - base_dir = '结果文件' - if not os.path.isdir(base_dir): - os.makedirs(base_dir) - db_name = settings.get('SQLITE_DATABASE', 'weibo.db') - self.conn = sqlite3.connect(os.path.join(base_dir, db_name)) - self.cursor = self.conn.cursor() - # 创建表 - sql = """ - CREATE TABLE IF NOT EXISTS weibo ( - id varchar(20) NOT NULL PRIMARY KEY, - bid varchar(12) NOT NULL, - user_id varchar(20), - screen_name varchar(30), - text varchar(2000), - article_url varchar(100), - topics varchar(200), - at_users varchar(1000), - pics varchar(3000), - video_url varchar(1000), - location varchar(100), - created_at DATETIME, - source varchar(30), - attitudes_count INTEGER, - comments_count INTEGER, - reposts_count INTEGER, - retweet_id varchar(20), - ip varchar(100), - user_authentication varchar(100), - vip_type varchar(50), - vip_level INTEGER - )""" - self.cursor.execute(sql) - self.conn.commit() - except Exception as e: - spider.logger.error("SQLite数据库创建失败: %s", e) - spider.sqlite3_error = True - - - def process_item(self, item, spider): - data = dict(item['weibo']) - data['pics'] = normalize_pics(data.get('pics', [])) - keys = ', '.join(data.keys()) - placeholders = ', '.join(['?'] * len(data)) - sql = f"""INSERT OR REPLACE INTO weibo ({keys}) - VALUES ({placeholders})""" - try: - self.cursor.execute(sql, tuple(data.values())) - self.conn.commit() - except Exception as e: - spider.logger.error("SQLite保存出错: %s", e) - spider.sqlite3_error = True - self.conn.rollback() - return item - - def close_spider(self, spider): - if hasattr(self, 'conn'): - self.conn.close() - -class MyImagesPipeline(ImagesPipeline): - def get_media_requests(self, item, info): - if len(item['weibo']['pics']) == 1: - yield scrapy.Request(item['weibo']['pics'][0], - meta={ - 'item': item, - 'sign': '' - }) - else: - sign = 0 - for image_url in item['weibo']['pics']: - yield scrapy.Request(image_url, - meta={ - 'item': item, - 'sign': '-' + str(sign) - }) - sign += 1 - - def file_path(self, request, response=None, info=None): - image_url = request.url - item = request.meta['item'] - sign = request.meta['sign'] - base_dir = '结果文件' + os.sep + item['keyword'] + os.sep + 'images' - if not os.path.isdir(base_dir): - os.makedirs(base_dir) - image_suffix = image_url[image_url.rfind('.'):] - file_path = base_dir + os.sep + item['weibo'][ - 'id'] + sign + image_suffix - return file_path - - -class MyVideoPipeline(FilesPipeline): - def get_media_requests(self, item, info): - if item['weibo']['video_url']: - yield scrapy.Request(item['weibo']['video_url'], - meta={'item': item}) - - def file_path(self, request, response=None, info=None): - item = request.meta['item'] - base_dir = '结果文件' + os.sep + item['keyword'] + os.sep + 'videos' - if not os.path.isdir(base_dir): - os.makedirs(base_dir) - file_path = base_dir + os.sep + item['weibo']['id'] + '.mp4' - return file_path - - -class MongoPipeline(object): - def open_spider(self, spider): - try: - from pymongo import MongoClient - self.client = MongoClient(settings.get('MONGO_URI')) - self.db = self.client['weibo'] - self.collection = self.db['weibo'] - except ModuleNotFoundError: - spider.pymongo_error = True - - def process_item(self, item, spider): - try: - import pymongo - - new_item = copy.deepcopy(item) - if not self.collection.find_one({'id': new_item['weibo']['id']}): - self.collection.insert_one(dict(new_item['weibo'])) - else: - self.collection.update_one({'id': new_item['weibo']['id']}, - {'$set': dict(new_item['weibo'])}) - except pymongo.errors.ServerSelectionTimeoutError: - spider.mongo_error = True - - def close_spider(self, spider): - try: - self.client.close() - except AttributeError: - pass - - -class MysqlPipeline(object): - def create_database(self, mysql_config): - """创建MySQL数据库""" - import pymysql - sql = """CREATE DATABASE IF NOT EXISTS %s DEFAULT - CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci""" % settings.get( - 'MYSQL_DATABASE', 'weibo') - db = pymysql.connect(**mysql_config) - cursor = db.cursor() - cursor.execute(sql) - db.close() - - def create_table(self): - """创建MySQL表""" - sql = """ - CREATE TABLE IF NOT EXISTS weibo ( - id varchar(20) NOT NULL, - bid varchar(12) NOT NULL, - user_id varchar(20), - screen_name varchar(30), - text varchar(2000), - article_url varchar(100), - topics varchar(200), - at_users varchar(1000), - pics varchar(3000), - video_url varchar(1000), - location varchar(100), - created_at DATETIME, - source varchar(30), - attitudes_count INT, - comments_count INT, - reposts_count INT, - retweet_id varchar(20), - PRIMARY KEY (id), - ip varchar(100), - user_authentication varchar(100), - vip_type varchar(50), - vip_level INT - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4""" - self.cursor.execute(sql) - - def open_spider(self, spider): - try: - import pymysql - mysql_config = { - 'host': settings.get('MYSQL_HOST', 'localhost'), - 'port': settings.get('MYSQL_PORT', 3306), - 'user': settings.get('MYSQL_USER', 'root'), - 'password': settings.get('MYSQL_PASSWORD', '123456'), - 'charset': 'utf8mb4' - } - self.create_database(mysql_config) - mysql_config['db'] = settings.get('MYSQL_DATABASE', 'weibo') - self.db = pymysql.connect(**mysql_config) - self.cursor = self.db.cursor() - self.create_table() - except ImportError: - spider.pymysql_error = True - except pymysql.OperationalError: - spider.mysql_error = True - - def process_item(self, item, spider): - data = dict(item['weibo']) - data['pics'] = normalize_pics(data.get('pics', [])) - keys = ', '.join(data.keys()) - values = ', '.join(['%s'] * len(data)) - sql = """INSERT INTO {table}({keys}) VALUES ({values}) ON - DUPLICATE KEY UPDATE""".format(table='weibo', - keys=keys, - values=values) - update = ','.join([" {key} = {key}".format(key=key) for key in data]) - sql += update - try: - self.cursor.execute(sql, tuple(data.values())) - self.db.commit() - except Exception: - self.db.rollback() - return item - - def close_spider(self, spider): - try: - self.db.close() - except Exception: - pass - - -class DuplicatesPipeline(object): - def __init__(self): - self.ids_seen = set() - - def process_item(self, item, spider): - if item['weibo']['id'] in self.ids_seen: - raise DropItem("过滤重复微博: %s" % item) - else: - self.ids_seen.add(item['weibo']['id']) - return item +# -*- coding: utf-8 -*- + +# Define your item pipelines here +# +# Don't forget to add your pipeline to the ITEM_PIPELINES setting +# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html + +import copy +import csv +import os + +import scrapy +from scrapy.exceptions import DropItem +from scrapy.pipelines.files import FilesPipeline +from scrapy.pipelines.images import ImagesPipeline +from scrapy.utils.project import get_project_settings + +settings = get_project_settings() + + +def normalize_pics(pics): + if not pics: + return '' + if isinstance(pics, str): + return pics + return ','.join(pics) + + +class CsvPipeline(object): + def process_item(self, item): + base_dir = '结果文件' + os.sep + item['keyword'] + if not os.path.isdir(base_dir): + os.makedirs(base_dir) + file_path = base_dir + os.sep + item['keyword'] + '.csv' + if not os.path.isfile(file_path): + is_first_write = 1 + else: + is_first_write = 0 + + if item: + with open(file_path, 'a', encoding='utf-8-sig', newline='') as f: + writer = csv.writer(f) + if is_first_write: + header = [ + 'id', 'bid', 'user_id', '用户昵称', '微博正文', '头条文章url', + '发布位置', '艾特用户', '话题', '转发数', '评论数', '点赞数', '发布时间', + '发布工具', '微博图片url', '微博视频url', 'retweet_id', 'ip', 'user_authentication', + '会员类型', '会员等级' + ] + writer.writerow(header) + + writer.writerow([ + item['weibo'].get('id', ''), + item['weibo'].get('bid', ''), + item['weibo'].get('user_id', ''), + item['weibo'].get('screen_name', ''), + item['weibo'].get('text', ''), + item['weibo'].get('article_url', ''), + item['weibo'].get('location', ''), + item['weibo'].get('at_users', ''), + item['weibo'].get('topics', ''), + item['weibo'].get('reposts_count', ''), + item['weibo'].get('comments_count', ''), + item['weibo'].get('attitudes_count', ''), + item['weibo'].get('created_at', ''), + item['weibo'].get('source', ''), + normalize_pics(item['weibo'].get('pics', [])), + item['weibo'].get('video_url', ''), + item['weibo'].get('retweet_id', ''), + item['weibo'].get('ip', ''), + item['weibo'].get('user_authentication', ''), + item['weibo'].get('vip_type', ''), + item['weibo'].get('vip_level', 0) + ]) + return item + +class SQLitePipeline(object): + def open_spider(self, spider): + try: + import sqlite3 + # 在结果文件目录下创建SQLite数据库 + base_dir = '结果文件' + if not os.path.isdir(base_dir): + os.makedirs(base_dir) + db_name = settings.get('SQLITE_DATABASE', 'weibo.db') + self.conn = sqlite3.connect(os.path.join(base_dir, db_name)) + self.cursor = self.conn.cursor() + # 创建表 + sql = """ + CREATE TABLE IF NOT EXISTS weibo ( + id varchar(20) NOT NULL PRIMARY KEY, + bid varchar(12) NOT NULL, + user_id varchar(20), + screen_name varchar(30), + text varchar(2000), + article_url varchar(100), + topics varchar(200), + at_users varchar(1000), + pics varchar(3000), + video_url varchar(1000), + location varchar(100), + created_at DATETIME, + source varchar(30), + attitudes_count INTEGER, + comments_count INTEGER, + reposts_count INTEGER, + retweet_id varchar(20), + ip varchar(100), + user_authentication varchar(100), + vip_type varchar(50), + vip_level INTEGER + )""" + self.cursor.execute(sql) + self.conn.commit() + except Exception as e: + spider.logger.error("SQLite数据库创建失败: %s", e) + spider.sqlite3_error = True + + + def process_item(self, item, spider): + data = dict(item['weibo']) + data['pics'] = normalize_pics(data.get('pics', [])) + keys = ', '.join(data.keys()) + placeholders = ', '.join(['?'] * len(data)) + sql = f"""INSERT OR REPLACE INTO weibo ({keys}) + VALUES ({placeholders})""" + try: + self.cursor.execute(sql, tuple(data.values())) + self.conn.commit() + except Exception as e: + spider.logger.error("SQLite保存出错: %s", e) + spider.sqlite3_error = True + self.conn.rollback() + return item + + def close_spider(self, spider): + if hasattr(self, 'conn'): + self.conn.close() + +class MyImagesPipeline(ImagesPipeline): + def get_media_requests(self, item, info): + if len(item['weibo']['pics']) == 1: + yield scrapy.Request(item['weibo']['pics'][0], + meta={ + 'item': item, + 'sign': '' + }) + else: + sign = 0 + for image_url in item['weibo']['pics']: + yield scrapy.Request(image_url, + meta={ + 'item': item, + 'sign': '-' + str(sign) + }) + sign += 1 + + def file_path(self, request, response=None, info=None): + image_url = request.url + item = request.meta['item'] + sign = request.meta['sign'] + base_dir = '结果文件' + os.sep + item['keyword'] + os.sep + 'images' + if not os.path.isdir(base_dir): + os.makedirs(base_dir) + image_suffix = image_url[image_url.rfind('.'):] + file_path = base_dir + os.sep + item['weibo'][ + 'id'] + sign + image_suffix + return file_path + + +class MyVideoPipeline(FilesPipeline): + def get_media_requests(self, item, info): + if item['weibo']['video_url']: + yield scrapy.Request(item['weibo']['video_url'], + meta={'item': item}) + + def file_path(self, request, response=None, info=None): + item = request.meta['item'] + base_dir = '结果文件' + os.sep + item['keyword'] + os.sep + 'videos' + if not os.path.isdir(base_dir): + os.makedirs(base_dir) + file_path = base_dir + os.sep + item['weibo']['id'] + '.mp4' + return file_path + + +class MongoPipeline(object): + def open_spider(self, spider): + try: + from pymongo import MongoClient + self.client = MongoClient(settings.get('MONGO_URI')) + self.db = self.client['weibo'] + self.collection = self.db['weibo'] + except ModuleNotFoundError: + spider.pymongo_error = True + + def process_item(self, item, spider): + try: + import pymongo + + new_item = copy.deepcopy(item) + if not self.collection.find_one({'id': new_item['weibo']['id']}): + self.collection.insert_one(dict(new_item['weibo'])) + else: + self.collection.update_one({'id': new_item['weibo']['id']}, + {'$set': dict(new_item['weibo'])}) + except pymongo.errors.ServerSelectionTimeoutError: + spider.mongo_error = True + + def close_spider(self, spider): + try: + self.client.close() + except AttributeError: + pass + + +class MysqlPipeline(object): + def create_database(self, mysql_config): + """创建MySQL数据库""" + import pymysql + sql = """CREATE DATABASE IF NOT EXISTS %s DEFAULT + CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci""" % settings.get( + 'MYSQL_DATABASE', 'weibo') + db = pymysql.connect(**mysql_config) + cursor = db.cursor() + cursor.execute(sql) + db.close() + + def create_table(self): + """创建MySQL表""" + sql = """ + CREATE TABLE IF NOT EXISTS weibo ( + id varchar(20) NOT NULL, + bid varchar(12) NOT NULL, + user_id varchar(20), + screen_name varchar(30), + text varchar(2000), + article_url varchar(100), + topics varchar(200), + at_users varchar(1000), + pics varchar(3000), + video_url varchar(1000), + location varchar(100), + created_at DATETIME, + source varchar(30), + attitudes_count INT, + comments_count INT, + reposts_count INT, + retweet_id varchar(20), + PRIMARY KEY (id), + ip varchar(100), + user_authentication varchar(100), + vip_type varchar(50), + vip_level INT + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4""" + self.cursor.execute(sql) + + def open_spider(self, spider): + try: + import pymysql + mysql_config = { + 'host': settings.get('MYSQL_HOST', 'localhost'), + 'port': settings.get('MYSQL_PORT', 3306), + 'user': settings.get('MYSQL_USER', 'root'), + 'password': settings.get('MYSQL_PASSWORD', '123456'), + 'charset': 'utf8mb4' + } + self.create_database(mysql_config) + mysql_config['db'] = settings.get('MYSQL_DATABASE', 'weibo') + self.db = pymysql.connect(**mysql_config) + self.cursor = self.db.cursor() + self.create_table() + except ImportError: + spider.pymysql_error = True + except pymysql.OperationalError: + spider.mysql_error = True + + def process_item(self, item, spider): + data = dict(item['weibo']) + data['pics'] = normalize_pics(data.get('pics', [])) + keys = ', '.join(data.keys()) + values = ', '.join(['%s'] * len(data)) + sql = """INSERT INTO {table}({keys}) VALUES ({values}) ON + DUPLICATE KEY UPDATE""".format(table='weibo', + keys=keys, + values=values) + update = ','.join([" {key} = {key}".format(key=key) for key in data]) + sql += update + try: + self.cursor.execute(sql, tuple(data.values())) + self.db.commit() + except Exception: + self.db.rollback() + return item + + def close_spider(self, spider): + try: + self.db.close() + except Exception: + pass + + +class DuplicatesPipeline(object): + def __init__(self): + self.ids_seen = set() + + def process_item(self, item): + if item['weibo']['id'] in self.ids_seen: + raise DropItem("过滤重复微博: %s" % item) + else: + self.ids_seen.add(item['weibo']['id']) + return item diff --git a/weibo/settings.py b/weibo/settings.py deleted file mode 100644 index 75533a0..0000000 --- a/weibo/settings.py +++ /dev/null @@ -1,84 +0,0 @@ -# -*- coding: utf-8 -*- -import json -import os - - -def env_int(name, default): - value = os.getenv(name) - if value is None or value == '': - return default - return int(value) - - -def env_list(name, default): - value = os.getenv(name) - if value is None or value == '': - return default - try: - parsed = json.loads(value) - except json.JSONDecodeError: - parsed = [item.strip() for item in value.split(',')] - if isinstance(parsed, str): - parsed = [parsed] - return [item for item in parsed if item] - -BOT_NAME = 'weibo' -SPIDER_MODULES = ['weibo.spiders'] -NEWSPIDER_MODULE = 'weibo.spiders' -COOKIES_ENABLED = False -TELNETCONSOLE_ENABLED = False -LOG_LEVEL = 'ERROR' -# 访问完一个页面再访问下一个时需要等待的时间，默认为10秒 -DOWNLOAD_DELAY = 10 -DEFAULT_REQUEST_HEADERS = { - 'Accept': - 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', - 'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8,en-US;q=0.7', - 'cookie': os.getenv('WEIBO_COOKIE', ''), -} -ITEM_PIPELINES = { - 'weibo.pipelines.DuplicatesPipeline': 300, - 'weibo.pipelines.CsvPipeline': 301, - # 'weibo.pipelines.MysqlPipeline': 302, - # 'weibo.pipelines.MongoPipeline': 303, - # 'weibo.pipelines.MyImagesPipeline': 304, - # 'weibo.pipelines.MyVideoPipeline': 305, - # 'weibo.pipelines.SQLitePipeline': 306 -} -# 要搜索的关键词列表，可写多个, 值可以是由关键词或话题组成的列表，也可以是包含关键词的txt文件路径， -# 如'keyword_list.txt'，txt文件中每个关键词占一行 -KEYWORD_LIST = env_list('WEIBO_KEYWORDS', ['迪丽热巴']) # 或者 KEYWORD_LIST = 'keyword_list.txt' -# 要搜索的微博类型，0代表搜索全部微博，1代表搜索全部原创微博，2代表热门微博，3代表关注人微博，4代表认证用户微博，5代表媒体微博，6代表观点微博 -WEIBO_TYPE = env_int('WEIBO_TYPE', 1) -# 筛选结果微博中必需包含的内容，0代表不筛选，获取全部微博，1代表搜索包含图片的微博，2代表包含视频的微博，3代表包含音乐的微博，4代表包含短链接的微博 -CONTAIN_TYPE = env_int('WEIBO_CONTAIN_TYPE', 0) -# 筛选微博的发布地区，精确到省或直辖市，值不应包含“省”或“市”等字，如想筛选北京市的微博请用“北京”而不是“北京市”，想要筛选安徽省的微博请用“安徽”而不是“安徽省”，可以写多个地区， -# 具体支持的地名见region.py文件，注意只支持省或直辖市的名字，省下面的市名及直辖市下面的区县名不支持，不筛选请用“全部” -REGION = env_list('WEIBO_REGION', ['全部']) -# 搜索的起始日期，为yyyy-mm-dd形式，搜索结果包含该日期 -START_DATE = os.getenv('WEIBO_START_DATE', '2020-03-01') -# 搜索的终止日期，为yyyy-mm-dd形式，搜索结果包含该日期 -END_DATE = os.getenv('WEIBO_END_DATE', '2020-03-01') -# 进一步细分搜索的阈值，若结果页数大于等于该值，则认为结果没有完全展示，细分搜索条件重新搜索以获取更多微博。数值越大速度越快，也越有可能漏掉微博；数值越小速度越慢，获取的微博就越多。 -# 建议数值大小设置在40到50之间。 -FURTHER_THRESHOLD = env_int('WEIBO_FURTHER_THRESHOLD', 46) -# 爬取结果的数量限制，爬取到该数量的微博后自动停止，设置为0代表不限制 -LIMIT_RESULT = env_int('WEIBO_LIMIT_RESULT', 0) -# 是否请求微博 AJAX 接口补充 IP 属地。默认关闭，避免额外请求阻塞主抓取链路。 -FETCH_IP = os.getenv('WEIBO_FETCH_IP', '0') == '1' -# IP 属地接口超时时间，仅在 FETCH_IP=True 时使用。 -IP_REQUEST_TIMEOUT = 5 -# 图片文件存储路径 -IMAGES_STORE = './' -# 视频文件存储路径 -FILES_STORE = './' -# 配置MongoDB数据库 -# MONGO_URI = 'localhost' -# 配置MySQL数据库，以下为默认配置，可以根据实际情况更改，程序会自动生成一个名为weibo的数据库，如果想换其它名字请更改MYSQL_DATABASE值 -# MYSQL_HOST = 'localhost' -# MYSQL_PORT = 3306 -# MYSQL_USER = 'root' -# MYSQL_PASSWORD = '123456' -# MYSQL_DATABASE = 'weibo' -# 配置SQLite数据库 -# SQLITE_DATABASE = 'weibo.db' diff --git a/weibo/spiders/__init__.py b/weibo/spiders/__init__.py index ebd689a..5ca581d 100644 --- a/weibo/spiders/__init__.py +++ b/weibo/spiders/__init__.py @@ -1,4 +1,4 @@ -# This package will contain the spiders of your Scrapy project -# -# Please refer to the documentation for information on how to create and manage -# your spiders. +# This package will contain the spiders of your Scrapy project +# +# Please refer to the documentation for information on how to create and manage +# your spiders. diff --git a/weibo/spiders/search.py b/weibo/spiders/search.py index 46f9143..9234694 100644 --- a/weibo/spiders/search.py +++ b/weibo/spiders/search.py @@ -1,689 +1,805 @@ -# -*- coding: utf-8 -*- -import os -import re -from datetime import datetime, timedelta -from urllib.parse import unquote - -import requests -import scrapy - -import weibo.utils.util as util -from scrapy.exceptions import CloseSpider -from scrapy.utils.project import get_project_settings -from weibo.items import WeiboItem - - -class SearchSpider(scrapy.Spider): - name = 'search' - allowed_domains = ['weibo.com'] - base_url = 'https://s.weibo.com' - - @classmethod - def from_crawler(cls, crawler, *args, **kwargs): - spider = super().from_crawler(crawler, *args, **kwargs) - spider.configure(crawler.settings) - return spider - - def __init__(self, *args, settings=None, **kwargs): - super().__init__(*args, **kwargs) - self.configure(settings or get_project_settings()) - - def configure(self, settings): - self.project_settings = settings - self.keyword_list = self.load_keyword_list( - self.project_settings.get('KEYWORD_LIST')) - self.weibo_type = util.convert_weibo_type( - self.project_settings.get('WEIBO_TYPE')) - self.contain_type = util.convert_contain_type( - self.project_settings.get('CONTAIN_TYPE')) - self.regions = util.get_regions(self.project_settings.get('REGION')) - self.start_date = self.project_settings.get( - 'START_DATE', datetime.now().strftime('%Y-%m-%d')) - self.end_date = self.project_settings.get( - 'END_DATE', datetime.now().strftime('%Y-%m-%d')) - self.further_threshold = int( - self.project_settings.get('FURTHER_THRESHOLD', 46)) - self.limit_result = int(self.project_settings.get('LIMIT_RESULT', 0)) - self.fetch_ip = bool(self.project_settings.getbool('FETCH_IP', False)) - self.ip_request_timeout = int( - self.project_settings.get('IP_REQUEST_TIMEOUT', 5)) - self.result_count = 0 - self.mongo_error = False - self.pymongo_error = False - self.mysql_error = False - self.pymysql_error = False - self.sqlite3_error = False - - def load_keyword_list(self, keyword_list): - """Load keywords from settings or a UTF-8 text file.""" - if not keyword_list: - raise CloseSpider('KEYWORD_LIST不能为空，请在settings.py中配置关键词') - if not isinstance(keyword_list, list): - if not os.path.isabs(keyword_list): - keyword_list = os.path.join(os.getcwd(), keyword_list) - if not os.path.isfile(keyword_list): - raise CloseSpider('不存在%s文件' % keyword_list) - try: - keyword_list = util.get_keyword_list(keyword_list) - except ValueError as exc: - raise CloseSpider(str(exc)) from exc - keywords = [] - for keyword in keyword_list: - if not keyword: - continue - if len(keyword) > 2 and keyword[0] == '#' and keyword[-1] == '#': - keyword = '%23' + keyword[1:-1] + '%23' - keywords.append(keyword) - if not keywords: - raise CloseSpider('KEYWORD_LIST不能为空，请至少配置一个关键词') - return keywords - - def validate_runtime_settings(self): - """Validate settings that are only required when the crawl starts.""" - headers = self.project_settings.get('DEFAULT_REQUEST_HEADERS') or {} - cookie = headers.get('cookie', '') - if not cookie or cookie == 'your_cookie_here': - raise CloseSpider( - '未配置微博Cookie。请设置环境变量WEIBO_COOKIE，或在本地settings.py中配置DEFAULT_REQUEST_HEADERS["cookie"]') - if util.str_to_time(self.start_date) > util.str_to_time(self.end_date): - raise CloseSpider( - 'settings.py配置错误，START_DATE值应早于或等于END_DATE值，请重新配置settings.py') - - def check_limit(self): - """检查是否达到爬取结果数量限制""" - if self.limit_result > 0 and self.result_count >= self.limit_result: - self.logger.info('已达到爬取结果数量限制：%s条，停止爬取', self.limit_result) - raise CloseSpider('已达到爬取结果数量限制') - return False - - def start_requests(self): - self.validate_runtime_settings() - start_date = datetime.strptime(self.start_date, '%Y-%m-%d') - end_date = datetime.strptime(self.end_date, - '%Y-%m-%d') + timedelta(days=1) - start_str = start_date.strftime('%Y-%m-%d') + '-0' - end_str = end_date.strftime('%Y-%m-%d') + '-0' - for keyword in self.keyword_list: - if not self.project_settings.get('REGION') or '全部' in self.project_settings.get( - 'REGION'): - base_url = 'https://s.weibo.com/weibo?q=%s' % keyword - url = base_url + self.weibo_type - url += self.contain_type - url += '×cope=custom:{}:{}'.format(start_str, end_str) - yield scrapy.Request(url=url, - callback=self.parse, - meta={ - 'base_url': base_url, - 'keyword': keyword - }) - else: - for region in self.regions.values(): - base_url = ( - 'https://s.weibo.com/weibo?q={}®ion=custom:{}:1000' - ).format(keyword, region['code']) - url = base_url + self.weibo_type - url += self.contain_type - url += '×cope=custom:{}:{}'.format(start_str, end_str) - # 获取一个省的搜索结果 - yield scrapy.Request(url=url, - callback=self.parse, - meta={ - 'base_url': base_url, - 'keyword': keyword, - 'province': region - }) - - def check_environment(self): - """判断配置要求的软件是否已安装""" - if self.pymongo_error: - self.logger.error('系统中可能没有安装pymongo库，请先运行 pip install pymongo ，再运行程序') - raise CloseSpider() - if self.mongo_error: - self.logger.error('系统中可能没有安装或启动MongoDB数据库，请先根据系统环境安装或启动MongoDB，再运行程序') - raise CloseSpider() - if self.pymysql_error: - self.logger.error('系统中可能没有安装pymysql库，请先运行 pip install pymysql ，再运行程序') - raise CloseSpider() - if self.mysql_error: - self.logger.error('系统中可能没有安装或正确配置MySQL数据库，请先根据系统环境安装或配置MySQL，再运行程序') - raise CloseSpider() - if self.sqlite3_error: - self.logger.error( - '系统中可能没有安装或正确配置SQLite3数据库，请检查SQLITE_DATABASE配置后再运行程序') - raise CloseSpider() - - def parse(self, response): - base_url = response.meta.get('base_url') - keyword = response.meta.get('keyword') - province = response.meta.get('province') - is_empty = response.xpath( - '//div[@class="card card-no-result s-pt20b40"]') - page_count = len(response.xpath('//ul[@class="s-scroll"]/li')) - if is_empty: - self.logger.info('当前页面搜索结果为空: %s', response.url) - elif page_count < self.further_threshold: - # 解析当前页面 - for weibo in self.parse_weibo(response): - self.check_environment() - # 检查是否达到爬取结果数量限制 - if self.check_limit(): - return - yield weibo - next_url = response.xpath( - '//a[@class="next"]/@href').extract_first() - if next_url: - # 检查是否达到爬取结果数量限制 - if self.check_limit(): - return - next_url = self.base_url + next_url - yield scrapy.Request(url=next_url, - callback=self.parse_page, - meta=response.meta) - else: - start_date = datetime.strptime(self.start_date, '%Y-%m-%d') - end_date = datetime.strptime(self.end_date, '%Y-%m-%d') - while start_date <= end_date: - start_str = start_date.strftime('%Y-%m-%d') + '-0' - start_date = start_date + timedelta(days=1) - end_str = start_date.strftime('%Y-%m-%d') + '-0' - url = base_url + self.weibo_type - url += self.contain_type - url += '×cope=custom:{}:{}&page=1'.format( - start_str, end_str) - # 获取一天的搜索结果 - yield scrapy.Request(url=url, - callback=self.parse_by_day, - meta={ - 'base_url': base_url, - 'keyword': keyword, - 'province': province, - 'date': start_str[:-2] - }) - - def parse_by_day(self, response): - """以天为单位筛选""" - base_url = response.meta.get('base_url') - keyword = response.meta.get('keyword') - province = response.meta.get('province') - is_empty = response.xpath( - '//div[@class="card card-no-result s-pt20b40"]') - date = response.meta.get('date') - page_count = len(response.xpath('//ul[@class="s-scroll"]/li')) - if is_empty: - self.logger.info('当前页面搜索结果为空: %s', response.url) - elif page_count < self.further_threshold: - # 解析当前页面 - for weibo in self.parse_weibo(response): - self.check_environment() - # 检查是否达到爬取结果数量限制 - if self.check_limit(): - return - yield weibo - next_url = response.xpath( - '//a[@class="next"]/@href').extract_first() - if next_url: - # 检查是否达到爬取结果数量限制 - if self.check_limit(): - return - next_url = self.base_url + next_url - yield scrapy.Request(url=next_url, - callback=self.parse_page, - meta=response.meta) - else: - start_date_str = date + '-0' - start_date = datetime.strptime(start_date_str, '%Y-%m-%d-%H') - for i in range(1, 25): - start_str = start_date.strftime('%Y-%m-%d-X%H').replace( - 'X0', 'X').replace('X', '') - start_date = start_date + timedelta(hours=1) - end_str = start_date.strftime('%Y-%m-%d-X%H').replace( - 'X0', 'X').replace('X', '') - url = base_url + self.weibo_type - url += self.contain_type - url += '×cope=custom:{}:{}&page=1'.format( - start_str, end_str) - # 获取一小时的搜索结果 - yield scrapy.Request(url=url, - callback=self.parse_by_hour_province - if province else self.parse_by_hour, - meta={ - 'base_url': base_url, - 'keyword': keyword, - 'province': province, - 'start_time': start_str, - 'end_time': end_str - }) - - def parse_by_hour(self, response): - """以小时为单位筛选""" - keyword = response.meta.get('keyword') - is_empty = response.xpath( - '//div[@class="card card-no-result s-pt20b40"]') - start_time = response.meta.get('start_time') - end_time = response.meta.get('end_time') - page_count = len(response.xpath('//ul[@class="s-scroll"]/li')) - if is_empty: - self.logger.info('当前页面搜索结果为空: %s', response.url) - elif page_count < self.further_threshold: - # 解析当前页面 - for weibo in self.parse_weibo(response): - self.check_environment() - yield weibo - next_url = response.xpath( - '//a[@class="next"]/@href').extract_first() - if next_url: - next_url = self.base_url + next_url - yield scrapy.Request(url=next_url, - callback=self.parse_page, - meta=response.meta) - else: - for region in self.regions.values(): - url = ('https://s.weibo.com/weibo?q={}®ion=custom:{}:1000' - ).format(keyword, region['code']) - url += self.weibo_type - url += self.contain_type - url += '×cope=custom:{}:{}&page=1'.format( - start_time, end_time) - # 获取一小时一个省的搜索结果 - yield scrapy.Request(url=url, - callback=self.parse_by_hour_province, - meta={ - 'keyword': keyword, - 'start_time': start_time, - 'end_time': end_time, - 'province': region - }) - - def parse_by_hour_province(self, response): - """以小时和直辖市/省为单位筛选""" - keyword = response.meta.get('keyword') - is_empty = response.xpath( - '//div[@class="card card-no-result s-pt20b40"]') - start_time = response.meta.get('start_time') - end_time = response.meta.get('end_time') - province = response.meta.get('province') - page_count = len(response.xpath('//ul[@class="s-scroll"]/li')) - if is_empty: - self.logger.info('当前页面搜索结果为空: %s', response.url) - elif page_count < self.further_threshold: - # 解析当前页面 - for weibo in self.parse_weibo(response): - self.check_environment() - yield weibo - next_url = response.xpath( - '//a[@class="next"]/@href').extract_first() - if next_url: - next_url = self.base_url + next_url - yield scrapy.Request(url=next_url, - callback=self.parse_page, - meta=response.meta) - else: - for city in province['city'].values(): - url = ('https://s.weibo.com/weibo?q={}®ion=custom:{}:{}' - ).format(keyword, province['code'], city) - url += self.weibo_type - url += self.contain_type - url += '×cope=custom:{}:{}&page=1'.format( - start_time, end_time) - # 获取一小时一个城市的搜索结果 - yield scrapy.Request(url=url, - callback=self.parse_page, - meta={ - 'keyword': keyword, - 'start_time': start_time, - 'end_time': end_time, - 'province': province, - 'city': city - }) - - def parse_page(self, response): - """解析一页搜索结果的信息""" - keyword = response.meta.get('keyword') - is_empty = response.xpath( - '//div[@class="card card-no-result s-pt20b40"]') - if is_empty: - self.logger.info('当前页面搜索结果为空: %s', response.url) - else: - for weibo in self.parse_weibo(response): - self.check_environment() - # 检查是否达到爬取结果数量限制 - if self.check_limit(): - return - yield weibo - next_url = response.xpath( - '//a[@class="next"]/@href').extract_first() - if next_url: - # 检查是否达到爬取结果数量限制 - if self.check_limit(): - return - next_url = self.base_url + next_url - yield scrapy.Request(url=next_url, - callback=self.parse_page, - meta=response.meta) - - def get_ip(self, bid): - if not self.fetch_ip or not bid: - return "" - url = f"https://weibo.com/ajax/statuses/show?id={bid}&locale=zh-CN" - try: - response = requests.get( - url, - headers=self.project_settings.get('DEFAULT_REQUEST_HEADERS'), - timeout=self.ip_request_timeout) - except requests.RequestException as exc: - self.logger.debug('IP属地请求失败 bid=%s: %s', bid, exc) - return "" - if response.status_code != 200: - return "" - try: - data = response.json() - except requests.exceptions.JSONDecodeError: - return "" - ip_str = data.get("region_name", "") - if ip_str: - ip_str = ip_str.split()[-1] - return ip_str - - def get_article_url(self, selector): - """获取微博头条文章url""" - article_url = '' - text = (selector.xpath('string(.)').extract_first() or '').replace( - '\u200b', '').replace('\ue627', '').replace('\n', - '').replace(' ', '') - if text.startswith('发布了头条文章'): - urls = selector.xpath('.//a') - for url in urls: - if url.xpath( - 'i[@class="wbicon"]/text()').extract_first() == 'O': - if url.xpath('@href').extract_first() and url.xpath( - '@href').extract_first().startswith('http://t.cn'): - article_url = url.xpath('@href').extract_first() - break - return article_url - - def get_location(self, selector): - """获取微博发布位置""" - a_list = selector.xpath('.//a') - location = '' - for a in a_list: - if a.xpath('./i[@class="wbicon"]') and a.xpath( - './i[@class="wbicon"]/text()').extract_first() == '2': - location = a.xpath('string(.)').extract_first()[1:] - break - return location - - def get_at_users(self, selector): - """获取微博中@的用户昵称""" - a_list = selector.xpath('.//a') - at_users = '' - at_list = [] - for a in a_list: - href = a.xpath('@href').extract_first() or '' - text = a.xpath('string(.)').extract_first() or '' - if len(unquote(href)) > 14 and len(text) > 1: - if unquote(href)[14:] == text[1:]: - at_user = text[1:] - if at_user not in at_list: - at_list.append(at_user) - if at_list: - at_users = ','.join(at_list) - return at_users - - def get_topics(self, selector): - """获取参与的微博话题""" - a_list = selector.xpath('.//a') - topics = '' - topic_list = [] - for a in a_list: - text = a.xpath('string(.)').extract_first() or '' - if len(text) > 2 and text[0] == '#' and text[-1] == '#': - if text[1:-1] not in topic_list: - topic_list.append(text[1:-1]) - if topic_list: - topics = ','.join(topic_list) - return topics - - def get_vip(self, selector): - """获取用户的VIP类型和等级信息""" - vip_type = "非会员" - vip_level = 0 - - vip_container = selector.xpath('.//div[@class="user_vip_icon_container"]') - if vip_container: - svvip_img = vip_container.xpath('.//img[contains(@src, "svvip_")]') - if svvip_img: - vip_type = "超级会员" - src = svvip_img.xpath('@src').extract_first() or '' - level_match = re.search(r'svvip_(\d+)\.png', src) - if level_match: - vip_level = int(level_match.group(1)) - else: - vip_img = vip_container.xpath('.//img[contains(@src, "vip_")]') - if vip_img: - vip_type = "会员" - src = vip_img.xpath('@src').extract_first() or '' - level_match = re.search(r'vip_(\d+)\.png', src) - if level_match: - vip_level = int(level_match.group(1)) - - return vip_type, vip_level - - def extract_count(self, text): - """Extract Weibo count text; missing or non-numeric labels become 0.""" - matches = re.findall(r'\d+.*', text or '') - return matches[0] if matches else '0' - - def clean_weibo_text(self, selector, is_long=False): - text = (selector.xpath('string(.)').extract_first() or '').replace( - '\u200b', '').replace('\ue627', '') - location = self.get_location(selector) - if location: - text = text.replace('2' + location, '') - text = text[2:].replace(' ', '') if len(text) >= 2 else text.strip() - if is_long and len(text) >= 4: - text = text[:-4] - return text, location - - def parse_weibo(self, response): - """解析网页中的微博信息""" - keyword = response.meta.get('keyword') - for sel in response.xpath("//div[@class='card-wrap']"): - # 检查是否达到爬取结果数量限制 - if self.check_limit(): - return - - info = sel.xpath( - "div[@class='card']/div[@class='card-feed']/div[@class='content']/div[@class='info']" - ) - if info: - weibo = WeiboItem() - weibo['id'] = sel.xpath('@mid').extract_first() - from_href = sel.xpath( - './/div[@class="from"]/a[1]/@href').extract_first() - user_href = info[0].xpath('div[2]/a/@href').extract_first() - txt_nodes = sel.xpath('.//p[@class="txt"]') - if not weibo['id'] or not from_href or not user_href or not txt_nodes: - self.logger.warning('跳过无法解析的微博卡片: %s', response.url) - continue - bid = from_href.split('/')[-1].split('?')[0] - weibo['bid'] = bid - weibo['user_id'] = user_href.split('?')[0].split('/')[-1] - weibo['screen_name'] = info[0].xpath( - 'div[2]/a/@nick-name').extract_first() or '' - # 获取VIP信息 - weibo['vip_type'], weibo['vip_level'] = self.get_vip(info[0]) - txt_sel = txt_nodes[0] - retweet_sel = sel.xpath('.//div[@class="card-comment"]') - retweet_txt_sel = '' - if retweet_sel and retweet_sel[0].xpath('.//p[@class="txt"]'): - retweet_txt_sel = retweet_sel[0].xpath( - './/p[@class="txt"]')[0] - content_full = sel.xpath( - './/p[@node-type="feed_list_content_full"]') - - is_long_weibo = False - is_long_retweet = False - if content_full: - if not retweet_sel: - txt_sel = content_full[0] - is_long_weibo = True - elif len(content_full) == 2: - txt_sel = content_full[0] - retweet_txt_sel = content_full[1] - is_long_weibo = True - is_long_retweet = True - elif retweet_sel[0].xpath( - './/p[@node-type="feed_list_content_full"]'): - retweet_txt_sel = retweet_sel[0].xpath( - './/p[@node-type="feed_list_content_full"]')[0] - is_long_retweet = True - else: - txt_sel = content_full[0] - is_long_weibo = True - weibo['article_url'] = self.get_article_url(txt_sel) - weibo['text'], weibo['location'] = self.clean_weibo_text( - txt_sel, is_long_weibo) - weibo['at_users'] = self.get_at_users(txt_sel) - weibo['topics'] = self.get_topics(txt_sel) - reposts_count = sel.xpath( - './/a[@action-type="feed_list_forward"]/text()').extract() - reposts_count = "".join(reposts_count) - weibo['reposts_count'] = self.extract_count(reposts_count) - comments_count = sel.xpath( - './/a[@action-type="feed_list_comment"]/text()' - ).extract_first() - weibo['comments_count'] = self.extract_count(comments_count) - attitudes_count = sel.xpath( - './/a[@action-type="feed_list_like"]/button/span[2]/text()').extract_first() - weibo['attitudes_count'] = self.extract_count(attitudes_count) - created_at = sel.xpath( - './/div[@class="from"]/a[1]/text()').extract_first() - created_at = (created_at or '').replace(' ', '').replace( - '\n', '').split('前')[0] - weibo['created_at'] = util.standardize_date( - created_at) if created_at else '' - source = sel.xpath('.//div[@class="from"]/a[2]/text()' - ).extract_first() - weibo['source'] = source if source else '' - pics = [] - is_exist_pic = sel.xpath( - './/div[@class="media media-piclist"]') - if is_exist_pic: - pics = is_exist_pic[0].xpath('ul[1]/li/img/@src').extract() - pics = [pic[8:] for pic in pics] - pics = [ - re.sub(r'/.*?/', '/large/', pic, 1) for pic in pics - ] - pics = ['https://' + pic for pic in pics] - video_url = '' - is_exist_video = sel.xpath( - './/div[@class="thumbnail"]//video-player').extract_first() - if is_exist_video: - video_matches = re.findall(r'src:\'(.*?)\'', is_exist_video) - if video_matches: - video_url = video_matches[0].replace('&', '&') - video_url = 'http:' + video_url - if not retweet_sel: - weibo['pics'] = pics - weibo['video_url'] = video_url - else: - weibo['pics'] = [] - weibo['video_url'] = '' - weibo['retweet_id'] = '' - if retweet_sel and retweet_sel[0].xpath( - './/div[@node-type="feed_list_forwardContent"]/a[1]'): - retweet_id_data = retweet_sel[0].xpath( - './/a[@action-type="feed_list_like"]/@action-data' - ).extract_first() or '' - retweet_from_href = retweet_sel[0].xpath( - './/p[@class="from"]/a/@href').extract_first() or '' - retweet_info = retweet_sel[0].xpath( - './/div[@node-type="feed_list_forwardContent"]/a[1]' - ) - if not retweet_id_data.startswith('mid=') or not retweet_from_href or not retweet_info or not retweet_txt_sel: - self.logger.warning('跳过无法解析的转发微博: %s', response.url) - else: - retweet = WeiboItem() - retweet['id'] = retweet_id_data[4:] - retweet['bid'] = retweet_from_href.split( - '/')[-1].split('?')[0] - info = retweet_info[0] - retweet_user_href = info.xpath( - '@href').extract_first() or '' - retweet['user_id'] = retweet_user_href.split('/')[-1] - retweet['screen_name'] = info.xpath( - '@nick-name').extract_first() or '' - retweet['vip_type'], retweet['vip_level'] = self.get_vip( - info) - retweet['article_url'] = self.get_article_url( - retweet_txt_sel) - retweet['text'], retweet['location'] = self.clean_weibo_text( - retweet_txt_sel, is_long_retweet) - retweet['at_users'] = self.get_at_users(retweet_txt_sel) - retweet['topics'] = self.get_topics(retweet_txt_sel) - reposts_count = retweet_sel[0].xpath( - './/ul[@class="act s-fr"]/li[1]/a[1]/text()' - ).extract_first() - retweet['reposts_count'] = self.extract_count( - reposts_count) - comments_count = retweet_sel[0].xpath( - './/ul[@class="act s-fr"]/li[2]/a[1]/text()' - ).extract_first() - retweet['comments_count'] = self.extract_count( - comments_count) - attitudes_count = retweet_sel[0].xpath( - './/a[@class="woo-box-flex woo-box-alignCenter woo-box-justifyCenter"]//span[@class="woo-like-count"]/text()' - ).extract_first() - retweet['attitudes_count'] = self.extract_count( - attitudes_count) - created_at = retweet_sel[0].xpath( - './/p[@class="from"]/a[1]/text()').extract_first() - created_at = (created_at or '').replace(' ', '').replace( - '\n', '').split('前')[0] - retweet['created_at'] = util.standardize_date( - created_at) if created_at else '' - source = retweet_sel[0].xpath( - './/p[@class="from"]/a[2]/text()').extract_first() - retweet['source'] = source if source else '' - retweet['pics'] = pics - retweet['video_url'] = video_url - retweet['retweet_id'] = '' - retweet['ip'] = '' - retweet['user_authentication'] = '' - - self.result_count += 1 - - yield {'weibo': retweet, 'keyword': keyword} - - if self.check_limit(): - return - - weibo['retweet_id'] = retweet['id'] - weibo["ip"] = self.get_ip(bid) - - avator = sel.xpath( - "div[@class='card']/div[@class='card-feed']/div[@class='avator']" - ) - if avator: - user_auth = avator.xpath('.//svg/@id').extract_first() - if user_auth == 'woo_svg_vblue': - weibo['user_authentication'] = '蓝V' - elif user_auth == 'woo_svg_vyellow': - weibo['user_authentication'] = '黄V' - elif user_auth == 'woo_svg_vorange': - weibo['user_authentication'] = '红V' - elif user_auth == 'woo_svg_vgold': - weibo['user_authentication'] = '金V' - else: - weibo['user_authentication'] = '普通用户' - else: - weibo['user_authentication'] = '普通用户' - - # 增加结果计数（主微博） - self.result_count += 1 - - yield {'weibo': weibo, 'keyword': keyword} - - # 检查是否达到爬取结果数量限制 - if self.check_limit(): - return +# -*- coding: utf-8 -*- +import os +import re +import time +from datetime import datetime, timedelta +from urllib.parse import unquote + +import requests +import scrapy + +import weibo.utils.util as util +from scrapy.exceptions import CloseSpider +from scrapy.utils.project import get_project_settings +from weibo.items import WeiboItem + + +class SearchSpider(scrapy.Spider): + name = 'search' + allowed_domains = ['weibo.com'] + base_url = 'https://s.weibo.com' + + @classmethod + def from_crawler(cls, crawler, *args, **kwargs): + spider = super().from_crawler(crawler, *args, **kwargs) + spider.configure(crawler.settings) + return spider + + def __init__(self, *args, settings=None, **kwargs): + super().__init__(*args, **kwargs) + self.configure(settings or get_project_settings()) + + def configure(self, settings): + self.project_settings = settings + self.keyword_list = self.load_keyword_list( + self.project_settings.get('KEYWORD_LIST')) + self.weibo_type = util.convert_weibo_type( + self.project_settings.get('WEIBO_TYPE')) + self.contain_type = util.convert_contain_type( + self.project_settings.get('CONTAIN_TYPE')) + self.regions = util.get_regions(self.project_settings.get('REGION')) + self.start_date = self.project_settings.get( + 'START_DATE', datetime.now().strftime('%Y-%m-%d')) + self.end_date = self.project_settings.get( + 'END_DATE', datetime.now().strftime('%Y-%m-%d')) + self.further_threshold = int( + self.project_settings.get('FURTHER_THRESHOLD', 46)) + self.limit_result = int(self.project_settings.get('LIMIT_RESULT', 0)) + self.fetch_ip = bool(self.project_settings.getbool('FETCH_IP', False)) + self.ip_request_timeout = int( + self.project_settings.get('IP_REQUEST_TIMEOUT', 5)) + self.ip_request_delay = float( + self.project_settings.get('IP_REQUEST_DELAY', 6.0)) + self.ip_session = requests.Session() + self.ip_failure_counts = {} + self.ip_cache = {} + self.last_ip_request_at = 0.0 + self.result_count = 0 + self.mongo_error = False + self.pymongo_error = False + self.mysql_error = False + self.pymysql_error = False + self.sqlite3_error = False + + def load_keyword_list(self, keyword_list): + """Load keywords from settings or a UTF-8 text file.""" + if not keyword_list: + raise CloseSpider('KEYWORD_LIST不能为空，请在settings.py中配置关键词') + if not isinstance(keyword_list, list): + if not os.path.isabs(keyword_list): + keyword_list = os.path.join(os.getcwd(), keyword_list) + if not os.path.isfile(keyword_list): + raise CloseSpider('不存在%s文件' % keyword_list) + try: + keyword_list = util.get_keyword_list(keyword_list) + except ValueError as exc: + raise CloseSpider(str(exc)) from exc + keywords = [] + for keyword in keyword_list: + if not keyword: + continue + if len(keyword) > 2 and keyword[0] == '#' and keyword[-1] == '#': + keyword = '%23' + keyword[1:-1] + '%23' + keywords.append(keyword) + if not keywords: + raise CloseSpider('KEYWORD_LIST不能为空，请至少配置一个关键词') + return keywords + + def validate_runtime_settings(self): + """Validate settings that are only required when the crawl starts.""" + headers = self.project_settings.get('DEFAULT_REQUEST_HEADERS') or {} + cookie = headers.get('cookie', '') + if not cookie or cookie == 'your_cookie_here': + raise CloseSpider( + '未配置微博Cookie。请设置环境变量WEIBO_COOKIE，或在本地settings.py中配置DEFAULT_REQUEST_HEADERS["cookie"]') + if util.str_to_time(self.start_date) > util.str_to_time(self.end_date): + raise CloseSpider( + 'settings.py配置错误，START_DATE值应早于或等于END_DATE值，请重新配置settings.py') + + def check_limit(self): + """检查是否达到爬取结果数量限制""" + if self.limit_result > 0 and self.result_count >= self.limit_result: + self.logger.info('已达到爬取结果数量限制：%s条，停止爬取', self.limit_result) + raise CloseSpider('已达到爬取结果数量限制') + return False + + async def start(self): + """Scrapy 2.13+ entrypoint; keep start_requests as the URL builder.""" + try: + for request in self.start_requests(): + yield request + except CloseSpider as exc: + self.logger.error('爬虫启动失败: %s', exc.reason or exc) + return + + def start_requests(self): + self.validate_runtime_settings() + start_date = datetime.strptime(self.start_date, '%Y-%m-%d') + end_date = datetime.strptime(self.end_date, + '%Y-%m-%d') + timedelta(days=1) + start_str = start_date.strftime('%Y-%m-%d') + '-0' + end_str = end_date.strftime('%Y-%m-%d') + '-0' + for keyword in self.keyword_list: + if not self.project_settings.get('REGION') or '全部' in self.project_settings.get( + 'REGION'): + base_url = 'https://s.weibo.com/weibo?q=%s' % keyword + url = base_url + self.weibo_type + url += self.contain_type + url += '×cope=custom:{}:{}'.format(start_str, end_str) + yield scrapy.Request(url=url, + callback=self.parse, + meta={ + 'base_url': base_url, + 'keyword': keyword + }) + else: + for region in self.regions.values(): + base_url = ( + 'https://s.weibo.com/weibo?q={}®ion=custom:{}:1000' + ).format(keyword, region['code']) + url = base_url + self.weibo_type + url += self.contain_type + url += '×cope=custom:{}:{}'.format(start_str, end_str) + # 获取一个省的搜索结果 + yield scrapy.Request(url=url, + callback=self.parse, + meta={ + 'base_url': base_url, + 'keyword': keyword, + 'province': region + }) + + def check_environment(self): + """判断配置要求的软件是否已安装""" + if self.pymongo_error: + self.logger.error('系统中可能没有安装pymongo库，请先运行 pip install pymongo ，再运行程序') + raise CloseSpider() + if self.mongo_error: + self.logger.error('系统中可能没有安装或启动MongoDB数据库，请先根据系统环境安装或启动MongoDB，再运行程序') + raise CloseSpider() + if self.pymysql_error: + self.logger.error('系统中可能没有安装pymysql库，请先运行 pip install pymysql ，再运行程序') + raise CloseSpider() + if self.mysql_error: + self.logger.error('系统中可能没有安装或正确配置MySQL数据库，请先根据系统环境安装或配置MySQL，再运行程序') + raise CloseSpider() + if self.sqlite3_error: + self.logger.error( + '系统中可能没有安装或正确配置SQLite3数据库，请检查SQLITE_DATABASE配置后再运行程序') + raise CloseSpider() + + def parse(self, response): + base_url = response.meta.get('base_url') + keyword = response.meta.get('keyword') + province = response.meta.get('province') + is_empty = response.xpath( + '//div[@class="card card-no-result s-pt20b40"]') + page_count = len(response.xpath('//ul[@class="s-scroll"]/li')) + if is_empty: + self.logger.info('当前页面搜索结果为空: %s', response.url) + elif page_count < self.further_threshold: + # 解析当前页面 + for weibo in self.parse_weibo(response): + self.check_environment() + # 检查是否达到爬取结果数量限制 + if self.check_limit(): + return + yield weibo + next_url = response.xpath( + '//a[@class="next"]/@href').extract_first() + if next_url: + # 检查是否达到爬取结果数量限制 + if self.check_limit(): + return + next_url = self.base_url + next_url + yield scrapy.Request(url=next_url, + callback=self.parse_page, + meta=response.meta) + else: + start_date = datetime.strptime(self.start_date, '%Y-%m-%d') + end_date = datetime.strptime(self.end_date, '%Y-%m-%d') + while start_date <= end_date: + start_str = start_date.strftime('%Y-%m-%d') + '-0' + start_date = start_date + timedelta(days=1) + end_str = start_date.strftime('%Y-%m-%d') + '-0' + url = base_url + self.weibo_type + url += self.contain_type + url += '×cope=custom:{}:{}&page=1'.format( + start_str, end_str) + # 获取一天的搜索结果 + yield scrapy.Request(url=url, + callback=self.parse_by_day, + meta={ + 'base_url': base_url, + 'keyword': keyword, + 'province': province, + 'date': start_str[:-2] + }) + + def parse_by_day(self, response): + """以天为单位筛选""" + base_url = response.meta.get('base_url') + keyword = response.meta.get('keyword') + province = response.meta.get('province') + is_empty = response.xpath( + '//div[@class="card card-no-result s-pt20b40"]') + date = response.meta.get('date') + page_count = len(response.xpath('//ul[@class="s-scroll"]/li')) + if is_empty: + self.logger.info('当前页面搜索结果为空: %s', response.url) + elif page_count < self.further_threshold: + # 解析当前页面 + for weibo in self.parse_weibo(response): + self.check_environment() + # 检查是否达到爬取结果数量限制 + if self.check_limit(): + return + yield weibo + next_url = response.xpath( + '//a[@class="next"]/@href').extract_first() + if next_url: + # 检查是否达到爬取结果数量限制 + if self.check_limit(): + return + next_url = self.base_url + next_url + yield scrapy.Request(url=next_url, + callback=self.parse_page, + meta=response.meta) + else: + start_date_str = date + '-0' + start_date = datetime.strptime(start_date_str, '%Y-%m-%d-%H') + for i in range(1, 25): + start_str = start_date.strftime('%Y-%m-%d-X%H').replace( + 'X0', 'X').replace('X', '') + start_date = start_date + timedelta(hours=1) + end_str = start_date.strftime('%Y-%m-%d-X%H').replace( + 'X0', 'X').replace('X', '') + url = base_url + self.weibo_type + url += self.contain_type + url += '×cope=custom:{}:{}&page=1'.format( + start_str, end_str) + # 获取一小时的搜索结果 + yield scrapy.Request(url=url, + callback=self.parse_by_hour_province + if province else self.parse_by_hour, + meta={ + 'base_url': base_url, + 'keyword': keyword, + 'province': province, + 'start_time': start_str, + 'end_time': end_str + }) + + def parse_by_hour(self, response): + """以小时为单位筛选""" + keyword = response.meta.get('keyword') + is_empty = response.xpath( + '//div[@class="card card-no-result s-pt20b40"]') + start_time = response.meta.get('start_time') + end_time = response.meta.get('end_time') + page_count = len(response.xpath('//ul[@class="s-scroll"]/li')) + if is_empty: + self.logger.info('当前页面搜索结果为空: %s', response.url) + elif page_count < self.further_threshold: + # 解析当前页面 + for weibo in self.parse_weibo(response): + self.check_environment() + yield weibo + next_url = response.xpath( + '//a[@class="next"]/@href').extract_first() + if next_url: + next_url = self.base_url + next_url + yield scrapy.Request(url=next_url, + callback=self.parse_page, + meta=response.meta) + else: + for region in self.regions.values(): + url = ('https://s.weibo.com/weibo?q={}®ion=custom:{}:1000' + ).format(keyword, region['code']) + url += self.weibo_type + url += self.contain_type + url += '×cope=custom:{}:{}&page=1'.format( + start_time, end_time) + # 获取一小时一个省的搜索结果 + yield scrapy.Request(url=url, + callback=self.parse_by_hour_province, + meta={ + 'keyword': keyword, + 'start_time': start_time, + 'end_time': end_time, + 'province': region + }) + + def parse_by_hour_province(self, response): + """以小时和直辖市/省为单位筛选""" + keyword = response.meta.get('keyword') + is_empty = response.xpath( + '//div[@class="card card-no-result s-pt20b40"]') + start_time = response.meta.get('start_time') + end_time = response.meta.get('end_time') + province = response.meta.get('province') + page_count = len(response.xpath('//ul[@class="s-scroll"]/li')) + if is_empty: + self.logger.info('当前页面搜索结果为空: %s', response.url) + elif page_count < self.further_threshold: + # 解析当前页面 + for weibo in self.parse_weibo(response): + self.check_environment() + yield weibo + next_url = response.xpath( + '//a[@class="next"]/@href').extract_first() + if next_url: + next_url = self.base_url + next_url + yield scrapy.Request(url=next_url, + callback=self.parse_page, + meta=response.meta) + else: + for city in province['city'].values(): + url = ('https://s.weibo.com/weibo?q={}®ion=custom:{}:{}' + ).format(keyword, province['code'], city) + url += self.weibo_type + url += self.contain_type + url += '×cope=custom:{}:{}&page=1'.format( + start_time, end_time) + # 获取一小时一个城市的搜索结果 + yield scrapy.Request(url=url, + callback=self.parse_page, + meta={ + 'keyword': keyword, + 'start_time': start_time, + 'end_time': end_time, + 'province': province, + 'city': city + }) + + def parse_page(self, response): + """解析一页搜索结果的信息""" + keyword = response.meta.get('keyword') + is_empty = response.xpath( + '//div[@class="card card-no-result s-pt20b40"]') + if is_empty: + self.logger.info('当前页面搜索结果为空: %s', response.url) + else: + for weibo in self.parse_weibo(response): + self.check_environment() + # 检查是否达到爬取结果数量限制 + if self.check_limit(): + return + yield weibo + next_url = response.xpath( + '//a[@class="next"]/@href').extract_first() + if next_url: + # 检查是否达到爬取结果数量限制 + if self.check_limit(): + return + next_url = self.base_url + next_url + yield scrapy.Request(url=next_url, + callback=self.parse_page, + meta=response.meta) + + def record_ip_failure(self, reason, bid): + count = self.ip_failure_counts.get(reason, 0) + 1 + self.ip_failure_counts[reason] = count + if count == 1: + self.logger.error( + 'IP属地请求失败（%s），首条 bid=%s；后续同类错误将静默统计', + reason, bid) + + @staticmethod + def normalize_header_value(value): + if isinstance(value, (list, tuple)): + value = '; '.join(str(item) for item in value) + elif isinstance(value, bytes): + value = value.decode('latin-1') + else: + value = str(value or '') + return value.replace('\r', '').replace('\n', '').strip() + + def get_ip_headers(self, bid, mobile=False): + default_headers = self.project_settings.get( + 'DEFAULT_REQUEST_HEADERS') or {} + cookie = default_headers.get('cookie', + default_headers.get('Cookie', '')) + cookie = self.normalize_header_value(cookie) + if cookie.lower().startswith('cookie:'): + cookie = cookie[7:].strip() + + user_agents = self.project_settings.get('USER_AGENT_LIST') or [] + user_agent = user_agents[0] if user_agents else 'Mozilla/5.0' + headers = { + 'Accept': 'application/json, text/plain, */*', + 'Cookie': cookie, + 'Referer': f'https://weibo.com/detail/{bid}', + 'User-Agent': self.normalize_header_value(user_agent), + 'X-Requested-With': 'XMLHttpRequest', + } + if mobile: + headers.update({ + 'MWeibo-Pwa': '1', + 'Referer': f'https://m.weibo.cn/status/{bid}', + 'User-Agent': ( + 'Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) ' + 'AppleWebKit/605.1.15 (KHTML, like Gecko) ' + 'Version/17.5 Mobile/15E148 Safari/604.1'), + }) + return headers + + def wait_for_ip_request(self): + if self.ip_request_delay <= 0: + return + elapsed = time.monotonic() - self.last_ip_request_at + wait_seconds = self.ip_request_delay - elapsed + if wait_seconds > 0: + time.sleep(wait_seconds) + self.last_ip_request_at = time.monotonic() + + def request_ip_json(self, url, bid, mobile=False): + self.wait_for_ip_request() + try: + response = self.ip_session.get( + url, + headers=self.get_ip_headers(bid, mobile=mobile), + timeout=self.ip_request_timeout) + except requests.RequestException as exc: + return None, type(exc).__name__ + if response.status_code != 200: + return None, f'HTTP {response.status_code}' + try: + return response.json(), '' + except ValueError: + return None, '响应不是JSON' + + @staticmethod + def extract_ip_region(data): + if not isinstance(data, dict): + return '' + candidates = [data] + for key in ('data', 'status', 'mblog'): + nested = data.get(key) + if isinstance(nested, dict): + candidates.append(nested) + for candidate in candidates: + region = candidate.get('region_name', '') + if isinstance(region, str) and region.strip(): + return re.sub(r'^发布于\s*', '', region).strip() + return '' + + def get_ip(self, bid, weibo_id=''): + if not self.fetch_ip or not bid: + return "" + cache_key = str(weibo_id or bid) + if cache_key in self.ip_cache: + return self.ip_cache[cache_key] + + endpoints = [( + '网页接口', + f"https://weibo.com/ajax/statuses/show?id={bid}&locale=zh-CN", + False, + )] + if weibo_id: + endpoints.append(( + '移动端接口', + f"https://m.weibo.cn/statuses/show?id={weibo_id}", + True, + )) + + failures = [] + for name, url, mobile in endpoints: + data, error = self.request_ip_json( + url, bid, mobile=mobile) + if data is not None: + region = self.extract_ip_region(data) + if region: + self.ip_cache[cache_key] = region + return region + error = '缺少region_name' + failures.append(f'{name}{error}') + + self.record_ip_failure('；'.join(failures), bid) + self.ip_cache[cache_key] = '' + return '' + + def get_article_url(self, selector): + """获取微博头条文章url""" + article_url = '' + text = (selector.xpath('string(.)').extract_first() or '').replace( + '\u200b', '').replace('\ue627', '').replace('\n', + '').replace(' ', '') + if text.startswith('发布了头条文章'): + urls = selector.xpath('.//a') + for url in urls: + if url.xpath( + 'i[@class="wbicon"]/text()').extract_first() == 'O': + if url.xpath('@href').extract_first() and url.xpath( + '@href').extract_first().startswith('http://t.cn'): + article_url = url.xpath('@href').extract_first() + break + return article_url + + def get_location(self, selector): + """获取微博发布位置""" + a_list = selector.xpath('.//a') + location = '' + for a in a_list: + if a.xpath('./i[@class="wbicon"]') and a.xpath( + './i[@class="wbicon"]/text()').extract_first() == '2': + location = a.xpath('string(.)').extract_first()[1:] + break + return location + + def get_at_users(self, selector): + """获取微博中@的用户昵称""" + a_list = selector.xpath('.//a') + at_users = '' + at_list = [] + for a in a_list: + href = a.xpath('@href').extract_first() or '' + text = a.xpath('string(.)').extract_first() or '' + if len(unquote(href)) > 14 and len(text) > 1: + if unquote(href)[14:] == text[1:]: + at_user = text[1:] + if at_user not in at_list: + at_list.append(at_user) + if at_list: + at_users = ','.join(at_list) + return at_users + + def get_topics(self, selector): + """获取参与的微博话题""" + a_list = selector.xpath('.//a') + topics = '' + topic_list = [] + for a in a_list: + text = a.xpath('string(.)').extract_first() or '' + if len(text) > 2 and text[0] == '#' and text[-1] == '#': + if text[1:-1] not in topic_list: + topic_list.append(text[1:-1]) + if topic_list: + topics = ','.join(topic_list) + return topics + + def get_vip(self, selector): + """获取用户的VIP类型和等级信息""" + vip_type = "非会员" + vip_level = 0 + + vip_container = selector.xpath('.//div[@class="user_vip_icon_container"]') + if vip_container: + svvip_img = vip_container.xpath('.//img[contains(@src, "svvip_")]') + if svvip_img: + vip_type = "超级会员" + src = svvip_img.xpath('@src').extract_first() or '' + level_match = re.search(r'svvip_(\d+)\.png', src) + if level_match: + vip_level = int(level_match.group(1)) + else: + vip_img = vip_container.xpath('.//img[contains(@src, "vip_")]') + if vip_img: + vip_type = "会员" + src = vip_img.xpath('@src').extract_first() or '' + level_match = re.search(r'vip_(\d+)\.png', src) + if level_match: + vip_level = int(level_match.group(1)) + + return vip_type, vip_level + + def extract_count(self, text): + """Extract Weibo count text; missing or non-numeric labels become 0.""" + matches = re.findall(r'\d+.*', text or '') + return matches[0] if matches else '0' + + def clean_weibo_text(self, selector, is_long=False): + text = (selector.xpath('string(.)').extract_first() or '').replace( + '\u200b', '').replace('\ue627', '') + location = self.get_location(selector) + if location: + text = text.replace('2' + location, '') + text = text[2:].replace(' ', '') if len(text) >= 2 else text.strip() + if is_long and len(text) >= 4: + text = text[:-4] + return text, location + + def parse_weibo(self, response): + """解析网页中的微博信息""" + keyword = response.meta.get('keyword') + for sel in response.xpath("//div[@class='card-wrap']"): + # 检查是否达到爬取结果数量限制 + if self.check_limit(): + return + + info = sel.xpath( + "div[@class='card']/div[@class='card-feed']/div[@class='content']/div[@class='info']" + ) + if info: + weibo = WeiboItem() + weibo['id'] = sel.xpath('@mid').extract_first() + from_href = sel.xpath( + './/div[@class="from"]/a[1]/@href').extract_first() + user_href = info[0].xpath('div[2]/a/@href').extract_first() + txt_nodes = sel.xpath('.//p[@class="txt"]') + if not weibo['id'] or not from_href or not user_href or not txt_nodes: + self.logger.warning('跳过无法解析的微博卡片: %s', response.url) + continue + bid = from_href.split('/')[-1].split('?')[0] + weibo['bid'] = bid + weibo['user_id'] = user_href.split('?')[0].split('/')[-1] + weibo['screen_name'] = info[0].xpath( + 'div[2]/a/@nick-name').extract_first() or '' + # 获取VIP信息 + weibo['vip_type'], weibo['vip_level'] = self.get_vip(info[0]) + txt_sel = txt_nodes[0] + retweet_sel = sel.xpath('.//div[@class="card-comment"]') + retweet_txt_sel = '' + if retweet_sel and retweet_sel[0].xpath('.//p[@class="txt"]'): + retweet_txt_sel = retweet_sel[0].xpath( + './/p[@class="txt"]')[0] + content_full = sel.xpath( + './/p[@node-type="feed_list_content_full"]') + + is_long_weibo = False + is_long_retweet = False + if content_full: + if not retweet_sel: + txt_sel = content_full[0] + is_long_weibo = True + elif len(content_full) == 2: + txt_sel = content_full[0] + retweet_txt_sel = content_full[1] + is_long_weibo = True + is_long_retweet = True + elif retweet_sel[0].xpath( + './/p[@node-type="feed_list_content_full"]'): + retweet_txt_sel = retweet_sel[0].xpath( + './/p[@node-type="feed_list_content_full"]')[0] + is_long_retweet = True + else: + txt_sel = content_full[0] + is_long_weibo = True + weibo['article_url'] = self.get_article_url(txt_sel) + weibo['text'], weibo['location'] = self.clean_weibo_text( + txt_sel, is_long_weibo) + weibo['at_users'] = self.get_at_users(txt_sel) + weibo['topics'] = self.get_topics(txt_sel) + reposts_count = sel.xpath( + './/a[@action-type="feed_list_forward"]/text()').extract() + reposts_count = "".join(reposts_count) + weibo['reposts_count'] = self.extract_count(reposts_count) + comments_count = sel.xpath( + './/a[@action-type="feed_list_comment"]/text()' + ).extract_first() + weibo['comments_count'] = self.extract_count(comments_count) + attitudes_count = sel.xpath( + './/a[@action-type="feed_list_like"]/button/span[2]/text()').extract_first() + weibo['attitudes_count'] = self.extract_count(attitudes_count) + created_at = sel.xpath( + './/div[@class="from"]/a[1]/text()').extract_first() + created_at = (created_at or '').replace(' ', '').replace( + '\n', '').split('前')[0] + weibo['created_at'] = util.standardize_date( + created_at) if created_at else '' + source = sel.xpath('.//div[@class="from"]/a[2]/text()' + ).extract_first() + weibo['source'] = source if source else '' + pics = [] + is_exist_pic = sel.xpath( + './/div[@class="media media-piclist"]') + if is_exist_pic: + pics = is_exist_pic[0].xpath('ul[1]/li/img/@src').extract() + pics = [pic[8:] for pic in pics] + pics = [ + re.sub(r'/.*?/', '/large/', pic, 1) for pic in pics + ] + pics = ['https://' + pic for pic in pics] + video_url = '' + is_exist_video = sel.xpath( + './/div[@class="thumbnail"]//video-player').extract_first() + if is_exist_video: + video_matches = re.findall(r'src:\'(.*?)\'', is_exist_video) + if video_matches: + video_url = video_matches[0].replace('&', '&') + video_url = 'http:' + video_url + if not retweet_sel: + weibo['pics'] = pics + weibo['video_url'] = video_url + else: + weibo['pics'] = [] + weibo['video_url'] = '' + weibo['retweet_id'] = '' + if retweet_sel and retweet_sel[0].xpath( + './/div[@node-type="feed_list_forwardContent"]/a[1]'): + retweet_id_data = retweet_sel[0].xpath( + './/a[@action-type="feed_list_like"]/@action-data' + ).extract_first() or '' + retweet_from_href = retweet_sel[0].xpath( + './/p[@class="from"]/a/@href').extract_first() or '' + retweet_info = retweet_sel[0].xpath( + './/div[@node-type="feed_list_forwardContent"]/a[1]' + ) + if not retweet_id_data.startswith('mid=') or not retweet_from_href or not retweet_info or not retweet_txt_sel: + self.logger.warning('跳过无法解析的转发微博: %s', response.url) + else: + retweet = WeiboItem() + retweet['id'] = retweet_id_data[4:] + retweet['bid'] = retweet_from_href.split( + '/')[-1].split('?')[0] + info = retweet_info[0] + retweet_user_href = info.xpath( + '@href').extract_first() or '' + retweet['user_id'] = retweet_user_href.split('/')[-1] + retweet['screen_name'] = info.xpath( + '@nick-name').extract_first() or '' + retweet['vip_type'], retweet['vip_level'] = self.get_vip( + info) + retweet['article_url'] = self.get_article_url( + retweet_txt_sel) + retweet['text'], retweet['location'] = self.clean_weibo_text( + retweet_txt_sel, is_long_retweet) + retweet['at_users'] = self.get_at_users(retweet_txt_sel) + retweet['topics'] = self.get_topics(retweet_txt_sel) + reposts_count = retweet_sel[0].xpath( + './/ul[@class="act s-fr"]/li[1]/a[1]/text()' + ).extract_first() + retweet['reposts_count'] = self.extract_count( + reposts_count) + comments_count = retweet_sel[0].xpath( + './/ul[@class="act s-fr"]/li[2]/a[1]/text()' + ).extract_first() + retweet['comments_count'] = self.extract_count( + comments_count) + attitudes_count = retweet_sel[0].xpath( + './/a[@class="woo-box-flex woo-box-alignCenter woo-box-justifyCenter"]//span[@class="woo-like-count"]/text()' + ).extract_first() + retweet['attitudes_count'] = self.extract_count( + attitudes_count) + created_at = retweet_sel[0].xpath( + './/p[@class="from"]/a[1]/text()').extract_first() + created_at = (created_at or '').replace(' ', '').replace( + '\n', '').split('前')[0] + retweet['created_at'] = util.standardize_date( + created_at) if created_at else '' + source = retweet_sel[0].xpath( + './/p[@class="from"]/a[2]/text()').extract_first() + retweet['source'] = source if source else '' + retweet['pics'] = pics + retweet['video_url'] = video_url + retweet['retweet_id'] = '' + retweet['ip'] = self.get_ip( + retweet['bid'], retweet['id']) + retweet['user_authentication'] = '' + + self.result_count += 1 + + yield {'weibo': retweet, 'keyword': keyword} + + if self.check_limit(): + return + + weibo['retweet_id'] = retweet['id'] + weibo["ip"] = self.get_ip(bid, weibo['id']) + + avator = sel.xpath( + "div[@class='card']/div[@class='card-feed']/div[@class='avator']" + ) + if avator: + user_auth = avator.xpath('.//svg/@id').extract_first() + if user_auth == 'woo_svg_vblue': + weibo['user_authentication'] = '蓝V' + elif user_auth == 'woo_svg_vyellow': + weibo['user_authentication'] = '黄V' + elif user_auth == 'woo_svg_vorange': + weibo['user_authentication'] = '红V' + elif user_auth == 'woo_svg_vgold': + weibo['user_authentication'] = '金V' + else: + weibo['user_authentication'] = '普通用户' + else: + weibo['user_authentication'] = '普通用户' + + # 增加结果计数（主微博） + self.result_count += 1 + + yield {'weibo': weibo, 'keyword': keyword} + + # 检查是否达到爬取结果数量限制 + if self.check_limit(): + return diff --git a/weibo/utils/region.py b/weibo/utils/region.py index 7ba9351..af9e93e 100644 --- a/weibo/utils/region.py +++ b/weibo/utils/region.py @@ -1,639 +1,639 @@ -region_dict = { - "安徽": { - "code": 34, - "city": { - "合肥": 1, - "芜湖": 2, - "蚌埠": 3, - "淮南": 4, - "马鞍山": 5, - "淮北": 6, - "铜陵": 7, - "安庆": 8, - "黄山": 10, - "滁州": 11, - "阜阳": 12, - "宿州": 13, - "巢湖": 14, - "六安": 15, - "亳州": 16, - "池州": 17, - "宣城": 18 - } - }, - "北京": { - "code": 11, - "city": { - "东城区": 1, - "西城区": 2, - "崇文区": 3, - "宣武区": 4, - "朝阳区": 5, - "丰台区": 6, - "石景山区": 7, - "海淀区": 8, - "门头沟区": 9, - "房山区": 11, - "通州区": 12, - "顺义区": 13, - "昌平区": 14, - "大兴区": 15, - "怀柔区": 16, - "平谷区": 17, - "密云县": 28, - "延庆县": 29 - } - }, - "重庆": { - "code": 50, - "city": { - "万州区": 1, - "涪陵区": 2, - "渝中区": 3, - "大渡口区": 4, - "江北区": 5, - "沙坪坝区": 6, - "九龙坡区": 7, - "南岸区": 8, - "北碚区": 9, - "万盛区": 10, - "双桥区": 11, - "渝北区": 12, - "巴南区": 13, - "黔江区": 14, - "长寿区": 15, - "綦江县": 22, - "潼南县": 23, - "铜梁县": 24, - "大足县": 25, - "荣昌县": 26, - "璧山县": 27, - "梁平县": 28, - "城口县": 29, - "丰都县": 30, - "垫江县": 31, - "武隆县": 32, - "忠县": 33, - "开县": 34, - "云阳县": 35, - "奉节县": 36, - "巫山县": 37, - "巫溪县": 38, - "石柱土家族自治县": 40, - "秀山土家族苗族自治县": 41, - "酉阳土家族苗族自治县": 42, - "彭水苗族土家族自治县": 43, - "江津区": 81, - "合川市": 82, - "永川区": 83, - "南川市": 84 - } - }, - "福建": { - "code": 35, - "city": { - "福州": 1, - "厦门": 2, - "莆田": 3, - "三明": 4, - "泉州": 5, - "漳州": 6, - "南平": 7, - "龙岩": 8, - "宁德": 9 - } - }, - "甘肃": { - "code": 62, - "city": { - "兰州": 1, - "嘉峪关": 2, - "金昌": 3, - "白银": 4, - "天水": 5, - "武威": 6, - "张掖": 7, - "平凉": 8, - "酒泉": 9, - "庆阳": 10, - "定西": 24, - "陇南": 26, - "临夏": 29, - "甘南": 30 - } - }, - "广东": { - "code": 44, - "city": { - "广州": 1, - "韶关": 2, - "深圳": 3, - "珠海": 4, - "汕头": 5, - "佛山": 6, - "江门": 7, - "湛江": 8, - "茂名": 9, - "肇庆": 12, - "惠州": 13, - "梅州": 14, - "汕尾": 15, - "河源": 16, - "阳江": 17, - "清远": 18, - "东莞": 19, - "中山": 20, - "潮州": 51, - "揭阳": 52, - "云浮": 53 - } - }, - "广西": { - "code": 45, - "city": { - "南宁": 1, - "柳州": 2, - "桂林": 3, - "梧州": 4, - "北海": 5, - "防城港": 6, - "钦州": 7, - "贵港": 8, - "玉林": 9, - "百色": 10, - "贺州": 11, - "河池": 12, - "来宾": 13, - "崇左": 14 - } - }, - "贵州": { - "code": 52, - "city": { - "贵阳": 1, - "六盘水": 2, - "遵义": 3, - "安顺": 4, - "铜仁": 22, - "黔西南": 23, - "毕节": 24, - "黔东南": 26, - "黔南": 27 - } - }, - "海南": { - "code": 46, - "city": { - "海口": 1, - "三亚": 2, - "其他": 90 - } - }, - "河北": { - "code": 13, - "city": { - "石家庄": 1, - "唐山": 2, - "秦皇岛": 3, - "邯郸": 4, - "邢台": 5, - "保定": 6, - "张家口": 7, - "承德": 8, - "沧州": 9, - "廊坊": 10, - "衡水": 11 - } - }, - "黑龙江": { - "code": 23, - "city": { - "哈尔滨": 1, - "齐齐哈尔": 2, - "鸡西": 3, - "鹤岗": 4, - "双鸭山": 5, - "大庆": 6, - "伊春": 7, - "佳木斯": 8, - "七台河": 9, - "牡丹江": 10, - "黑河": 11, - "绥化": 12, - "大兴安岭": 27 - } - }, - "河南": { - "code": 41, - "city": { - "郑州": 1, - "开封": 2, - "洛阳": 3, - "平顶山": 4, - "安阳": 5, - "鹤壁": 6, - "新乡": 7, - "焦作": 8, - "濮阳": 9, - "许昌": 10, - "漯河": 11, - "三门峡": 12, - "南阳": 13, - "商丘": 14, - "信阳": 15, - "周口": 16, - "驻马店": 17 - } - }, - "湖北": { - "code": 42, - "city": { - "武汉": 1, - "黄石": 2, - "十堰": 3, - "宜昌": 5, - "襄阳": 6, - "鄂州": 7, - "荆门": 8, - "孝感": 9, - "荆州": 10, - "黄冈": 11, - "咸宁": 12, - "随州": 13, - "恩施土家族苗族自治州": 28 - } - }, - "湖南": { - "code": 43, - "city": { - "长沙": 1, - "株洲": 2, - "湘潭": 3, - "衡阳": 4, - "邵阳": 5, - "岳阳": 6, - "常德": 7, - "张家界": 8, - "益阳": 9, - "郴州": 10, - "永州": 11, - "怀化": 12, - "娄底": 13, - "湘西土家族苗族自治州": 31 - } - }, - "内蒙古": { - "code": 15, - "city": { - "呼和浩特": 1, - "包头": 2, - "乌海": 3, - "赤峰": 4, - "通辽": 5, - "鄂尔多斯": 6, - "呼伦贝尔": 7, - "兴安盟": 22, - "锡林郭勒盟": 25, - "乌兰察布盟": 26, - "巴彦淖尔盟": 28, - "阿拉善盟": 29 - } - }, - "江苏": { - "code": 32, - "city": { - "南京": 1, - "无锡": 2, - "徐州": 3, - "常州": 4, - "苏州": 5, - "南通": 6, - "连云港": 7, - "淮安": 8, - "盐城": 9, - "扬州": 10, - "镇江": 11, - "泰州": 12, - "宿迁": 13 - } - }, - "江西": { - "code": 36, - "city": { - "南昌": 1, - "景德镇": 2, - "萍乡": 3, - "九江": 4, - "新余": 5, - "鹰潭": 6, - "赣州": 7, - "吉安": 8, - "宜春": 9, - "抚州": 10, - "上饶": 11 - } - }, - "吉林": { - "code": 22, - "city": { - "长春": 1, - "吉林": 2, - "四平": 3, - "辽源": 4, - "通化": 5, - "白山": 6, - "松原": 7, - "白城": 8, - "延边朝鲜族自治州": 24 - } - }, - "辽宁": { - "code": 21, - "city": { - "沈阳": 1, - "大连": 2, - "鞍山": 3, - "抚顺": 4, - "本溪": 5, - "丹东": 6, - "锦州": 7, - "营口": 8, - "阜新": 9, - "辽阳": 10, - "盘锦": 11, - "铁岭": 12, - "朝阳": 13, - "葫芦岛": 14 - } - }, - "宁夏": { - "code": 64, - "city": { - "银川": 1, - "石嘴山": 2, - "吴忠": 3, - "固原": 4, - "中卫": 5 - } - }, - "青海": { - "code": 63, - "city": { - "西宁": 1, - "海东": 21, - "海北": 22, - "黄南": 23, - "海南": 25, - "果洛": 26, - "玉树": 27, - "海西": 28 - } - }, - "山西": { - "code": 14, - "city": { - "太原": 1, - "大同": 2, - "阳泉": 3, - "长治": 4, - "晋城": 5, - "朔州": 6, - "晋中": 7, - "运城": 8, - "忻州": 9, - "临汾": 10, - "吕梁": 23 - } - }, - "山东": { - "code": 37, - "city": { - "济南": 1, - "青岛": 2, - "淄博": 3, - "枣庄": 4, - "东营": 5, - "烟台": 6, - "潍坊": 7, - "济宁": 8, - "泰安": 9, - "威海": 10, - "日照": 11, - "莱芜": 12, - "临沂": 13, - "德州": 14, - "聊城": 15, - "滨州": 16, - "菏泽": 17 - } - }, - "上海": { - "code": 31, - "city": { - "黄浦区": 1, - "卢湾区": 3, - "徐汇区": 4, - "长宁区": 5, - "静安区": 6, - "普陀区": 7, - "闸北区": 8, - "虹口区": 9, - "杨浦区": 10, - "闵行区": 12, - "宝山区": 13, - "嘉定区": 14, - "浦东新区": 15, - "金山区": 16, - "松江区": 17, - "青浦区": 18, - "南汇区": 19, - "奉贤区": 20, - "崇明县": 30 - } - }, - "四川": { - "code": 51, - "city": { - "成都": 1, - "自贡": 3, - "攀枝花": 4, - "泸州": 5, - "德阳": 6, - "绵阳": 7, - "广元": 8, - "遂宁": 9, - "内江": 10, - "乐山": 11, - "南充": 13, - "眉山": 14, - "宜宾": 15, - "广安": 16, - "达州": 17, - "雅安": 18, - "巴中": 19, - "资阳": 20, - "阿坝": 32, - "甘孜": 33, - "凉山": 34 - } - }, - "天津": { - "code": 12, - "city": { - "和平区": 1, - "河东区": 2, - "河西区": 3, - "南开区": 4, - "河北区": 5, - "红桥区": 6, - "塘沽区": 7, - "汉沽区": 8, - "大港区": 9, - "东丽区": 10, - "西青区": 11, - "津南区": 12, - "北辰区": 13, - "武清区": 14, - "宝坻区": 15, - "宁河县": 21, - "静海县": 23, - "蓟县": 25 - } - }, - "西藏": { - "code": 54, - "city": { - "拉萨": 1, - "昌都": 21, - "山南": 22, - "日喀则": 23, - "那曲": 24, - "阿里": 25, - "林芝": 26 - } - }, - "新疆": { - "code": 65, - "city": { - "乌鲁木齐": 1, - "克拉玛依": 2, - "吐鲁番": 21, - "哈密": 22, - "昌吉": 23, - "博尔塔拉": 27, - "巴音郭楞": 28, - "阿克苏": 29, - "克孜勒苏": 30, - "喀什": 31, - "和田": 32, - "伊犁": 40, - "塔城": 42, - "阿勒泰": 43, - "石河子": 44 - } - }, - "云南": { - "code": 53, - "city": { - "昆明": 1, - "曲靖": 3, - "玉溪": 4, - "保山": 5, - "昭通": 6, - "楚雄": 23, - "红河": 25, - "文山": 26, - "思茅": 27, - "西双版纳": 28, - "大理": 29, - "德宏": 31, - "丽江": 32, - "怒江": 33, - "迪庆": 34, - "临沧": 35 - } - }, - "浙江": { - "code": 33, - "city": { - "杭州": 1, - "宁波": 2, - "温州": 3, - "嘉兴": 4, - "湖州": 5, - "绍兴": 6, - "金华": 7, - "衢州": 8, - "舟山": 9, - "台州": 10, - "丽水": 11 - } - }, - "陕西": { - "code": 61, - "city": { - "西安": 1, - "铜川": 2, - "宝鸡": 3, - "咸阳": 4, - "渭南": 5, - "延安": 6, - "汉中": 7, - "榆林": 8, - "安康": 9, - "商洛": 10 - } - }, - "台湾": { - "code": 71, - "city": { - "台北": 1, - "高雄": 2, - "基隆": 3, - "台中": 4, - "台南": 5, - "新竹": 6, - "嘉义": 7, - "其他": 90 - } - }, - "香港": { - "code": 81, - "city": { - "香港": 1 - } - }, - "澳门": { - "code": 82, - "city": { - "澳门": 1 - } - }, - "海外": { - "code": 400, - "city": { - "美国": 1, - "英国": 2, - "法国": 3, - "俄罗斯": 4, - "加拿大": 5, - "巴西": 6, - "澳大利亚": 7, - "印尼": 8, - "泰国": 9, - "马来西亚": 10, - "新加坡": 11, - "菲律宾": 12, - "越南": 13, - "印度": 14, - "日本": 15, - "其他": 16 - } - }, - "其他": { - "code": 100, - "city": { - "不限": 1000 - } - } -} +region_dict = { + "安徽": { + "code": 34, + "city": { + "合肥": 1, + "芜湖": 2, + "蚌埠": 3, + "淮南": 4, + "马鞍山": 5, + "淮北": 6, + "铜陵": 7, + "安庆": 8, + "黄山": 10, + "滁州": 11, + "阜阳": 12, + "宿州": 13, + "巢湖": 14, + "六安": 15, + "亳州": 16, + "池州": 17, + "宣城": 18 + } + }, + "北京": { + "code": 11, + "city": { + "东城区": 1, + "西城区": 2, + "崇文区": 3, + "宣武区": 4, + "朝阳区": 5, + "丰台区": 6, + "石景山区": 7, + "海淀区": 8, + "门头沟区": 9, + "房山区": 11, + "通州区": 12, + "顺义区": 13, + "昌平区": 14, + "大兴区": 15, + "怀柔区": 16, + "平谷区": 17, + "密云县": 28, + "延庆县": 29 + } + }, + "重庆": { + "code": 50, + "city": { + "万州区": 1, + "涪陵区": 2, + "渝中区": 3, + "大渡口区": 4, + "江北区": 5, + "沙坪坝区": 6, + "九龙坡区": 7, + "南岸区": 8, + "北碚区": 9, + "万盛区": 10, + "双桥区": 11, + "渝北区": 12, + "巴南区": 13, + "黔江区": 14, + "长寿区": 15, + "綦江县": 22, + "潼南县": 23, + "铜梁县": 24, + "大足县": 25, + "荣昌县": 26, + "璧山县": 27, + "梁平县": 28, + "城口县": 29, + "丰都县": 30, + "垫江县": 31, + "武隆县": 32, + "忠县": 33, + "开县": 34, + "云阳县": 35, + "奉节县": 36, + "巫山县": 37, + "巫溪县": 38, + "石柱土家族自治县": 40, + "秀山土家族苗族自治县": 41, + "酉阳土家族苗族自治县": 42, + "彭水苗族土家族自治县": 43, + "江津区": 81, + "合川市": 82, + "永川区": 83, + "南川市": 84 + } + }, + "福建": { + "code": 35, + "city": { + "福州": 1, + "厦门": 2, + "莆田": 3, + "三明": 4, + "泉州": 5, + "漳州": 6, + "南平": 7, + "龙岩": 8, + "宁德": 9 + } + }, + "甘肃": { + "code": 62, + "city": { + "兰州": 1, + "嘉峪关": 2, + "金昌": 3, + "白银": 4, + "天水": 5, + "武威": 6, + "张掖": 7, + "平凉": 8, + "酒泉": 9, + "庆阳": 10, + "定西": 24, + "陇南": 26, + "临夏": 29, + "甘南": 30 + } + }, + "广东": { + "code": 44, + "city": { + "广州": 1, + "韶关": 2, + "深圳": 3, + "珠海": 4, + "汕头": 5, + "佛山": 6, + "江门": 7, + "湛江": 8, + "茂名": 9, + "肇庆": 12, + "惠州": 13, + "梅州": 14, + "汕尾": 15, + "河源": 16, + "阳江": 17, + "清远": 18, + "东莞": 19, + "中山": 20, + "潮州": 51, + "揭阳": 52, + "云浮": 53 + } + }, + "广西": { + "code": 45, + "city": { + "南宁": 1, + "柳州": 2, + "桂林": 3, + "梧州": 4, + "北海": 5, + "防城港": 6, + "钦州": 7, + "贵港": 8, + "玉林": 9, + "百色": 10, + "贺州": 11, + "河池": 12, + "来宾": 13, + "崇左": 14 + } + }, + "贵州": { + "code": 52, + "city": { + "贵阳": 1, + "六盘水": 2, + "遵义": 3, + "安顺": 4, + "铜仁": 22, + "黔西南": 23, + "毕节": 24, + "黔东南": 26, + "黔南": 27 + } + }, + "海南": { + "code": 46, + "city": { + "海口": 1, + "三亚": 2, + "其他": 90 + } + }, + "河北": { + "code": 13, + "city": { + "石家庄": 1, + "唐山": 2, + "秦皇岛": 3, + "邯郸": 4, + "邢台": 5, + "保定": 6, + "张家口": 7, + "承德": 8, + "沧州": 9, + "廊坊": 10, + "衡水": 11 + } + }, + "黑龙江": { + "code": 23, + "city": { + "哈尔滨": 1, + "齐齐哈尔": 2, + "鸡西": 3, + "鹤岗": 4, + "双鸭山": 5, + "大庆": 6, + "伊春": 7, + "佳木斯": 8, + "七台河": 9, + "牡丹江": 10, + "黑河": 11, + "绥化": 12, + "大兴安岭": 27 + } + }, + "河南": { + "code": 41, + "city": { + "郑州": 1, + "开封": 2, + "洛阳": 3, + "平顶山": 4, + "安阳": 5, + "鹤壁": 6, + "新乡": 7, + "焦作": 8, + "濮阳": 9, + "许昌": 10, + "漯河": 11, + "三门峡": 12, + "南阳": 13, + "商丘": 14, + "信阳": 15, + "周口": 16, + "驻马店": 17 + } + }, + "湖北": { + "code": 42, + "city": { + "武汉": 1, + "黄石": 2, + "十堰": 3, + "宜昌": 5, + "襄阳": 6, + "鄂州": 7, + "荆门": 8, + "孝感": 9, + "荆州": 10, + "黄冈": 11, + "咸宁": 12, + "随州": 13, + "恩施土家族苗族自治州": 28 + } + }, + "湖南": { + "code": 43, + "city": { + "长沙": 1, + "株洲": 2, + "湘潭": 3, + "衡阳": 4, + "邵阳": 5, + "岳阳": 6, + "常德": 7, + "张家界": 8, + "益阳": 9, + "郴州": 10, + "永州": 11, + "怀化": 12, + "娄底": 13, + "湘西土家族苗族自治州": 31 + } + }, + "内蒙古": { + "code": 15, + "city": { + "呼和浩特": 1, + "包头": 2, + "乌海": 3, + "赤峰": 4, + "通辽": 5, + "鄂尔多斯": 6, + "呼伦贝尔": 7, + "兴安盟": 22, + "锡林郭勒盟": 25, + "乌兰察布盟": 26, + "巴彦淖尔盟": 28, + "阿拉善盟": 29 + } + }, + "江苏": { + "code": 32, + "city": { + "南京": 1, + "无锡": 2, + "徐州": 3, + "常州": 4, + "苏州": 5, + "南通": 6, + "连云港": 7, + "淮安": 8, + "盐城": 9, + "扬州": 10, + "镇江": 11, + "泰州": 12, + "宿迁": 13 + } + }, + "江西": { + "code": 36, + "city": { + "南昌": 1, + "景德镇": 2, + "萍乡": 3, + "九江": 4, + "新余": 5, + "鹰潭": 6, + "赣州": 7, + "吉安": 8, + "宜春": 9, + "抚州": 10, + "上饶": 11 + } + }, + "吉林": { + "code": 22, + "city": { + "长春": 1, + "吉林": 2, + "四平": 3, + "辽源": 4, + "通化": 5, + "白山": 6, + "松原": 7, + "白城": 8, + "延边朝鲜族自治州": 24 + } + }, + "辽宁": { + "code": 21, + "city": { + "沈阳": 1, + "大连": 2, + "鞍山": 3, + "抚顺": 4, + "本溪": 5, + "丹东": 6, + "锦州": 7, + "营口": 8, + "阜新": 9, + "辽阳": 10, + "盘锦": 11, + "铁岭": 12, + "朝阳": 13, + "葫芦岛": 14 + } + }, + "宁夏": { + "code": 64, + "city": { + "银川": 1, + "石嘴山": 2, + "吴忠": 3, + "固原": 4, + "中卫": 5 + } + }, + "青海": { + "code": 63, + "city": { + "西宁": 1, + "海东": 21, + "海北": 22, + "黄南": 23, + "海南": 25, + "果洛": 26, + "玉树": 27, + "海西": 28 + } + }, + "山西": { + "code": 14, + "city": { + "太原": 1, + "大同": 2, + "阳泉": 3, + "长治": 4, + "晋城": 5, + "朔州": 6, + "晋中": 7, + "运城": 8, + "忻州": 9, + "临汾": 10, + "吕梁": 23 + } + }, + "山东": { + "code": 37, + "city": { + "济南": 1, + "青岛": 2, + "淄博": 3, + "枣庄": 4, + "东营": 5, + "烟台": 6, + "潍坊": 7, + "济宁": 8, + "泰安": 9, + "威海": 10, + "日照": 11, + "莱芜": 12, + "临沂": 13, + "德州": 14, + "聊城": 15, + "滨州": 16, + "菏泽": 17 + } + }, + "上海": { + "code": 31, + "city": { + "黄浦区": 1, + "卢湾区": 3, + "徐汇区": 4, + "长宁区": 5, + "静安区": 6, + "普陀区": 7, + "闸北区": 8, + "虹口区": 9, + "杨浦区": 10, + "闵行区": 12, + "宝山区": 13, + "嘉定区": 14, + "浦东新区": 15, + "金山区": 16, + "松江区": 17, + "青浦区": 18, + "南汇区": 19, + "奉贤区": 20, + "崇明县": 30 + } + }, + "四川": { + "code": 51, + "city": { + "成都": 1, + "自贡": 3, + "攀枝花": 4, + "泸州": 5, + "德阳": 6, + "绵阳": 7, + "广元": 8, + "遂宁": 9, + "内江": 10, + "乐山": 11, + "南充": 13, + "眉山": 14, + "宜宾": 15, + "广安": 16, + "达州": 17, + "雅安": 18, + "巴中": 19, + "资阳": 20, + "阿坝": 32, + "甘孜": 33, + "凉山": 34 + } + }, + "天津": { + "code": 12, + "city": { + "和平区": 1, + "河东区": 2, + "河西区": 3, + "南开区": 4, + "河北区": 5, + "红桥区": 6, + "塘沽区": 7, + "汉沽区": 8, + "大港区": 9, + "东丽区": 10, + "西青区": 11, + "津南区": 12, + "北辰区": 13, + "武清区": 14, + "宝坻区": 15, + "宁河县": 21, + "静海县": 23, + "蓟县": 25 + } + }, + "西藏": { + "code": 54, + "city": { + "拉萨": 1, + "昌都": 21, + "山南": 22, + "日喀则": 23, + "那曲": 24, + "阿里": 25, + "林芝": 26 + } + }, + "新疆": { + "code": 65, + "city": { + "乌鲁木齐": 1, + "克拉玛依": 2, + "吐鲁番": 21, + "哈密": 22, + "昌吉": 23, + "博尔塔拉": 27, + "巴音郭楞": 28, + "阿克苏": 29, + "克孜勒苏": 30, + "喀什": 31, + "和田": 32, + "伊犁": 40, + "塔城": 42, + "阿勒泰": 43, + "石河子": 44 + } + }, + "云南": { + "code": 53, + "city": { + "昆明": 1, + "曲靖": 3, + "玉溪": 4, + "保山": 5, + "昭通": 6, + "楚雄": 23, + "红河": 25, + "文山": 26, + "思茅": 27, + "西双版纳": 28, + "大理": 29, + "德宏": 31, + "丽江": 32, + "怒江": 33, + "迪庆": 34, + "临沧": 35 + } + }, + "浙江": { + "code": 33, + "city": { + "杭州": 1, + "宁波": 2, + "温州": 3, + "嘉兴": 4, + "湖州": 5, + "绍兴": 6, + "金华": 7, + "衢州": 8, + "舟山": 9, + "台州": 10, + "丽水": 11 + } + }, + "陕西": { + "code": 61, + "city": { + "西安": 1, + "铜川": 2, + "宝鸡": 3, + "咸阳": 4, + "渭南": 5, + "延安": 6, + "汉中": 7, + "榆林": 8, + "安康": 9, + "商洛": 10 + } + }, + "台湾": { + "code": 71, + "city": { + "台北": 1, + "高雄": 2, + "基隆": 3, + "台中": 4, + "台南": 5, + "新竹": 6, + "嘉义": 7, + "其他": 90 + } + }, + "香港": { + "code": 81, + "city": { + "香港": 1 + } + }, + "澳门": { + "code": 82, + "city": { + "澳门": 1 + } + }, + "海外": { + "code": 400, + "city": { + "美国": 1, + "英国": 2, + "法国": 3, + "俄罗斯": 4, + "加拿大": 5, + "巴西": 6, + "澳大利亚": 7, + "印尼": 8, + "泰国": 9, + "马来西亚": 10, + "新加坡": 11, + "菲律宾": 12, + "越南": 13, + "印度": 14, + "日本": 15, + "其他": 16 + } + }, + "其他": { + "code": 100, + "city": { + "不限": 1000 + } + } +} diff --git a/weibo/utils/util.py b/weibo/utils/util.py index cec3b99..4c78b55 100644 --- a/weibo/utils/util.py +++ b/weibo/utils/util.py @@ -1,106 +1,106 @@ -import sys -from datetime import datetime, timedelta - -from weibo.utils.region import region_dict - - -def convert_weibo_type(weibo_type): - """将微博类型转换成字符串""" - if weibo_type == 0: - return '&typeall=1' - elif weibo_type == 1: - return '&scope=ori' - elif weibo_type == 2: - return '&xsort=hot' - elif weibo_type == 3: - return '&atten=1' - elif weibo_type == 4: - return '&vip=1' - elif weibo_type == 5: - return '&category=4' - elif weibo_type == 6: - return '&viewpoint=1' - return '&scope=ori' - - -def convert_contain_type(contain_type): - """将包含类型转换成字符串""" - if contain_type == 0: - return '&suball=1' - elif contain_type == 1: - return '&haspic=1' - elif contain_type == 2: - return '&hasvideo=1' - elif contain_type == 3: - return '&hasmusic=1' - elif contain_type == 4: - return '&haslink=1' - return '&suball=1' - - -def get_keyword_list(file_name): - """获取文件中的关键词列表""" - with open(file_name, 'rb') as f: - try: - lines = f.read().splitlines() - lines = [line.decode('utf-8-sig') for line in lines] - except UnicodeDecodeError: - raise ValueError('%s文件应为utf-8编码，请先将文件编码转为utf-8再运行程序' % - file_name) - keyword_list = [] - for line in lines: - if line: - keyword_list.append(line) - return keyword_list - - -def get_regions(region): - """根据区域筛选条件返回符合要求的region""" - new_region = {} - if region: - for key in region: - if region_dict.get(key): - new_region[key] = region_dict[key] - if not new_region: - new_region = region_dict - return new_region - - -def standardize_date(created_at): - """标准化微博发布时间""" - if "刚刚" in created_at: - created_at = datetime.now().strftime("%Y-%m-%d %H:%M") - elif "秒" in created_at: - second = created_at[:created_at.find(u"秒")] - second = timedelta(seconds=int(second)) - created_at = (datetime.now() - second).strftime("%Y-%m-%d %H:%M") - elif "分钟" in created_at: - minute = created_at[:created_at.find(u"分钟")] - minute = timedelta(minutes=int(minute)) - created_at = (datetime.now() - minute).strftime("%Y-%m-%d %H:%M") - elif "小时" in created_at: - hour = created_at[:created_at.find(u"小时")] - hour = timedelta(hours=int(hour)) - created_at = (datetime.now() - hour).strftime("%Y-%m-%d %H:%M") - elif "今天" in created_at: - today = datetime.now().strftime('%Y-%m-%d') - created_at = today + ' ' + created_at[2:] - elif '年' not in created_at: - year = datetime.now().strftime("%Y") - month = created_at[:2] - day = created_at[3:5] - time = created_at[6:] - created_at = year + '-' + month + '-' + day + ' ' + time - else: - year = created_at[:4] - month = created_at[5:7] - day = created_at[8:10] - time = created_at[11:] - created_at = year + '-' + month + '-' + day + ' ' + time - return created_at - - -def str_to_time(text): - """将字符串转换成时间类型""" - result = datetime.strptime(text, '%Y-%m-%d') - return result +import sys +from datetime import datetime, timedelta + +from weibo.utils.region import region_dict + + +def convert_weibo_type(weibo_type): + """将微博类型转换成字符串""" + if weibo_type == 0: + return '&typeall=1' + elif weibo_type == 1: + return '&scope=ori' + elif weibo_type == 2: + return '&xsort=hot' + elif weibo_type == 3: + return '&atten=1' + elif weibo_type == 4: + return '&vip=1' + elif weibo_type == 5: + return '&category=4' + elif weibo_type == 6: + return '&viewpoint=1' + return '&scope=ori' + + +def convert_contain_type(contain_type): + """将包含类型转换成字符串""" + if contain_type == 0: + return '&suball=1' + elif contain_type == 1: + return '&haspic=1' + elif contain_type == 2: + return '&hasvideo=1' + elif contain_type == 3: + return '&hasmusic=1' + elif contain_type == 4: + return '&haslink=1' + return '&suball=1' + + +def get_keyword_list(file_name): + """获取文件中的关键词列表""" + with open(file_name, 'rb') as f: + try: + lines = f.read().splitlines() + lines = [line.decode('utf-8-sig') for line in lines] + except UnicodeDecodeError: + raise ValueError('%s文件应为utf-8编码，请先将文件编码转为utf-8再运行程序' % + file_name) + keyword_list = [] + for line in lines: + if line: + keyword_list.append(line) + return keyword_list + + +def get_regions(region): + """根据区域筛选条件返回符合要求的region""" + new_region = {} + if region: + for key in region: + if region_dict.get(key): + new_region[key] = region_dict[key] + if not new_region: + new_region = region_dict + return new_region + + +def standardize_date(created_at): + """标准化微博发布时间""" + if "刚刚" in created_at: + created_at = datetime.now().strftime("%Y-%m-%d %H:%M") + elif "秒" in created_at: + second = created_at[:created_at.find(u"秒")] + second = timedelta(seconds=int(second)) + created_at = (datetime.now() - second).strftime("%Y-%m-%d %H:%M") + elif "分钟" in created_at: + minute = created_at[:created_at.find(u"分钟")] + minute = timedelta(minutes=int(minute)) + created_at = (datetime.now() - minute).strftime("%Y-%m-%d %H:%M") + elif "小时" in created_at: + hour = created_at[:created_at.find(u"小时")] + hour = timedelta(hours=int(hour)) + created_at = (datetime.now() - hour).strftime("%Y-%m-%d %H:%M") + elif "今天" in created_at: + today = datetime.now().strftime('%Y-%m-%d') + created_at = today + ' ' + created_at[2:] + elif '年' not in created_at: + year = datetime.now().strftime("%Y") + month = created_at[:2] + day = created_at[3:5] + time = created_at[6:] + created_at = year + '-' + month + '-' + day + ' ' + time + else: + year = created_at[:4] + month = created_at[5:7] + day = created_at[8:10] + time = created_at[11:] + created_at = year + '-' + month + '-' + day + ' ' + time + return created_at + + +def str_to_time(text): + """将字符串转换成时间类型""" + result = datetime.strptime(text, '%Y-%m-%d') + return result diff --git "a/\345\271\277\350\245\277\346\264\252\347\201\276\347\210\254\345\217\226\345\217\202\346\225\260.txt" "b/\345\271\277\350\245\277\346\264\252\347\201\276\347\210\254\345\217\226\345\217\202\346\225\260.txt" new file mode 100644 index 0000000..858e6a6 --- /dev/null +++ "b/\345\271\277\350\245\277\346\264\252\347\201\276\347\210\254\345\217\226\345\217\202\346\225\260.txt" @@ -0,0 +1,113 @@ +# 广西洪灾微博爬取参数 +# 使用方法： +# 1. 把下面 WEIBO_COOKIE 的占位内容替换为当前有效的微博 Cookie。 +# 2. 在项目根目录执行：source "广西洪灾爬取参数.txt" +# 3. 再执行文末对应日期批次的 scrapy 命令。 +# +# 注意： +# - 不要把包含真实 Cookie 的文件提交到 Git。 +# - 修改参数不会影响已经运行的爬虫，必须停止后重新启动。 +# - 每个日期批次使用不同的 JOBDIR；同一批次中断后可使用原 JOBDIR 续跑。 + + +# ==================== 本次爬取参数 ==================== + +# 微博登录 Cookie，必填。 +export WEIBO_COOKIE='请替换为当前有效的微博Cookie' + +# 五个搜索关键词。空格表示希望微博搜索同时匹配两个词， +# 比连续短语更容易覆盖“广西等地发生洪水”等表达。 +export WEIBO_KEYWORDS='["广西 洪灾","广西 洪水","广西 暴雨","广西 内涝","美莎克 广西"]' + +# 微博类型： +# 0=全部微博，1=原创微博，2=热门微博，3=关注人微博， +# 4=认证用户微博，5=媒体微博，6=观点微博。 +export WEIBO_TYPE='1' + +# 内容类型： +# 0=不限，1=包含图片，2=包含视频，3=包含音乐，4=包含短链接。 +export WEIBO_CONTAIN_TYPE='0' + +# 微博搜索页面的发布地区筛选。 +# “全部”表示不在搜索阶段限制地区，之后可根据 CSV 的 IP 属地筛选广西。 +export WEIBO_REGION='["全部"]' + +# 当前日期批次，起止日期都包含在爬取范围内。 +export WEIBO_START_DATE='2026-07-09' +export WEIBO_END_DATE='2026-07-10' + +# 结果页达到该值时继续细分搜索时间；越小越完整，但速度越慢。 +export WEIBO_FURTHER_THRESHOLD='46' + +# 抓取数量上限：0=不限制；测试时可临时改为20。 +export WEIBO_LIMIT_RESULT='0' + + +# ==================== IP 属地参数 ==================== + +# 1=请求额外接口补充 IP 属地，0=关闭。 +# 本研究以正文中的事件地点为主要空间变量，因此关闭 IP 属地请求。 +export WEIBO_FETCH_IP='0' + +# 每次 IP 属地接口请求的超时时间，单位为秒。 +export WEIBO_IP_REQUEST_TIMEOUT='5' + +# IP 属地接口请求之间的独立间隔，单位为秒。 +# 它不受 WEIBO_DOWNLOAD_DELAY 控制。 +export WEIBO_IP_REQUEST_DELAY='8' + + +# ==================== 下载和限速参数 ==================== + +# Scrapy 搜索页面请求的基础间隔，单位为秒；程序会随机化实际等待时间。 +export WEIBO_DOWNLOAD_DELAY='10' + +# 单个 Scrapy 请求的超时时间，单位为秒。 +export WEIBO_DOWNLOAD_TIMEOUT='30' + +# 所有域名合计的最大并发请求数。 +export WEIBO_CONCURRENT_REQUESTS='2' + +# 同一域名的最大并发请求数；设为1表示同一域名串行访问。 +export WEIBO_CONCURRENT_REQUESTS_PER_DOMAIN='1' + +# 请求失败后的最大重试次数。 +export WEIBO_RETRY_TIMES='5' + +# AutoThrottle 初始等待时间，单位为秒。 +export WEIBO_AUTOTHROTTLE_START_DELAY='3' + +# AutoThrottle 最大等待时间，单位为秒。 +export WEIBO_AUTOTHROTTLE_MAX_DELAY='60' + +# AutoThrottle 目标并发量。 +export WEIBO_AUTOTHROTTLE_TARGET_CONCURRENCY='1.0' + + +# ==================== 可选参数 ==================== + +# 自定义 User-Agent 列表。通常不需要设置，项目已有默认值。 +# 如需设置，取消下一行注释并填写 JSON 数组： +# export WEIBO_USER_AGENT_LIST='["Mozilla/5.0 ..."]' + + +# ==================== 运行命令 ==================== + +# 首次运行 2026-07-10 至 2026-07-14 批次： +# scrapy crawl search -L INFO \ +# -s JOBDIR=crawls/guangxi-original-2026-07-10_2026-07-14 + +# 中断后继续该批次：再次执行上面完全相同的命令。 +# +# 新日期批次必须同时修改 WEIBO_START_DATE、WEIBO_END_DATE 和 JOBDIR。 +# 不要复用其他日期批次的 JOBDIR，否则请求可能被去重而不发送。 + + +# ==================== 查看当前参数 ==================== + +# 查看全部已导出的微博参数： +# env | sort | grep '^WEIBO_' + +# 单独查看一个参数： +# echo "$WEIBO_TYPE" +# echo "$WEIBO_KEYWORDS"