Main

requirements.txt

@@ -1,4 +1,4 @@
-Scrapy>=2.11,<3
-requests>=2.31,<3
-Pillow>=8.1.1
-pytest>=8,<9
+Scrapy>=2.11,<3
+requests>=2.31,<3
+Pillow>=8.1.1
+pytest>=8,<9

scrapy.cfg

@@ -1,11 +1,11 @@
-# Automatically created by: scrapy startproject
-#
-# For more information about the [deploy] section see:
-# https://scrapyd.readthedocs.io/en/latest/deploy.html
-
-[settings]
-default = weibo.settings
-
-[deploy]
-#url = http://localhost:6800/
-project = weibo
+# Automatically created by: scrapy startproject
+#
+# For more information about the [deploy] section see:
+# https://scrapyd.readthedocs.io/en/latest/deploy.html
+
+[settings]
+default = weibo.settings
+
+[deploy]
+#url = http://localhost:6800/
+project = weibo

tests/test_search_ip.py

@@ -0,0 +1,83 @@
+from unittest.mock import Mock, PropertyMock, patch
+
+from scrapy.settings import Settings
+
+from weibo.spiders.search import SearchSpider
+
+
+def make_spider():
+    settings = Settings({
+        'KEYWORD_LIST': ['广西洪灾'],
+        'WEIBO_TYPE': 0,
+        'CONTAIN_TYPE': 0,
+        'REGION': ['全部'],
+        'FETCH_IP': True,
+        'IP_REQUEST_DELAY': 0,
+        'DEFAULT_REQUEST_HEADERS': {
+            'cookie': 'Cookie: test-cookie\r\n',
+        },
+        'USER_AGENT_LIST': ['test-browser-user-agent'],
+    })
+    return SearchSpider(settings=settings)
+
+
+def test_get_ip_uses_browser_headers_and_parses_region():
+    spider = make_spider()
+    response = Mock(status_code=200)
+    response.json.return_value = {'region_name': '发布于 广西'}
+    spider.ip_session.get = Mock(return_value=response)
+
+    assert spider.get_ip('R8wCeyBvq') == '广西'
+
+    _, kwargs = spider.ip_session.get.call_args
+    assert kwargs['headers']['Cookie'] == 'test-cookie'
+    assert kwargs['headers']['User-Agent'] == 'test-browser-user-agent'
+    assert kwargs['headers']['Accept'] == 'application/json, text/plain, */*'
+    assert kwargs['headers']['Referer'] == (
+        'https://weibo.com/detail/R8wCeyBvq')
+    assert kwargs['headers']['X-Requested-With'] == 'XMLHttpRequest'
+
+
+def test_get_ip_accepts_region_without_display_prefix():
+    spider = make_spider()
+    response = Mock(status_code=200)
+    response.json.return_value = {'region_name': '广西'}
+    spider.ip_session.get = Mock(return_value=response)
+
+    assert spider.get_ip('R8wCeyBvq') == '广西'
+
+
+def test_get_ip_falls_back_to_mobile_endpoint_with_numeric_id():
+    spider = make_spider()
+    web_response = Mock(status_code=200)
+    web_response.json.return_value = {'id': '123'}
+    mobile_response = Mock(status_code=200)
+    mobile_response.json.return_value = {
+        'data': {
+            'region_name': '发布于 山东',
+        },
+    }
+    spider.ip_session.get = Mock(
+        side_effect=[web_response, mobile_response])
+
+    assert spider.get_ip('R8wCbiAEM', '5194600000000000') == '山东'
+
+    calls = spider.ip_session.get.call_args_list
+    assert calls[0].args[0].startswith(
+        'https://weibo.com/ajax/statuses/show?id=R8wCbiAEM')
+    assert calls[1].args[0] == (
+        'https://m.weibo.cn/statuses/show?id=5194600000000000')
+    assert calls[1].kwargs['headers']['MWeibo-Pwa'] == '1'
+
+
+def test_get_ip_caches_http_failure_and_reports_it_once():
+    spider = make_spider()
+    spider.ip_session.get = Mock(return_value=Mock(status_code=403))
+    with patch.object(SearchSpider, 'logger', new_callable=PropertyMock) as logger:
+        logger.return_value.error = Mock()
+        assert spider.get_ip('first') == ''
+        assert spider.get_ip('second') == ''
+        logger.return_value.error.assert_called_once()
+
+    spider.ip_session.get.assert_called_once()
+    assert spider.ip_failure_counts == {'网页接口HTTP 403': 1}

weibo/items.py

@@ -1,33 +1,33 @@
-# -*- coding: utf-8 -*-
-
-# Define here the models for your scraped items
-#
-# See documentation in:
-# https://docs.scrapy.org/en/latest/topics/items.html
-
-import scrapy
-
-
-class WeiboItem(scrapy.Item):
-    # define the fields for your item here like:
-    id = scrapy.Field()
-    bid = scrapy.Field()
-    user_id = scrapy.Field()
-    screen_name = scrapy.Field()
-    text = scrapy.Field()
-    article_url = scrapy.Field()
-    location = scrapy.Field()
-    at_users = scrapy.Field()
-    topics = scrapy.Field()
-    reposts_count = scrapy.Field()
-    comments_count = scrapy.Field()
-    attitudes_count = scrapy.Field()
-    created_at = scrapy.Field()
-    source = scrapy.Field()
-    pics = scrapy.Field()
-    video_url = scrapy.Field()
-    retweet_id = scrapy.Field()
-    ip = scrapy.Field()
-    user_authentication = scrapy.Field()
-    vip_type = scrapy.Field()
-    vip_level = scrapy.Field()
+# -*- coding: utf-8 -*-
+
+# Define here the models for your scraped items
+#
+# See documentation in:
+# https://docs.scrapy.org/en/latest/topics/items.html
+
+import scrapy
+
+
+class WeiboItem(scrapy.Item):
+    # define the fields for your item here like:
+    id = scrapy.Field()
+    bid = scrapy.Field()
+    user_id = scrapy.Field()
+    screen_name = scrapy.Field()
+    text = scrapy.Field()
+    article_url = scrapy.Field()
+    location = scrapy.Field()
+    at_users = scrapy.Field()
+    topics = scrapy.Field()
+    reposts_count = scrapy.Field()
+    comments_count = scrapy.Field()
+    attitudes_count = scrapy.Field()
+    created_at = scrapy.Field()
+    source = scrapy.Field()
+    pics = scrapy.Field()
+    video_url = scrapy.Field()
+    retweet_id = scrapy.Field()
+    ip = scrapy.Field()
+    user_authentication = scrapy.Field()
+    vip_type = scrapy.Field()
+    vip_level = scrapy.Field()