diff --git a/scripts/collect_hashtags.py b/scripts/collect_hashtags.py index be6c80d..71ffdd6 100644 --- a/scripts/collect_hashtags.py +++ b/scripts/collect_hashtags.py @@ -4,7 +4,7 @@ from sseclient import SSEClient as EventSource import sys -from common import valid_hashtag, valid_edit, hashtag_match +from common import valid_hashtag, valid_edit, hashtag_match, is_canary import db import mwapi @@ -151,6 +151,9 @@ def populate_media_information(change): except ValueError: continue + if is_canary(change): + continue + if "comment" in change: hashtag_matches = hashtag_match(change["comment"]) if hashtag_matches and valid_edit(change): diff --git a/scripts/common.py b/scripts/common.py index 9a0c520..6c739a7 100644 --- a/scripts/common.py +++ b/scripts/common.py @@ -71,6 +71,11 @@ def valid_hashtag(hashtag): return True +def is_canary(change): + # Artificial heartbeat events, which must be discarded. + return change.get("meta", {}).get("domain") == "canary" + + def valid_edit(change): # Exclude Wikidata for now, just far too much data diff --git a/scripts/test_common.py b/scripts/test_common.py new file mode 100644 index 0000000..f8e4851 --- /dev/null +++ b/scripts/test_common.py @@ -0,0 +1,18 @@ +import unittest + +from common import is_canary + + +class IsCanaryTest(unittest.TestCase): + def test_canary_event(self): + self.assertTrue(is_canary({"meta": {"domain": "canary"}})) + + def test_normal_event(self): + self.assertFalse(is_canary({"meta": {"domain": "en.wikipedia.org"}})) + + def test_missing_meta(self): + self.assertFalse(is_canary({})) + + +if __name__ == "__main__": + unittest.main()