Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions editor-mode/src/editor/ingest.py
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,11 @@ def collect_archive(db: Database, archive: str, client: httpx.Client, only: set[
r = SourceReport(src.id, src.name)
report.sources.append(r)
collector = COLLECTORS.get(src.kind)
if collector is None:
r.error = f"unknown source kind: {src.kind}"[:500]
log.warning("source %s failed: %s", src.name, r.error)
failed.append((src.id, r.error, True))
continue
spec = SourceSpec(src.kind, src.url, src.config or {}, src.etag, src.last_modified)
try:
result = collector.fetch(client, spec)
Expand Down
14 changes: 13 additions & 1 deletion editor-mode/tests/test_pipeline.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
from sqlalchemy import text

from conftest import fixture
from editor import repo
from editor import ingest, repo
from editor.core_client import CoreClient
from editor.core_fetch import core_http_client
from editor.ingest import collect_tenant
Expand Down Expand Up @@ -78,6 +78,18 @@ def test_failing_source_is_recorded_and_others_continue(db, tenant):
assert errors == 2


@respx.mock
def test_an_unknown_source_kind_does_not_stop_the_other_sources(db, tenant, monkeypatch):
add_sources(db, tenant)
monkeypatch.setattr(ingest, "COLLECTORS", {k: v for k, v in ingest.COLLECTORS.items() if k != "hackernews"})
respx.get(FEED).mock(return_value=httpx.Response(200, content=fixture("rss.xml")))
respx.get(FEED2).mock(return_value=httpx.Response(200, content=MIRROR))
with httpx.Client() as c:
report = collect_tenant(db, tenant, c)
assert [r.error for r in report.sources][2] == "unknown source kind: hackernews"
assert report.new > 0


@respx.mock
def test_the_cores_trouble_is_not_the_sources_fault(db, tenant):
"""Out of daily fetches or the Core down: the sources stay active and
Expand Down