Search Arctic Shift alongside PullPush for Reddit threads

CLAUDE.md

@@ -39,7 +39,7 @@ tests/
 
 **WatchBack.Infrastructure** (library) — Concrete provider implementations and data access.
 - `WatchStateProviders/`: Jellyfin, Trakt, Manual (singleton — holds state in memory)
-- `ThoughtProviders/`: Reddit (via PullPush), Bluesky, Trakt
+- `ThoughtProviders/`: Reddit (via PullPush and Arctic Shift), Bluesky, Trakt
 - `MediaSearchProviders/`: OMDb (implements both `IMediaSearchProvider` and `IRatingsProvider`)
 - `Http/`: `ResilientHttpHandler` — retry with exponential backoff, 429 rate-limit awareness
 - `Persistence/`: `WatchBackDbContext` (SQLite), entities, migrations, repository

DEPLOYMENT.md

@@ -60,7 +60,7 @@ Add one or more thought providers to aggregate reactions:
 
 - **Trakt Comments**: Configure `TRAKT_CLIENT_ID`, `TRAKT_USERNAME`, `TRAKT_ACCESS_TOKEN`
 - **Bluesky Posts**: Configure `BLUESKY_HANDLE`, `BLUESKY_APP_PASSWORD` (optional)
-- **Reddit Comments**: Automatically enabled (uses public PullPush API, no credentials)
+- **Reddit Comments**: Automatically enabled (uses the public PullPush and Arctic Shift archives, no credentials)
 
 ## Docker Compose
 

README.md

@@ -6,7 +6,7 @@ WatchBack is a self-hosted companion app for your media server. While you watch
 
 - **Automatic sync** — detects what you're currently watching and fetches relevant discussions
 - **Time Machine** — filters results to only show reactions posted close to the original air date, so you get the authentic first-watch experience
-- **Multiple sources** — Reddit threads (via [PullPush](pullpush.io/)), Trakt comments, and Bluesky posts, all in one place
+- **Multiple sources** — Reddit threads (via [PullPush](https://pullpush.io/) and [Arctic Shift](https://github.com/ArthurHeitmann/arctic_shift)), Trakt comments, and Bluesky posts, all in one place
 - **Reddit search shortcut** — one-click search for the episode you're watching
 - **Multiple themes** — dark, light, Solarized Dark, Solarized Light, and Monokai
 

SECURITY.md

@@ -45,7 +45,7 @@ In scope:
 Out of scope:
 
 - Vulnerabilities in third-party services WatchBack talks to (Trakt, Reddit,
-  Bluesky, Jellyfin, OMDb, PullPush)
+  Bluesky, Jellyfin, OMDb, PullPush, Arctic Shift)
 - Issues that require a pre-compromised host or malicious local user
 - Denial of service caused by pointing WatchBack at an untrusted upstream
 

src/WatchBack.Api/builtin-subreddit-mappings.json

@@ -19,6 +19,7 @@
     { "title": "Curb Your Enthusiasm", "subreddits": ["curb"] },
     { "title": "It's Always Sunny in Philadelphia", "subreddits": ["IASIP"] },
     { "title": "The Rehearsal", "subreddits": ["nathanfielder"] },
-    { "title": "Nathan For You", "subreddits": ["nathanfielder"] }
+    { "title": "Nathan For You", "subreddits": ["nathanfielder"] },
+    { "title": "Love Is Blind", "subreddits": ["LoveIsBlindNetflix", "LoveIsBlindOnNetflix"] }
   ]
 }

src/WatchBack.Core/Options/RedditOptions.cs

@@ -5,4 +5,8 @@ public class RedditOptions
     public int MaxThreads { get; init; } = 3;
     public int MaxComments { get; init; } = 250;
     public int CacheTtlSeconds { get; init; } = 86400;
+
+    // Reddit archives to search. Both are queried and merged; either can be turned off if it goes dark.
+    public bool UsePullPush { get; init; } = true;
+    public bool UseArcticShift { get; init; } = true;
 }

src/WatchBack.Infrastructure/Http/RateLimitSuppressionStrategy.cs

@@ -1,3 +1,4 @@
+using System.Globalization;
 using System.Net;
 using System.Net.Http.Headers;
 
@@ -65,7 +66,7 @@ private static TimeSpan RateLimitWindow(HttpResponseMessage response)
         TimeSpan window = retryAfter?.Delta
                           ?? (retryAfter?.Date is { } date
                               ? date - DateTimeOffset.UtcNow
-                              : TimeSpan.FromSeconds(60));
+                              : ResetHeaderWindow(response) ?? TimeSpan.FromSeconds(60));
 
         if (window <= TimeSpan.Zero)
         {
@@ -74,6 +75,16 @@ private static TimeSpan RateLimitWindow(HttpResponseMessage response)
 
         return window > s_maxWindow ? s_maxWindow : window;
     }
+
+    // Some APIs (Arctic Shift, Reddit) send seconds-until-reset in X-RateLimit-Reset instead of Retry-After.
+    private static TimeSpan? ResetHeaderWindow(HttpResponseMessage response)
+    {
+        return response.Headers.TryGetValues("X-RateLimit-Reset", out IEnumerable<string>? values) &&
+               double.TryParse(values.FirstOrDefault(), NumberStyles.Float, CultureInfo.InvariantCulture,
+                   out double seconds)
+            ? TimeSpan.FromSeconds(seconds)
+            : null;
+    }
 }
 
 /// <summary>

src/WatchBack.Infrastructure/ThoughtProviders/ArcticShiftSource.cs

@@ -0,0 +1,242 @@
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+using Microsoft.Extensions.Logging;
+
+namespace WatchBack.Infrastructure.ThoughtProviders;
+
+// Which Reddit archive(s) returned a submission. Flags because both can find the same thread.
+[Flags]
+internal enum RedditSource
+{
+    None = 0,
+    PullPush = 1,
+    ArcticShift = 2
+}
+
+// Arctic Shift (https://github.com/ArthurHeitmann/arctic_shift/tree/master/api) client helpers.
+// Submissions reuse the PullPush DTOs — the field names are identical to Reddit's own.
+// Every method returns null on failure so callers can tell "request failed" from "no results".
+internal static class ArcticShiftSource
+{
+    private const string BaseUrl = "https://arctic-shift.photon-reddit.com";
+    private const int PageSize = 100;
+
+    public const string HealthUrl = BaseUrl + "/api/posts/search?subreddit=announcements&limit=1&fields=id";
+    private const int MaxPages = 3;
+
+    // Arctic Shift rate limits are "calculated dynamically based on server load", and the heavy
+    // queries can take seconds. Prefetch runs concurrently, so cap in-flight requests process-wide.
+    private static readonly SemaphoreSlim s_gate = new(2, 2);
+
+    // Time-window listing of a subreddit's posts, oldest first. Keyword (title=) search requires a
+    // subreddit and times out on large ones, so the window is fetched and matched locally instead.
+    // A 100-post page may not cover the window on a busy subreddit, so it pages forward.
+    public static async Task<PullPushSubmissionDto[]?> SearchWindowAsync(
+        HttpClient httpClient, string subreddit, DateTimeOffset after, DateTimeOffset before,
+        ILogger logger, CancellationToken ct)
+    {
+        List<PullPushSubmissionDto> all = [];
+        long cursor = after.ToUnixTimeSeconds();
+        long end = before.ToUnixTimeSeconds();
+
+        for (int page = 0; page < MaxPages; page++)
+        {
+            string url =
+                $"{BaseUrl}/api/posts/search?subreddit={Uri.EscapeDataString(subreddit)}&after={cursor}&before={end}&limit={PageSize}&sort=asc";
+            PullPushSubmissionDto[]? batch = await GetSubmissionsAsync(httpClient, url, logger, ct);
+            if (batch is null)
+            {
+                // A later page failing still leaves earlier pages usable.
+                return page == 0 ? null : [.. all];
+            }
+
+            all.AddRange(batch);
+            if (batch.Length < PageSize || batch[^1].CreatedUtc is not { } last)
+            {
+                break;
+            }
+
+            cursor = last;
+            if (page == MaxPages - 1)
+            {
+                logger.LogDebug(
+                    "Arctic Shift window for r/{Subreddit} hit the {Pages}-page cap before reaching the end; later posts were not searched",
+                    subreddit, MaxPages);
+            }
+        }
+
+        return [.. all];
+    }
+
+    // Newest posts first, for when the release date is unknown (prefetch targets the next episode, which
+    // has none). Cheaper and more reliable than keyword search, and the next episode's threads are recent.
+    public static async Task<PullPushSubmissionDto[]?> SearchRecentAsync(
+        HttpClient httpClient, string subreddit, ILogger logger, CancellationToken ct)
+    {
+        List<PullPushSubmissionDto> all = [];
+        long? cursor = null;
+
+        for (int page = 0; page < MaxPages; page++)
+        {
+            string url =
+                $"{BaseUrl}/api/posts/search?subreddit={Uri.EscapeDataString(subreddit)}&limit={PageSize}&sort=desc";
+            if (cursor.HasValue)
+            {
+                url += $"&before={cursor.Value}";
+            }
+
+            PullPushSubmissionDto[]? batch = await GetSubmissionsAsync(httpClient, url, logger, ct);
+            if (batch is null)
+            {
+                return page == 0 ? null : [.. all];
+            }
+
+            all.AddRange(batch);
+            if (batch.Length < PageSize || batch[^1].CreatedUtc is not { } oldest)
+            {
+                break;
+            }
+
+            cursor = oldest;
+        }
+
+        return [.. all];
+    }
+
+    // Full comment tree for one submission, flattened. Unlike /comments/search it isn't capped at 100.
+    public static async Task<PullPushCommentDto[]?> GetCommentsAsync(
+        HttpClient httpClient, string submissionId, int limit, ILogger logger, CancellationToken ct)
+    {
+        string url = $"{BaseUrl}/api/comments/tree?link_id=t3_{Uri.EscapeDataString(submissionId)}&limit={limit}";
+
+        await s_gate.WaitAsync(ct);
+        try
+        {
+            HttpResponseMessage response = await httpClient.GetAsync(url, ct);
+            if (!response.IsSuccessStatusCode)
+            {
+                logger.LogWarning("Arctic Shift comment fetch failed: HTTP {Status} for thread {ThreadId}",
+                    (int)response.StatusCode, submissionId);
+                return null;
+            }
+
+            string content = await response.Content.ReadAsStringAsync(ct);
+            ArcticShiftCommentTreeResponseDto? tree = JsonSerializer.Deserialize(
+                content, RedditJsonContext.Default.ArcticShiftCommentTreeResponseDto);
+            if (tree?.Data is null || tree.Error is not null)
+            {
+                logger.LogWarning("Arctic Shift comment fetch returned an error body for thread {ThreadId}: {Error}",
+                    submissionId, tree?.Error);
+                return null;
+            }
+
+            List<PullPushCommentDto> flat = [];
+            Flatten(tree.Data, flat);
+            return [.. flat];
+        }
+        catch (Exception ex) when (ex is not OperationCanceledException)
+        {
+            logger.LogWarning(ex, "Arctic Shift comment fetch exception for thread {ThreadId}", submissionId);
+            return null;
+        }
+        finally
+        {
+            s_gate.Release();
+        }
+    }
+
+    private static async Task<PullPushSubmissionDto[]?> GetSubmissionsAsync(
+        HttpClient httpClient, string url, ILogger logger, CancellationToken ct)
+    {
+        await s_gate.WaitAsync(ct);
+        try
+        {
+            HttpResponseMessage response = await httpClient.GetAsync(url, ct);
+            if (!response.IsSuccessStatusCode)
+            {
+                logger.LogWarning("Arctic Shift submission search failed: HTTP {Status}", (int)response.StatusCode);
+                return null;
+            }
+
+            string content = await response.Content.ReadAsStringAsync(ct);
+            ArcticShiftSubmissionsResponseDto? listing = JsonSerializer.Deserialize(
+                content, RedditJsonContext.Default.ArcticShiftSubmissionsResponseDto);
+
+            // Timeouts arrive as {"data":null,"error":"Timeout. Maybe slow down a bit"} — that is a
+            // failure, not an empty result, or a transient timeout would be cached as "no threads".
+            if (listing?.Data is null || listing.Error is not null)
+            {
+                logger.LogWarning("Arctic Shift submission search returned an error body: {Error}", listing?.Error);
+                return null;
+            }
+
+            return listing.Data;
+        }
+        catch (Exception ex) when (ex is not OperationCanceledException)
+        {
+            logger.LogWarning(ex, "Arctic Shift submission search exception");
+            return null;
+        }
+        finally
+        {
+            s_gate.Release();
+        }
+    }
+
+    // The tree is Reddit-listing shaped: nodes of kind "t1" (a comment, with nested "replies") and
+    // "more" (a collapsed stub with no body). Only comments are kept; parent_id preserves the structure.
+    private static void Flatten(IEnumerable<ArcticShiftCommentNodeDto> nodes, List<PullPushCommentDto> output)
+    {
+        foreach (ArcticShiftCommentNodeDto node in nodes)
+        {
+            if (node.Kind != "t1" || node.Data is null)
+            {
+                continue;
+            }
+
+            ArcticShiftCommentDataDto data = node.Data;
+            output.Add(new PullPushCommentDto(data.Id, data.Body, data.Score, data.Author, data.CreatedUtc,
+                data.ParentId));
+
+            // "replies" is a listing object when present and an empty string when there are none.
+            if (data.Replies.ValueKind == JsonValueKind.Object &&
+                data.Replies.TryGetProperty("data", out JsonElement listing) &&
+                listing.TryGetProperty("children", out JsonElement children) &&
+                children.ValueKind == JsonValueKind.Array)
+            {
+                ArcticShiftCommentNodeDto[]? replies = children.Deserialize(
+                    RedditJsonContext.Default.ArcticShiftCommentNodeDtoArray);
+                if (replies is not null)
+                {
+                    Flatten(replies, output);
+                }
+            }
+        }
+    }
+}
+
+internal sealed record ArcticShiftSubmissionsResponseDto(
+    [property: JsonPropertyName("data")] PullPushSubmissionDto[]? Data,
+    [property: JsonPropertyName("error")] string? Error);
+
+internal sealed record ArcticShiftCommentTreeResponseDto(
+    [property: JsonPropertyName("data")] ArcticShiftCommentNodeDto[]? Data,
+    [property: JsonPropertyName("error")] string? Error);
+
+internal sealed record ArcticShiftCommentNodeDto(
+    [property: JsonPropertyName("kind")] string? Kind,
+    [property: JsonPropertyName("data")] ArcticShiftCommentDataDto? Data);
+
+internal sealed record ArcticShiftCommentDataDto(
+    [property: JsonPropertyName("id")] string? Id,
+    [property: JsonPropertyName("body")] string? Body,
+    [property: JsonPropertyName("score")] int? Score,
+    [property: JsonPropertyName("author")] string? Author,
+    [property: JsonPropertyName("created_utc")]
+    [property: JsonConverter(typeof(UnixTimestampConverter))]
+    long? CreatedUtc,
+    [property: JsonPropertyName("parent_id")]
+    string? ParentId,
+    [property: JsonPropertyName("replies")]
+    JsonElement Replies = default);