From eec278b19e78987363ed42084a7f996ff455cc5b Mon Sep 17 00:00:00 2001 From: Isaac Steiner Date: Tue, 27 Oct 2026 17:20:04 -0700 Subject: [PATCH 1/8] Honor X-RateLimit-Reset when backing off after a 429 Arctic Shift and Reddit send seconds-until-reset in X-RateLimit-Reset rather than Retry-After, so the suppression window fell back to a flat 60s regardless of what the server asked for. Co-Authored-By: Claude Sonnet 5.5 --- .../Http/RateLimitSuppressionStrategy.cs | 13 +++++++- .../RateLimitStrategyTests.cs | 30 +++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/src/WatchBack.Infrastructure/Http/RateLimitSuppressionStrategy.cs b/src/WatchBack.Infrastructure/Http/RateLimitSuppressionStrategy.cs index 20a610c..02d4ed0 100644 --- a/src/WatchBack.Infrastructure/Http/RateLimitSuppressionStrategy.cs +++ b/src/WatchBack.Infrastructure/Http/RateLimitSuppressionStrategy.cs @@ -1,3 +1,4 @@ +using System.Globalization; using System.Net; using System.Net.Http.Headers; @@ -65,7 +66,7 @@ private static TimeSpan RateLimitWindow(HttpResponseMessage response) TimeSpan window = retryAfter?.Delta ?? (retryAfter?.Date is { } date ? date - DateTimeOffset.UtcNow - : TimeSpan.FromSeconds(60)); + : ResetHeaderWindow(response) ?? TimeSpan.FromSeconds(60)); if (window <= TimeSpan.Zero) { @@ -74,6 +75,16 @@ private static TimeSpan RateLimitWindow(HttpResponseMessage response) return window > s_maxWindow ? s_maxWindow : window; } + + // Some APIs (Arctic Shift, Reddit) send seconds-until-reset in X-RateLimit-Reset instead of Retry-After. + private static TimeSpan? ResetHeaderWindow(HttpResponseMessage response) + { + return response.Headers.TryGetValues("X-RateLimit-Reset", out IEnumerable? values) && + double.TryParse(values.FirstOrDefault(), NumberStyles.Float, CultureInfo.InvariantCulture, + out double seconds) + ? TimeSpan.FromSeconds(seconds) + : null; + } } /// diff --git a/tests/WatchBack.Infrastructure.Tests/RateLimitStrategyTests.cs b/tests/WatchBack.Infrastructure.Tests/RateLimitStrategyTests.cs index 0b3240d..3d6e503 100644 --- a/tests/WatchBack.Infrastructure.Tests/RateLimitStrategyTests.cs +++ b/tests/WatchBack.Infrastructure.Tests/RateLimitStrategyTests.cs @@ -98,6 +98,36 @@ public async Task ExecuteAsync_After429WithRetryAfter_SubsequentRequestShortCirc second.StatusCode.Should().Be(HttpStatusCode.TooManyRequests); callCount.Should().Be(1, "only the first request should reach the network"); } + + [Fact] + public async Task ExecuteAsync_After429WithRateLimitResetHeader_HonorsResetInsteadOfDefaultWindow() + { + // Arrange — Arctic Shift sends X-RateLimit-Reset (seconds) rather than Retry-After + int callCount = 0; + ResiliencePipeline pipeline = BuildPipeline(); + HttpRequestMessage request = new(HttpMethod.Get, "https://example.com/api/data"); + + // Act + await ExecuteAsync(pipeline, request, () => + { + callCount++; + HttpResponseMessage msg = new(HttpStatusCode.TooManyRequests); + msg.Headers.Add("X-RateLimit-Reset", "1"); + return msg; + }); + + await Task.Delay(TimeSpan.FromMilliseconds(1500)); + + HttpResponseMessage second = await ExecuteAsync(pipeline, request, () => + { + callCount++; + return new HttpResponseMessage(HttpStatusCode.OK); + }); + + // Assert — the 1 s window has expired, so the second request reaches the network + second.StatusCode.Should().Be(HttpStatusCode.OK); + callCount.Should().Be(2); + } } public sealed class HttpLoggingTests From 89105ef75966a3f79776229a31c13c0b5a926130 Mon Sep 17 00:00:00 2001 From: Isaac Steiner Date: Tue, 27 Oct 2026 17:24:46 -0700 Subject: [PATCH 2/8] Search Arctic Shift alongside PullPush for Reddit threads PullPush now 429s every request, so the Reddit provider found nothing. Both archives are now queried in parallel and merged by submission id, so either one going dark no longer hides the other's results. Arctic Shift requires a subreddit for keyword search and times out on large ones, so it lists each subreddit's posts around the air date and matches titles locally. Comments come from /comments/tree (flattened, "more" stubs skipped) with PullPush as the fallback. Error bodies ({"data":null,"error":...}) count as failures, so a timeout is cached briefly instead of as a confirmed miss. Each archive can be disabled via Reddit:UsePullPush / Reddit:UseArcticShift. Co-Authored-By: Claude Sonnet 5.5 --- src/WatchBack.Core/Options/RedditOptions.cs | 4 + .../ThoughtProviders/ArcticShiftSource.cs | 214 ++++++++++++++ .../ThoughtProviders/RedditThoughtProvider.cs | 273 +++++++++++++----- .../RedditArcticShiftTests.cs | 227 +++++++++++++++ .../RedditThoughtProviderTests.cs | 11 +- 5 files changed, 656 insertions(+), 73 deletions(-) create mode 100644 src/WatchBack.Infrastructure/ThoughtProviders/ArcticShiftSource.cs create mode 100644 tests/WatchBack.Infrastructure.Tests/RedditArcticShiftTests.cs diff --git a/src/WatchBack.Core/Options/RedditOptions.cs b/src/WatchBack.Core/Options/RedditOptions.cs index 8810408..cc0c8c4 100644 --- a/src/WatchBack.Core/Options/RedditOptions.cs +++ b/src/WatchBack.Core/Options/RedditOptions.cs @@ -5,4 +5,8 @@ public class RedditOptions public int MaxThreads { get; init; } = 3; public int MaxComments { get; init; } = 250; public int CacheTtlSeconds { get; init; } = 86400; + + // Reddit archives to search. Both are queried and merged; either can be turned off if it goes dark. + public bool UsePullPush { get; init; } = true; + public bool UseArcticShift { get; init; } = true; } diff --git a/src/WatchBack.Infrastructure/ThoughtProviders/ArcticShiftSource.cs b/src/WatchBack.Infrastructure/ThoughtProviders/ArcticShiftSource.cs new file mode 100644 index 0000000..6291663 --- /dev/null +++ b/src/WatchBack.Infrastructure/ThoughtProviders/ArcticShiftSource.cs @@ -0,0 +1,214 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +using Microsoft.Extensions.Logging; + +namespace WatchBack.Infrastructure.ThoughtProviders; + +// Which Reddit archive(s) returned a submission. Flags because both can find the same thread. +[Flags] +internal enum RedditSource +{ + None = 0, + PullPush = 1, + ArcticShift = 2 +} + +// Arctic Shift (https://github.com/ArthurHeitmann/arctic_shift/tree/master/api) client helpers. +// Submissions reuse the PullPush DTOs — the field names are identical to Reddit's own. +// Every method returns null on failure so callers can tell "request failed" from "no results". +internal static class ArcticShiftSource +{ + private const string BaseUrl = "https://arctic-shift.photon-reddit.com"; + private const int PageSize = 100; + private const int MaxPages = 3; + + // Arctic Shift rate limits are "calculated dynamically based on server load", and the heavy + // queries can take seconds. Prefetch runs concurrently, so cap in-flight requests process-wide. + private static readonly SemaphoreSlim s_gate = new(2, 2); + + // Time-window listing of a subreddit's posts, oldest first. Keyword (title=) search requires a + // subreddit and times out on large ones, so the window is fetched and matched locally instead. + // A 100-post page may not cover the window on a busy subreddit, so it pages forward. + public static async Task SearchWindowAsync( + HttpClient httpClient, string subreddit, DateTimeOffset after, DateTimeOffset before, + ILogger logger, CancellationToken ct) + { + List all = []; + long cursor = after.ToUnixTimeSeconds(); + long end = before.ToUnixTimeSeconds(); + + for (int page = 0; page < MaxPages; page++) + { + string url = + $"{BaseUrl}/api/posts/search?subreddit={Uri.EscapeDataString(subreddit)}&after={cursor}&before={end}&limit={PageSize}&sort=asc"; + PullPushSubmissionDto[]? batch = await GetSubmissionsAsync(httpClient, url, logger, ct); + if (batch is null) + { + // A later page failing still leaves earlier pages usable. + return page == 0 ? null : [.. all]; + } + + all.AddRange(batch); + if (batch.Length < PageSize || batch[^1].CreatedUtc is not { } last) + { + break; + } + + cursor = last; + } + + return [.. all]; + } + + // Keyword search within one subreddit. Used when the release date is unknown, so no window exists. + public static Task SearchTitleAsync( + HttpClient httpClient, string subreddit, string title, DateTimeOffset? after, + ILogger logger, CancellationToken ct) + { + string url = + $"{BaseUrl}/api/posts/search?subreddit={Uri.EscapeDataString(subreddit)}&title={Uri.EscapeDataString(title)}&limit={PageSize}&sort=desc"; + if (after.HasValue) + { + url += $"&after={after.Value.ToUnixTimeSeconds()}"; + } + + return GetSubmissionsAsync(httpClient, url, logger, ct); + } + + // Full comment tree for one submission, flattened. Unlike /comments/search it isn't capped at 100. + public static async Task GetCommentsAsync( + HttpClient httpClient, string submissionId, int limit, ILogger logger, CancellationToken ct) + { + string url = $"{BaseUrl}/api/comments/tree?link_id=t3_{Uri.EscapeDataString(submissionId)}&limit={limit}"; + + await s_gate.WaitAsync(ct); + try + { + HttpResponseMessage response = await httpClient.GetAsync(url, ct); + if (!response.IsSuccessStatusCode) + { + logger.LogWarning("Arctic Shift comment fetch failed: HTTP {Status} for thread {ThreadId}", + (int)response.StatusCode, submissionId); + return null; + } + + string content = await response.Content.ReadAsStringAsync(ct); + ArcticShiftCommentTreeResponseDto? tree = JsonSerializer.Deserialize( + content, RedditJsonContext.Default.ArcticShiftCommentTreeResponseDto); + if (tree?.Data is null || tree.Error is not null) + { + logger.LogWarning("Arctic Shift comment fetch returned an error body for thread {ThreadId}: {Error}", + submissionId, tree?.Error); + return null; + } + + List flat = []; + Flatten(tree.Data, flat); + return [.. flat]; + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + logger.LogWarning(ex, "Arctic Shift comment fetch exception for thread {ThreadId}", submissionId); + return null; + } + finally + { + s_gate.Release(); + } + } + + private static async Task GetSubmissionsAsync( + HttpClient httpClient, string url, ILogger logger, CancellationToken ct) + { + await s_gate.WaitAsync(ct); + try + { + HttpResponseMessage response = await httpClient.GetAsync(url, ct); + if (!response.IsSuccessStatusCode) + { + logger.LogWarning("Arctic Shift submission search failed: HTTP {Status}", (int)response.StatusCode); + return null; + } + + string content = await response.Content.ReadAsStringAsync(ct); + ArcticShiftSubmissionsResponseDto? listing = JsonSerializer.Deserialize( + content, RedditJsonContext.Default.ArcticShiftSubmissionsResponseDto); + + // Timeouts arrive as {"data":null,"error":"Timeout. Maybe slow down a bit"} — that is a + // failure, not an empty result, or a transient timeout would be cached as "no threads". + if (listing?.Data is null || listing.Error is not null) + { + logger.LogWarning("Arctic Shift submission search returned an error body: {Error}", listing?.Error); + return null; + } + + return listing.Data; + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + logger.LogWarning(ex, "Arctic Shift submission search exception"); + return null; + } + finally + { + s_gate.Release(); + } + } + + // The tree is Reddit-listing shaped: nodes of kind "t1" (a comment, with nested "replies") and + // "more" (a collapsed stub with no body). Only comments are kept; parent_id preserves the structure. + private static void Flatten(IEnumerable nodes, List output) + { + foreach (ArcticShiftCommentNodeDto node in nodes) + { + if (node.Kind != "t1" || node.Data is null) + { + continue; + } + + ArcticShiftCommentDataDto data = node.Data; + output.Add(new PullPushCommentDto(data.Id, data.Body, data.Score, data.Author, data.CreatedUtc, + data.ParentId)); + + // "replies" is a listing object when present and an empty string when there are none. + if (data.Replies.ValueKind == JsonValueKind.Object && + data.Replies.TryGetProperty("data", out JsonElement listing) && + listing.TryGetProperty("children", out JsonElement children) && + children.ValueKind == JsonValueKind.Array) + { + ArcticShiftCommentNodeDto[]? replies = children.Deserialize( + RedditJsonContext.Default.ArcticShiftCommentNodeDtoArray); + if (replies is not null) + { + Flatten(replies, output); + } + } + } + } +} + +internal sealed record ArcticShiftSubmissionsResponseDto( + [property: JsonPropertyName("data")] PullPushSubmissionDto[]? Data, + [property: JsonPropertyName("error")] string? Error); + +internal sealed record ArcticShiftCommentTreeResponseDto( + [property: JsonPropertyName("data")] ArcticShiftCommentNodeDto[]? Data, + [property: JsonPropertyName("error")] string? Error); + +internal sealed record ArcticShiftCommentNodeDto( + [property: JsonPropertyName("kind")] string? Kind, + [property: JsonPropertyName("data")] ArcticShiftCommentDataDto? Data); + +internal sealed record ArcticShiftCommentDataDto( + [property: JsonPropertyName("id")] string? Id, + [property: JsonPropertyName("body")] string? Body, + [property: JsonPropertyName("score")] int? Score, + [property: JsonPropertyName("author")] string? Author, + [property: JsonPropertyName("created_utc")] + [property: JsonConverter(typeof(UnixTimestampConverter))] + long? CreatedUtc, + [property: JsonPropertyName("parent_id")] + string? ParentId, + [property: JsonPropertyName("replies")] + JsonElement Replies = default); diff --git a/src/WatchBack.Infrastructure/ThoughtProviders/RedditThoughtProvider.cs b/src/WatchBack.Infrastructure/ThoughtProviders/RedditThoughtProvider.cs index 1349a84..4b90c21 100644 --- a/src/WatchBack.Infrastructure/ThoughtProviders/RedditThoughtProvider.cs +++ b/src/WatchBack.Infrastructure/ThoughtProviders/RedditThoughtProvider.cs @@ -17,6 +17,9 @@ namespace WatchBack.Infrastructure.ThoughtProviders; [JsonSerializable(typeof(PullPushSubmissionsResponseDto))] [JsonSerializable(typeof(PullPushCommentsResponseDto))] +[JsonSerializable(typeof(ArcticShiftSubmissionsResponseDto))] +[JsonSerializable(typeof(ArcticShiftCommentTreeResponseDto))] +[JsonSerializable(typeof(ArcticShiftCommentNodeDto[]))] internal sealed partial class RedditJsonContext : JsonSerializerContext; public sealed class RedditThoughtProvider( @@ -31,7 +34,8 @@ public sealed class RedditThoughtProvider( // Upper bound on search spec count (before deduplication) — used for progress estimation. // Sync ticks are reported one-per-spec, so over-estimating is fine; the SSE layer forces // the bar to 100% before emitting the final result. - private const int SearchSpecCount = 8; + // 8 PullPush specs plus roughly two Arctic Shift subreddit lookups. + private const int SearchSpecCount = 10; private static readonly ThoughtResult s_empty = new("Reddit", null, null, null, [], null); private static readonly Regex s_excessiveNewlines = new(@"\n{3,}", RegexOptions.Compiled); @@ -102,57 +106,31 @@ public string GetCacheKey(MediaContext mediaContext) int normalSize = Math.Clamp(_options.MaxThreads * 8, 25, 100); int bypassSize = Math.Clamp(_options.MaxThreads * 4, 10, 100); - // Run all search specs in parallel — PullPush is the bottleneck, so fan-out wins here - (PullPushSubmissionDto[] Subs, bool IsBypass, bool Ok)[] specResults = await Task.WhenAll( - specs.Select(async spec => - { - int size = spec.BypassFilter ? bypassSize : normalSize; - string url = - $"https://api.pullpush.io/reddit/search/submission/?title={Uri.EscapeDataString(spec.Title)}&size={size}&sort_type=score&sort=desc&num_comments=%3E2"; - if (spec.Subreddit != null) - { - url += $"&subreddit={Uri.EscapeDataString(spec.Subreddit)}"; - } - - if (afterEpoch.HasValue) - { - url += $"&after={afterEpoch.Value}"; - } + // Query every enabled archive in parallel — they are the bottleneck, so fan-out wins here. + // A dead archive (PullPush is currently 429ing everything) must not hide the other's results. + List> searches = []; + if (_options.UsePullPush) + { + searches.AddRange(specs.Select(spec => + SearchPullPushAsync(spec, spec.BypassFilter ? bypassSize : normalSize, afterEpoch, progress, ct))); + } - try - { - HttpResponseMessage resp = await httpClient.GetAsync(url, ct); - if (!resp.IsSuccessStatusCode) - { - logger.LogWarning("PullPush submission search failed: HTTP {Status} for query '{Query}'", - (int)resp.StatusCode, spec.Title); - return (Subs: Array.Empty(), IsBypass: spec.BypassFilter, Ok: false); - } + if (_options.UseArcticShift) + { + searches.AddRange(subredditsToSearch.Select(subreddit => + SearchArcticShiftAsync(subreddit, mediaContext, episode, specs, progress, ct))); + } - string respContent = await resp.Content.ReadAsStringAsync(ct); - PullPushSubmissionsResponseDto? listing = - JsonSerializer.Deserialize( - respContent, RedditJsonContext.Default.PullPushSubmissionsResponseDto); - return (Subs: listing?.Data ?? [], IsBypass: spec.BypassFilter, Ok: true); - } - catch (Exception ex) - { - logger.LogWarning(ex, "PullPush submission search exception for query '{Query}'", spec.Title); - return (Subs: Array.Empty(), IsBypass: spec.BypassFilter, Ok: false); - } - finally - { - progress?.Report(new SyncProgressTick(3, "Reddit")); - } - })); + SpecResult[] specResults = (await Task.WhenAll(searches)).SelectMany(r => r).ToArray(); bool anySpecSucceeded = specResults.Any(r => r.Ok); Dictionary seenIds = new(); + Dictionary foundBy = new(); HashSet bypassIds = new(); - foreach ((PullPushSubmissionDto[] subs, bool isBypass, _) in specResults) + foreach (SpecResult specResult in specResults) { - foreach (PullPushSubmissionDto sub in subs) + foreach (PullPushSubmissionDto sub in specResult.Subs) { if (sub.Id == null) { @@ -160,7 +138,8 @@ public string GetCacheKey(MediaContext mediaContext) } seenIds.TryAdd(sub.Id, sub); - if (isBypass) + foundBy[sub.Id] = foundBy.GetValueOrDefault(sub.Id) | specResult.Source; + if (specResult.IsBypass) { bypassIds.Add(sub.Id); } @@ -176,16 +155,16 @@ public string GetCacheKey(MediaContext mediaContext) (bypassIds.Contains(id) && (episode == null || !MentionsDifferentSeason(sub.Title ?? "", episode.SeasonNumber))) || (episode != null && MatchesEpisode(sub.Title ?? "", episode.SeasonNumber, episode.EpisodeNumber)); - logger.LogDebug("PullPush raw: [{Id}] r/{Sub} \"{Title}\" → {Result}", + logger.LogDebug("Reddit raw: [{Id}] r/{Sub} \"{Title}\" → {Result}", id, sub.Subreddit, sub.Title, passes ? "KEEP" : "SKIP"); } } // Keep submissions whose title matches this episode, or that came from a bypass search - // (episode-title search, where PullPush already did the scoping). + // (episode-title search, where the archive or the local window filter already did the scoping). // For bypass results with an episode context, we still reject any submission that // explicitly names a *different* season — this prevents S1 threads bleeding into S2 - // results when the show is new and PullPush has no S2 content yet. + // results when the show is new and the archives have no S2 content yet. List submissions = seenIds.Values .Where(s => s.Id != null && ( (bypassIds.Contains(s.Id!) && @@ -194,28 +173,29 @@ public string GetCacheKey(MediaContext mediaContext) episode.EpisodeNumber)))) .OrderByDescending(s => s.Stickied) .ThenByDescending(s => s.Score ?? 0) + .ThenByDescending(s => s.NumComments ?? 0) // brand-new threads all score 0/1 .ToList(); if (episode != null) { logger.LogInformation( - "PullPush: found {Count} matching submission(s) for '{Title}' S{Season:D2}E{Episode:D2}", + "Reddit: found {Count} matching submission(s) for '{Title}' S{Season:D2}E{Episode:D2}", submissions.Count, mediaContext.Title, episode.SeasonNumber, episode.EpisodeNumber); } else { logger.LogInformation( - "PullPush: found {Count} matching submission(s) for movie '{Title}'", + "Reddit: found {Count} matching submission(s) for movie '{Title}'", submissions.Count, mediaContext.Title); } if (submissions.Count == 0) { ThoughtResult emptyResult = s_empty; - // Confirmed archive misses (at least one 2xx from PullPush) get the full TTL. + // Confirmed archive misses (at least one valid response) get the full TTL. // Timeout/network/429 failures are transient, so they're only cached briefly: // retrying every poll cycle would re-report progress ticks and flash the - // frontend loading bar for as long as PullPush keeps failing. + // frontend loading bar for as long as the archives keep failing. cache.Set(cacheKey, emptyResult, anySpecSucceeded ? TimeSpan.FromSeconds(_options.CacheTtlSeconds) : s_failureCacheTtl); @@ -245,25 +225,29 @@ public string GetCacheKey(MediaContext mediaContext) thisPostBody = selftext; } - string commentsUrl = - $"https://api.pullpush.io/reddit/search/comment/?link_id={Uri.EscapeDataString(submission.Id)}&size={_options.MaxComments}&sort_type=score&sort=desc"; - HttpResponseMessage commentsResponse = await httpClient.GetAsync(commentsUrl, ct); - progress?.Report(new SyncProgressTick(3, "Reddit")); - - if (!commentsResponse.IsSuccessStatusCode) + PullPushCommentDto[] comments = []; + try { - logger.LogWarning("PullPush comment fetch failed: HTTP {Status} for thread {ThreadId}", - (int)commentsResponse.StatusCode, submission.Id); - return Array.Empty(); + // Ask the archive that found the thread first, then fall back to the other. + foreach (RedditSource source in CommentSourceOrder(foundBy[submission.Id])) + { + PullPushCommentDto[]? fetched = source == RedditSource.ArcticShift + ? await ArcticShiftSource.GetCommentsAsync( + httpClient, submission.Id, _options.MaxComments, logger, ct) + : await GetPullPushCommentsAsync(submission.Id, ct); + if (fetched is { Length: > 0 }) + { + comments = fetched; + break; + } + } + } + finally + { + progress?.Report(new SyncProgressTick(3, "Reddit")); } - string commentsContent = await commentsResponse.Content.ReadAsStringAsync(ct); - PullPushCommentsResponseDto? commentsList = JsonSerializer.Deserialize( - commentsContent, - RedditJsonContext.Default.PullPushCommentsResponseDto); - - PullPushCommentDto[] comments = commentsList?.Data ?? []; - logger.LogInformation("PullPush: thread {ThreadId} ({Title}) returned {Count} comment(s)", + logger.LogInformation("Reddit: thread {ThreadId} ({Title}) returned {Count} comment(s)", submission.Id, submission.Title, comments.Length); return comments @@ -293,7 +277,7 @@ public string GetCacheKey(MediaContext mediaContext) } catch (Exception ex) { - logger.LogError(ex, "Reddit/PullPush thought fetch failed"); + logger.LogError(ex, "Reddit thought fetch failed"); return s_empty; } } @@ -358,6 +342,157 @@ public IReadOnlyList GetConfigSchema( ]; } + // One archive search outcome: the submissions, whether MatchesEpisode is bypassed for them, whether + // the request itself succeeded (an empty list from a successful request is a real miss), and who found them. + private readonly record struct SpecResult( + PullPushSubmissionDto[] Subs, bool IsBypass, bool Ok, RedditSource Source); + + private async Task SearchPullPushAsync( + SearchSpec spec, int size, long? afterEpoch, IProgress? progress, CancellationToken ct) + { + string url = + $"https://api.pullpush.io/reddit/search/submission/?title={Uri.EscapeDataString(spec.Title)}&size={size}&sort_type=score&sort=desc&num_comments=%3E2"; + if (spec.Subreddit != null) + { + url += $"&subreddit={Uri.EscapeDataString(spec.Subreddit)}"; + } + + if (afterEpoch.HasValue) + { + url += $"&after={afterEpoch.Value}"; + } + + try + { + HttpResponseMessage resp = await httpClient.GetAsync(url, ct); + if (!resp.IsSuccessStatusCode) + { + logger.LogWarning("PullPush submission search failed: HTTP {Status} for query '{Query}'", + (int)resp.StatusCode, spec.Title); + return [new SpecResult([], spec.BypassFilter, false, RedditSource.PullPush)]; + } + + string respContent = await resp.Content.ReadAsStringAsync(ct); + PullPushSubmissionsResponseDto? listing = JsonSerializer.Deserialize( + respContent, RedditJsonContext.Default.PullPushSubmissionsResponseDto); + return [new SpecResult(listing?.Data ?? [], spec.BypassFilter, true, RedditSource.PullPush)]; + } + catch (Exception ex) + { + logger.LogWarning(ex, "PullPush submission search exception for query '{Query}'", spec.Title); + return [new SpecResult([], spec.BypassFilter, false, RedditSource.PullPush)]; + } + finally + { + progress?.Report(new SyncProgressTick(3, "Reddit")); + } + } + + // Arctic Shift needs a subreddit for any keyword search, so unlike PullPush there are no global + // searches. With a release date it lists the subreddit's posts around the air date and matches titles + // locally; without one it falls back to keyword searches. + private async Task SearchArcticShiftAsync( + string subreddit, MediaContext mediaContext, EpisodeContext? episode, IReadOnlyList specs, + IProgress? progress, CancellationToken ct) + { + try + { + if (mediaContext.ReleaseDate is { } release) + { + // Episode threads go up around airing; movie discussion lingers far longer. + (DateTimeOffset after, DateTimeOffset before) = episode is null + ? (release.AddDays(-14), release.AddDays(90)) + : (release.AddDays(-2), release.AddDays(10)); + + PullPushSubmissionDto[]? window = + await ArcticShiftSource.SearchWindowAsync(httpClient, subreddit, after, before, logger, ct); + if (window is null) + { + return [new SpecResult([], false, false, RedditSource.ArcticShift)]; + } + + (PullPushSubmissionDto[] byCode, PullPushSubmissionDto[] byTitle) = + SplitWindowMatches(window, mediaContext, episode); + return + [ + new SpecResult(byCode, false, true, RedditSource.ArcticShift), + new SpecResult(byTitle, true, true, RedditSource.ArcticShift) + ]; + } + + List<(string Title, bool Bypass)> queries = episode is null + ? [(IThoughtProvider.BuildTextQuery(mediaContext), true)] + : specs.Where(s => string.Equals(s.Subreddit, subreddit, StringComparison.OrdinalIgnoreCase)) + .Select(s => (s.Title, s.BypassFilter)).ToList(); + + SpecResult[] results = await Task.WhenAll(queries.Select(async q => + { + PullPushSubmissionDto[]? subs = + await ArcticShiftSource.SearchTitleAsync(httpClient, subreddit, q.Title, null, logger, ct); + return new SpecResult(subs ?? [], q.Bypass, subs is not null, RedditSource.ArcticShift); + })); + return results; + } + finally + { + progress?.Report(new SyncProgressTick(3, "Reddit")); + } + } + + // Local equivalent of PullPush's server-side scoping for a window of a subreddit's posts: + // episode-code titles are matched by MatchesEpisode afterwards; titles naming the episode (or, for + // movies, the film) are returned separately so the caller can bypass the code check for them. + private static (PullPushSubmissionDto[] ByCode, PullPushSubmissionDto[] ByTitle) SplitWindowMatches( + PullPushSubmissionDto[] window, MediaContext mediaContext, EpisodeContext? episode) + { + if (episode is null) + { + string wanted = Regex.Replace(mediaContext.Title, "[^a-zA-Z0-9]", ""); + return ([], window.Where(s => + Regex.Replace(s.Title ?? "", "[^a-zA-Z0-9]", "") + .Contains(wanted, StringComparison.OrdinalIgnoreCase)).ToArray()); + } + + PullPushSubmissionDto[] byCode = window + .Where(s => MatchesEpisode(s.Title ?? "", episode.SeasonNumber, episode.EpisodeNumber)).ToArray(); + PullPushSubmissionDto[] byTitle = string.IsNullOrWhiteSpace(episode.EpisodeTitle) + ? [] + : window.Except(byCode) + .Where(s => (s.Title ?? "").Contains(episode.EpisodeTitle, StringComparison.OrdinalIgnoreCase)) + .ToArray(); + return (byCode, byTitle); + } + + private async Task GetPullPushCommentsAsync(string submissionId, CancellationToken ct) + { + string commentsUrl = + $"https://api.pullpush.io/reddit/search/comment/?link_id={Uri.EscapeDataString(submissionId)}&size={_options.MaxComments}&sort_type=score&sort=desc"; + HttpResponseMessage commentsResponse = await httpClient.GetAsync(commentsUrl, ct); + + if (!commentsResponse.IsSuccessStatusCode) + { + logger.LogWarning("PullPush comment fetch failed: HTTP {Status} for thread {ThreadId}", + (int)commentsResponse.StatusCode, submissionId); + return null; + } + + string commentsContent = await commentsResponse.Content.ReadAsStringAsync(ct); + PullPushCommentsResponseDto? commentsList = JsonSerializer.Deserialize( + commentsContent, RedditJsonContext.Default.PullPushCommentsResponseDto); + return commentsList?.Data ?? []; + } + + // The archive that found a thread goes first; the other (if enabled) is the fallback. + private IEnumerable CommentSourceOrder(RedditSource foundBy) + { + RedditSource[] order = foundBy.HasFlag(RedditSource.ArcticShift) + ? [RedditSource.ArcticShift, RedditSource.PullPush] + : [RedditSource.PullPush, RedditSource.ArcticShift]; + return order.Where(source => source == RedditSource.ArcticShift + ? _options.UseArcticShift + : _options.UsePullPush); + } + private static Thought MapCommentToThought(PullPushCommentDto data, string? postTitle, string? postUrl, string? postBody) { diff --git a/tests/WatchBack.Infrastructure.Tests/RedditArcticShiftTests.cs b/tests/WatchBack.Infrastructure.Tests/RedditArcticShiftTests.cs new file mode 100644 index 0000000..344aa55 --- /dev/null +++ b/tests/WatchBack.Infrastructure.Tests/RedditArcticShiftTests.cs @@ -0,0 +1,227 @@ +using System.Net; + +using FluentAssertions; + +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging.Abstractions; + +using NSubstitute; + +using WatchBack.Core.Interfaces; +using WatchBack.Core.Models; +using WatchBack.Core.Options; +using WatchBack.Infrastructure.ThoughtProviders; + +using Xunit; + +namespace WatchBack.Infrastructure.Tests; + +public sealed class RedditArcticShiftTests : IDisposable +{ + private const string ArcticShiftHost = "arctic-shift.photon-reddit.com"; + + // Trimmed from a real /api/posts/search response (r/LoveIsBlindNetflix, October 2026). + private const string WindowJson = """ + {"data":[ + {"id":"1x1lhgt","title":"Yasemin......","subreddit":"LoveIsBlindNetflix","created_utc":1791500000,"score":287,"num_comments":265,"is_self":true,"selftext":"","stickied":false,"permalink":"/r/LoveIsBlindNetflix/comments/1x1lhgt/yasemin/"}, + {"id":"1x5mt1l","title":"Season 11 Episode 1 Discussion","subreddit":"LoveIsBlindNetflix","created_utc":1791971000,"score":14,"num_comments":327,"is_self":true,"selftext":"Talk about it here","stickied":true,"permalink":"/r/LoveIsBlindNetflix/comments/1x5mt1l/season_11_episode_1_discussion/"} + ]} + """; + + // Shape of /api/comments/tree: nested Reddit listings, "more" stubs, and "" for no replies. + private const string TreeJson = """ + {"data":[ + {"kind":"t1","data":{"id":"c1","parent_id":"t3_1x5mt1l","link_id":"t3_1x5mt1l","author":"a","score":9,"created_utc":1791971281,"body":"top one", + "replies":{"kind":"Listing","data":{"children":[ + {"kind":"t1","data":{"id":"c2","parent_id":"t1_c1","link_id":"t3_1x5mt1l","author":"b","score":3,"created_utc":1791971400,"body":"a reply","replies":""}}, + {"kind":"more","data":{"count":4,"children":["x","y"],"parent_id":"t1_c1"}} + ]}}}}, + {"kind":"t1","data":{"id":"c3","parent_id":"t3_1x5mt1l","link_id":"t3_1x5mt1l","author":"c","score":5,"created_utc":1791972002,"body":"top two","replies":""}}, + {"kind":"more","data":{"count":10,"children":["z"],"parent_id":"t3_1x5mt1l"}} + ]} + """; + + private readonly MemoryCache _cache = new(new MemoryCacheOptions()); + + private static readonly EpisodeContext s_episode = new( + "Love Is Blind", new DateTimeOffset(2026, 10, 7, 0, 0, 0, TimeSpan.Zero), "Episode 1", 11, 1); + + public void Dispose() + { + _cache.Dispose(); + GC.SuppressFinalize(this); + } + + private static ISubredditMappingService Mapped(string subreddit) + { + ISubredditMappingService svc = Substitute.For(); + svc.GetSubreddits(Arg.Any()).Returns([subreddit]); + return svc; + } + + private static IReplyTreeBuilder PassThroughTree() + { + IReplyTreeBuilder tree = Substitute.For(); + tree.BuildTree(Arg.Any>()).Returns(call => call.Arg>()); + return tree; + } + + private static HttpResponseMessage Json(string json) => + new(HttpStatusCode.OK) { Content = new StringContent(json) }; + + private RedditThoughtProvider CreateProvider( + Func arcticShift, + Func pullPush, + RedditOptions? options = null) + { + MockHttpMessageHandler handler = new(req => + req.RequestUri!.Host == ArcticShiftHost ? arcticShift(req) : pullPush(req)); + return new RedditThoughtProvider( + new HttpClient(handler), + new OptionsSnapshotStub(options ?? new RedditOptions()), + _cache, PassThroughTree(), Mapped("LoveIsBlindNetflix"), + NullLogger.Instance); + } + + private static HttpResponseMessage TooManyRequests(HttpRequestMessage _) => new(HttpStatusCode.TooManyRequests); + + private static HttpResponseMessage ArcticShiftOk(HttpRequestMessage req) => + Json(req.RequestUri!.AbsolutePath.Contains("comments") ? TreeJson : WindowJson); + + [Fact] + public async Task GetThoughtsAsync_PullPushDown_FindsEpisodeThreadViaArcticShift() + { + RedditThoughtProvider provider = CreateProvider(ArcticShiftOk, TooManyRequests); + + ThoughtResult? result = await provider.GetThoughtsAsync(s_episode); + + result!.PostTitle.Should().Be("Season 11 Episode 1 Discussion"); + result.Thoughts!.Select(t => t.Id).Should().BeEquivalentTo("reddit:c1", "reddit:c2", "reddit:c3"); + result.Thoughts!.Single(t => t.Id == "reddit:c2").ParentId.Should().Be("reddit:c1"); + result.Thoughts!.Single(t => t.Id == "reddit:c1").ParentId.Should().BeNull("t3_ parents are top-level"); + } + + [Fact] + public async Task GetThoughtsAsync_ArcticShiftSearch_ListsSubredditWindowWithoutTitleKeyword() + { + List arcticShiftUris = []; + RedditThoughtProvider provider = CreateProvider(req => + { + arcticShiftUris.Add(req.RequestUri!); + return ArcticShiftOk(req); + }, TooManyRequests); + + await provider.GetThoughtsAsync(s_episode); + + Uri search = arcticShiftUris.First(u => u.AbsolutePath.EndsWith("/posts/search", StringComparison.Ordinal)); + search.Query.Should().Contain("subreddit=LoveIsBlindNetflix") + .And.Contain($"after={s_episode.ReleaseDate!.Value.AddDays(-2).ToUnixTimeSeconds()}") + .And.Contain($"before={s_episode.ReleaseDate!.Value.AddDays(10).ToUnixTimeSeconds()}") + .And.NotContain("title=", "keyword search times out on large subreddits"); + } + + [Fact] + public async Task GetThoughtsAsync_BothArchives_MergesThreadsAndDedupesById() + { + const string pullPushSubs = """ + {"data":[ + {"id":"1x5mt1l","title":"Season 11 Episode 1 Discussion","subreddit":"LoveIsBlindNetflix","score":14,"is_self":false}, + {"id":"pp2","title":"Love Is Blind S11E01 thoughts","subreddit":"LoveIsBlindNetflix","score":50,"is_self":false} + ]} + """; + const string pullPushComments = """ + {"data":[{"id":"pp-c1","body":"from pullpush","author":"d","score":1,"created_utc":1791971500,"parent_id":"t3_pp2"}]} + """; + List pullPushThreadsAsked = []; + RedditThoughtProvider provider = CreateProvider(ArcticShiftOk, req => + { + if (req.RequestUri!.AbsolutePath.Contains("comment")) + { + pullPushThreadsAsked.Add(req.RequestUri.Query); + return Json(pullPushComments); + } + + return Json(pullPushSubs); + }); + + ThoughtResult? result = await provider.GetThoughtsAsync(s_episode); + + result!.Thoughts!.Select(t => t.Id).Should().Contain(["reddit:c1", "reddit:pp-c1"]); + result.Thoughts!.Count(t => t.Id == "reddit:c1").Should().Be(1, "a thread found by both is fetched once"); + pullPushThreadsAsked.Should().ContainSingle("only the PullPush-only thread asks PullPush for comments") + .Which.Should().Contain("link_id=pp2"); + } + + [Fact] + public async Task GetThoughtsAsync_ArcticShiftTreeFails_FallsBackToPullPushComments() + { + const string pullPushComments = """ + {"data":[{"id":"fallback","body":"rescued","author":"e","score":2,"created_utc":1791971500,"parent_id":"t3_1x5mt1l"}]} + """; + RedditThoughtProvider provider = CreateProvider( + req => req.RequestUri!.AbsolutePath.Contains("comments") + ? Json("""{"data":null,"error":"Timeout. Maybe slow down a bit"}""") + : Json(WindowJson), + req => req.RequestUri!.AbsolutePath.Contains("comment") + ? Json(pullPushComments) + : TooManyRequests(req)); + + ThoughtResult? result = await provider.GetThoughtsAsync(s_episode); + + result!.Thoughts!.Should().ContainSingle().Which.Id.Should().Be("reddit:fallback"); + } + + // Regression: Arctic Shift reports timeouts as HTTP 200 with {"data":null,"error":...}. Counting that + // as a successful (empty) search cached "no threads" for the full TTL instead of retrying soon. + [Fact] + public async Task GetThoughtsAsync_ArcticShiftErrorBody_IsCachedAsTransientFailure() + { + int arcticShiftRequests = 0; + RedditOptions options = new() { CacheTtlSeconds = 1 }; // confirmed misses expire after 1s; failures after 60s + RedditThoughtProvider provider = CreateProvider(_ => + { + Interlocked.Increment(ref arcticShiftRequests); + return Json("""{"data":null,"error":"Timeout. Maybe slow down a bit"}"""); + }, TooManyRequests, options); + + ThoughtResult? first = await provider.GetThoughtsAsync(s_episode); + int requestsAfterFirst = arcticShiftRequests; + await Task.Delay(TimeSpan.FromMilliseconds(1500)); + await provider.GetThoughtsAsync(s_episode); + + first!.Thoughts!.Should().BeEmpty(); + arcticShiftRequests.Should().Be(requestsAfterFirst, "a timeout is cached with the short failure TTL, not the miss TTL"); + // The long-TTL entry would have expired by now and triggered a refetch. + } + + [Fact] + public async Task GetThoughtsAsync_ArcticShiftDisabled_NeverCallsArcticShift() + { + int arcticShiftRequests = 0; + RedditThoughtProvider provider = CreateProvider(req => + { + Interlocked.Increment(ref arcticShiftRequests); + return ArcticShiftOk(req); + }, TooManyRequests, new RedditOptions { UseArcticShift = false }); + + await provider.GetThoughtsAsync(s_episode); + + arcticShiftRequests.Should().Be(0); + } + + [Fact] + public async Task GetThoughtsAsync_PullPushDisabled_NeverCallsPullPush() + { + int pullPushRequests = 0; + RedditThoughtProvider provider = CreateProvider(ArcticShiftOk, req => + { + Interlocked.Increment(ref pullPushRequests); + return TooManyRequests(req); + }, new RedditOptions { UsePullPush = false }); + + ThoughtResult? result = await provider.GetThoughtsAsync(s_episode); + + pullPushRequests.Should().Be(0); + result!.Thoughts!.Should().NotBeEmpty(); + } +} diff --git a/tests/WatchBack.Infrastructure.Tests/RedditThoughtProviderTests.cs b/tests/WatchBack.Infrastructure.Tests/RedditThoughtProviderTests.cs index f537e86..7133acc 100644 --- a/tests/WatchBack.Infrastructure.Tests/RedditThoughtProviderTests.cs +++ b/tests/WatchBack.Infrastructure.Tests/RedditThoughtProviderTests.cs @@ -21,7 +21,7 @@ public class RedditThoughtProviderTests : IDisposable { private readonly MemoryCache _cache = new(new MemoryCacheOptions()); - private readonly RedditOptions _options = new() { MaxThreads = 3, MaxComments = 250, CacheTtlSeconds = 86400 }; + private readonly RedditOptions _options = new() { MaxThreads = 3, MaxComments = 250, CacheTtlSeconds = 86400, UseArcticShift = false }; public void Dispose() { @@ -277,8 +277,8 @@ public void ExpectedWeight_EqualsSearchesPlusMaxThreadsTimesThree() _cache, treeBuilder, NoMappings(), NullLogger.Instance); - // 7 base specs + 1 episode-title spec (upper bound) + MaxThreads comment fetches, 3 weight each - provider.ExpectedWeight.Should().Be((8 + _options.MaxThreads) * 3); + // 8 PullPush specs (upper bound) + ~2 Arctic Shift subreddit lookups + MaxThreads comment fetches, 3 weight each + provider.ExpectedWeight.Should().Be((10 + _options.MaxThreads) * 3); } // Regression: threads that use the episode title rather than an SxxExx code (e.g. @@ -1020,7 +1020,10 @@ public async Task GetThoughtsAsync_WithNullReleaseDate_OmitsAfterParameter() [Fact] public async Task GetThoughtsAsync_SizeScalesWithMaxThreads() { - RedditOptions scaledOptions = new() { MaxThreads = 10, MaxComments = 250, CacheTtlSeconds = 86400 }; + RedditOptions scaledOptions = new() + { + MaxThreads = 10, MaxComments = 250, CacheTtlSeconds = 86400, UseArcticShift = false + }; EpisodeContext mediaContext = new( "Test Show", From ae41537d8abecab0873f07f830c5d33d996bf267 Mon Sep 17 00:00:00 2001 From: Isaac Steiner Date: Tue, 27 Oct 2026 17:25:07 -0700 Subject: [PATCH 3/8] Map Love Is Blind to its Netflix subreddits The derived subreddit (r/loveisblind) is empty, so the show found no threads. Its discussion lives in r/LoveIsBlindNetflix and r/LoveIsBlindOnNetflix. Co-Authored-By: Claude Sonnet 5.5 --- src/WatchBack.Api/builtin-subreddit-mappings.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/WatchBack.Api/builtin-subreddit-mappings.json b/src/WatchBack.Api/builtin-subreddit-mappings.json index 915e7ad..6e11293 100644 --- a/src/WatchBack.Api/builtin-subreddit-mappings.json +++ b/src/WatchBack.Api/builtin-subreddit-mappings.json @@ -19,6 +19,7 @@ { "title": "Curb Your Enthusiasm", "subreddits": ["curb"] }, { "title": "It's Always Sunny in Philadelphia", "subreddits": ["IASIP"] }, { "title": "The Rehearsal", "subreddits": ["nathanfielder"] }, - { "title": "Nathan For You", "subreddits": ["nathanfielder"] } + { "title": "Nathan For You", "subreddits": ["nathanfielder"] }, + { "title": "Love Is Blind", "subreddits": ["LoveIsBlindNetflix", "LoveIsBlindOnNetflix"] } ] } From e896f05de56f6d4fd66bc3a8295328b620195773 Mon Sep 17 00:00:00 2001 From: Isaac Steiner Date: Tue, 27 Oct 2026 17:25:07 -0700 Subject: [PATCH 4/8] Document Arctic Shift as a Reddit source Co-Authored-By: Claude Sonnet 5.5 --- CLAUDE.md | 2 +- DEPLOYMENT.md | 2 +- README.md | 2 +- SECURITY.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index d800fe7..9cba23c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -39,7 +39,7 @@ tests/ **WatchBack.Infrastructure** (library) — Concrete provider implementations and data access. - `WatchStateProviders/`: Jellyfin, Trakt, Manual (singleton — holds state in memory) -- `ThoughtProviders/`: Reddit (via PullPush), Bluesky, Trakt +- `ThoughtProviders/`: Reddit (via PullPush and Arctic Shift), Bluesky, Trakt - `MediaSearchProviders/`: OMDb (implements both `IMediaSearchProvider` and `IRatingsProvider`) - `Http/`: `ResilientHttpHandler` — retry with exponential backoff, 429 rate-limit awareness - `Persistence/`: `WatchBackDbContext` (SQLite), entities, migrations, repository diff --git a/DEPLOYMENT.md b/DEPLOYMENT.md index 0b83ba8..eb574d4 100644 --- a/DEPLOYMENT.md +++ b/DEPLOYMENT.md @@ -60,7 +60,7 @@ Add one or more thought providers to aggregate reactions: - **Trakt Comments**: Configure `TRAKT_CLIENT_ID`, `TRAKT_USERNAME`, `TRAKT_ACCESS_TOKEN` - **Bluesky Posts**: Configure `BLUESKY_HANDLE`, `BLUESKY_APP_PASSWORD` (optional) -- **Reddit Comments**: Automatically enabled (uses public PullPush API, no credentials) +- **Reddit Comments**: Automatically enabled (uses the public PullPush and Arctic Shift archives, no credentials) ## Docker Compose diff --git a/README.md b/README.md index 4d886f1..7e534f6 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ WatchBack is a self-hosted companion app for your media server. While you watch - **Automatic sync** — detects what you're currently watching and fetches relevant discussions - **Time Machine** — filters results to only show reactions posted close to the original air date, so you get the authentic first-watch experience -- **Multiple sources** — Reddit threads (via [PullPush](pullpush.io/)), Trakt comments, and Bluesky posts, all in one place +- **Multiple sources** — Reddit threads (via [PullPush](https://pullpush.io/) and [Arctic Shift](https://github.com/ArthurHeitmann/arctic_shift)), Trakt comments, and Bluesky posts, all in one place - **Reddit search shortcut** — one-click search for the episode you're watching - **Multiple themes** — dark, light, Solarized Dark, Solarized Light, and Monokai diff --git a/SECURITY.md b/SECURITY.md index f0e7826..ae82f4a 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -45,7 +45,7 @@ In scope: Out of scope: - Vulnerabilities in third-party services WatchBack talks to (Trakt, Reddit, - Bluesky, Jellyfin, OMDb, PullPush) + Bluesky, Jellyfin, OMDb, PullPush, Arctic Shift) - Issues that require a pre-compromised host or malicious local user - Denial of service caused by pointing WatchBack at an untrusted upstream From 0d83a175b80dd55d8fa7c96895c570c20281fe4e Mon Sep 17 00:00:00 2001 From: Isaac Steiner Date: Tue, 27 Oct 2026 17:25:43 -0700 Subject: [PATCH 5/8] Report Reddit healthy while either archive answers The health check only probed PullPush, so the provider showed as down even when Arctic Shift was serving results. It now probes every enabled archive and names the ones that are failing. Co-Authored-By: Claude Sonnet 5.5 --- .../ThoughtProviders/ArcticShiftSource.cs | 2 + .../ThoughtProviders/RedditThoughtProvider.cs | 46 ++++++++++++------- .../RedditArcticShiftTests.cs | 21 +++++++++ 3 files changed, 53 insertions(+), 16 deletions(-) diff --git a/src/WatchBack.Infrastructure/ThoughtProviders/ArcticShiftSource.cs b/src/WatchBack.Infrastructure/ThoughtProviders/ArcticShiftSource.cs index 6291663..5a1821e 100644 --- a/src/WatchBack.Infrastructure/ThoughtProviders/ArcticShiftSource.cs +++ b/src/WatchBack.Infrastructure/ThoughtProviders/ArcticShiftSource.cs @@ -21,6 +21,8 @@ internal static class ArcticShiftSource { private const string BaseUrl = "https://arctic-shift.photon-reddit.com"; private const int PageSize = 100; + + public const string HealthUrl = BaseUrl + "/api/posts/search?subreddit=announcements&limit=1&fields=id"; private const int MaxPages = 3; // Arctic Shift rate limits are "calculated dynamically based on server load", and the heavy diff --git a/src/WatchBack.Infrastructure/ThoughtProviders/RedditThoughtProvider.cs b/src/WatchBack.Infrastructure/ThoughtProviders/RedditThoughtProvider.cs index 4b90c21..8f87a70 100644 --- a/src/WatchBack.Infrastructure/ThoughtProviders/RedditThoughtProvider.cs +++ b/src/WatchBack.Infrastructure/ThoughtProviders/RedditThoughtProvider.cs @@ -282,28 +282,42 @@ public string GetCacheKey(MediaContext mediaContext) } } + // Healthy while at least one enabled archive answers: the provider keeps working with either. public async Task GetServiceHealthAsync(CancellationToken ct = default) { - try + List<(string Name, string Url)> probes = []; + if (_options.UsePullPush) { - using CancellationTokenSource cts = CancellationTokenSource.CreateLinkedTokenSource(ct); - cts.CancelAfter(TimeSpan.FromSeconds(5)); - - HttpResponseMessage response = - await httpClient.GetAsync("https://api.pullpush.io/reddit/search/comment/?size=0", cts.Token); - - return new ServiceHealth( - response.IsSuccessStatusCode, - response.IsSuccessStatusCode ? "OK" : response.StatusCode.ToString(), - DateTimeOffset.UtcNow); + probes.Add(("PullPush", "https://api.pullpush.io/reddit/search/comment/?size=0")); } - catch (Exception ex) + + if (_options.UseArcticShift) { - return new ServiceHealth( - false, - ex.Message, - DateTimeOffset.UtcNow); + probes.Add(("Arctic Shift", ArcticShiftSource.HealthUrl)); } + + (string Name, string Status)[] results = await Task.WhenAll(probes.Select(async probe => + { + try + { + using CancellationTokenSource cts = CancellationTokenSource.CreateLinkedTokenSource(ct); + cts.CancelAfter(TimeSpan.FromSeconds(5)); + + HttpResponseMessage response = await httpClient.GetAsync(probe.Url, cts.Token); + return (probe.Name, response.IsSuccessStatusCode ? "OK" : response.StatusCode.ToString()); + } + catch (Exception ex) + { + return (probe.Name, ex.Message); + } + })); + + bool anyHealthy = results.Any(r => r.Status == "OK"); + string message = results.All(r => r.Status == "OK") + ? "OK" + : string.Join("; ", results.Where(r => r.Status != "OK").Select(r => $"{r.Name}: {r.Status}")); + + return new ServiceHealth(anyHealthy, message, DateTimeOffset.UtcNow); } public string ConfigSection => "Reddit"; diff --git a/tests/WatchBack.Infrastructure.Tests/RedditArcticShiftTests.cs b/tests/WatchBack.Infrastructure.Tests/RedditArcticShiftTests.cs index 344aa55..9ea7023 100644 --- a/tests/WatchBack.Infrastructure.Tests/RedditArcticShiftTests.cs +++ b/tests/WatchBack.Infrastructure.Tests/RedditArcticShiftTests.cs @@ -224,4 +224,25 @@ public async Task GetThoughtsAsync_PullPushDisabled_NeverCallsPullPush() pullPushRequests.Should().Be(0); result!.Thoughts!.Should().NotBeEmpty(); } + + [Fact] + public async Task GetServiceHealthAsync_PullPushDownButArcticShiftUp_IsHealthyAndNamesTheOutage() + { + RedditThoughtProvider provider = CreateProvider(_ => Json("""{"data":[]}"""), TooManyRequests); + + ServiceHealth health = await provider.GetServiceHealthAsync(); + + health.IsHealthy.Should().BeTrue("one working archive is enough"); + health.Message.Should().Contain("PullPush").And.Contain("TooManyRequests"); + } + + [Fact] + public async Task GetServiceHealthAsync_BothArchivesDown_IsUnhealthy() + { + RedditThoughtProvider provider = CreateProvider(TooManyRequests, TooManyRequests); + + ServiceHealth health = await provider.GetServiceHealthAsync(); + + health.IsHealthy.Should().BeFalse(); + } } From a886480a4abfdc42615010b6ed07be532a97eea3 Mon Sep 17 00:00:00 2001 From: Isaac Steiner Date: Tue, 27 Oct 2026 17:31:33 -0700 Subject: [PATCH 6/8] Match episode titles on whole words in Arctic Shift windows A generic episode title like "Episode 1" matched "Season 11 Episode 12 Discussion" as a substring, surfacing later-episode spoilers (and "Pilot" would match "Copilot"). Co-Authored-By: Claude Sonnet 5.5 --- .../ThoughtProviders/RedditThoughtProvider.cs | 12 +++++--- .../RedditArcticShiftTests.cs | 30 +++++++++++++++++++ 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/src/WatchBack.Infrastructure/ThoughtProviders/RedditThoughtProvider.cs b/src/WatchBack.Infrastructure/ThoughtProviders/RedditThoughtProvider.cs index 8f87a70..0aaa397 100644 --- a/src/WatchBack.Infrastructure/ThoughtProviders/RedditThoughtProvider.cs +++ b/src/WatchBack.Infrastructure/ThoughtProviders/RedditThoughtProvider.cs @@ -469,11 +469,15 @@ private static (PullPushSubmissionDto[] ByCode, PullPushSubmissionDto[] ByTitle) PullPushSubmissionDto[] byCode = window .Where(s => MatchesEpisode(s.Title ?? "", episode.SeasonNumber, episode.EpisodeNumber)).ToArray(); - PullPushSubmissionDto[] byTitle = string.IsNullOrWhiteSpace(episode.EpisodeTitle) + // Whole-word match: a generic "Episode 1" title must not pull in "Episode 12" (a spoiler), nor + // "Pilot" match "Copilot". Lookarounds rather than \b so titles ending in punctuation still work. + Regex? titlePattern = string.IsNullOrWhiteSpace(episode.EpisodeTitle) + ? null + : new Regex($"(? (s.Title ?? "").Contains(episode.EpisodeTitle, StringComparison.OrdinalIgnoreCase)) - .ToArray(); + : window.Except(byCode).Where(s => titlePattern.IsMatch(s.Title ?? "")).ToArray(); return (byCode, byTitle); } diff --git a/tests/WatchBack.Infrastructure.Tests/RedditArcticShiftTests.cs b/tests/WatchBack.Infrastructure.Tests/RedditArcticShiftTests.cs index 9ea7023..9eca076 100644 --- a/tests/WatchBack.Infrastructure.Tests/RedditArcticShiftTests.cs +++ b/tests/WatchBack.Infrastructure.Tests/RedditArcticShiftTests.cs @@ -245,4 +245,34 @@ public async Task GetServiceHealthAsync_BothArchivesDown_IsUnhealthy() health.IsHealthy.Should().BeFalse(); } + + // Regression: the episode-title match was a substring check, so a generic Jellyfin title like + // "Episode 1" matched "Season 11 Episode 12 Discussion" and surfaced later-episode spoilers. + [Fact] + public async Task GetThoughtsAsync_GenericEpisodeTitle_DoesNotMatchLaterEpisodeThreads() + { + const string window = """ + {"data":[ + {"id":"e12","title":"Season 11 Episode 12 Discussion","subreddit":"LoveIsBlindNetflix","created_utc":1791971000,"score":90}, + {"id":"e1x","title":"Episode 1 reactions (no spoilers)","subreddit":"LoveIsBlindNetflix","created_utc":1791971100,"score":10} + ]} + """; + EpisodeContext episode = new("Love Is Blind", s_episode.ReleaseDate, "Episode 1", 11, 1); + List commentThreads = []; + RedditThoughtProvider provider = CreateProvider(req => + { + if (req.RequestUri!.AbsolutePath.Contains("comments")) + { + commentThreads.Add(req.RequestUri.Query); + return Json("""{"data":[]}"""); + } + + return Json(window); + }, TooManyRequests); + + await provider.GetThoughtsAsync(episode); + + commentThreads.Should().ContainSingle().Which.Should().Contain("t3_e1x") + .And.NotContain("e12", "an \"Episode 12\" thread is a later episode"); + } } From 3f22c63804b0a10942e1a8f07d0bee48fbeff2e5 Mon Sep 17 00:00:00 2001 From: Isaac Steiner Date: Tue, 27 Oct 2026 17:31:37 -0700 Subject: [PATCH 7/8] Log when an Arctic Shift window hits the page cap Co-Authored-By: Claude Sonnet 5.5 --- .../ThoughtProviders/ArcticShiftSource.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/WatchBack.Infrastructure/ThoughtProviders/ArcticShiftSource.cs b/src/WatchBack.Infrastructure/ThoughtProviders/ArcticShiftSource.cs index 5a1821e..1e3cbb2 100644 --- a/src/WatchBack.Infrastructure/ThoughtProviders/ArcticShiftSource.cs +++ b/src/WatchBack.Infrastructure/ThoughtProviders/ArcticShiftSource.cs @@ -58,6 +58,12 @@ internal static class ArcticShiftSource } cursor = last; + if (page == MaxPages - 1) + { + logger.LogDebug( + "Arctic Shift window for r/{Subreddit} hit the {Pages}-page cap before reaching the end; later posts were not searched", + subreddit, MaxPages); + } } return [.. all]; From 71f4980725792867bcbfbfb90f607c5760d73b0c Mon Sep 17 00:00:00 2001 From: Isaac Steiner Date: Tue, 27 Oct 2026 17:32:31 -0700 Subject: [PATCH 8/8] List newest posts when the release date is unknown Prefetch targets the next episode with no release date, which sent Arctic Shift keyword searches that time out on large subreddits. The newest posts are listed and matched locally instead. Co-Authored-By: Claude Sonnet 5.5 --- .../ThoughtProviders/ArcticShiftSource.cs | 38 ++++++++++--- .../ThoughtProviders/RedditThoughtProvider.cs | 55 ++++++++----------- .../RedditArcticShiftTests.cs | 23 ++++++++ 3 files changed, 75 insertions(+), 41 deletions(-) diff --git a/src/WatchBack.Infrastructure/ThoughtProviders/ArcticShiftSource.cs b/src/WatchBack.Infrastructure/ThoughtProviders/ArcticShiftSource.cs index 1e3cbb2..6236119 100644 --- a/src/WatchBack.Infrastructure/ThoughtProviders/ArcticShiftSource.cs +++ b/src/WatchBack.Infrastructure/ThoughtProviders/ArcticShiftSource.cs @@ -69,19 +69,39 @@ internal static class ArcticShiftSource return [.. all]; } - // Keyword search within one subreddit. Used when the release date is unknown, so no window exists. - public static Task SearchTitleAsync( - HttpClient httpClient, string subreddit, string title, DateTimeOffset? after, - ILogger logger, CancellationToken ct) + // Newest posts first, for when the release date is unknown (prefetch targets the next episode, which + // has none). Cheaper and more reliable than keyword search, and the next episode's threads are recent. + public static async Task SearchRecentAsync( + HttpClient httpClient, string subreddit, ILogger logger, CancellationToken ct) { - string url = - $"{BaseUrl}/api/posts/search?subreddit={Uri.EscapeDataString(subreddit)}&title={Uri.EscapeDataString(title)}&limit={PageSize}&sort=desc"; - if (after.HasValue) + List all = []; + long? cursor = null; + + for (int page = 0; page < MaxPages; page++) { - url += $"&after={after.Value.ToUnixTimeSeconds()}"; + string url = + $"{BaseUrl}/api/posts/search?subreddit={Uri.EscapeDataString(subreddit)}&limit={PageSize}&sort=desc"; + if (cursor.HasValue) + { + url += $"&before={cursor.Value}"; + } + + PullPushSubmissionDto[]? batch = await GetSubmissionsAsync(httpClient, url, logger, ct); + if (batch is null) + { + return page == 0 ? null : [.. all]; + } + + all.AddRange(batch); + if (batch.Length < PageSize || batch[^1].CreatedUtc is not { } oldest) + { + break; + } + + cursor = oldest; } - return GetSubmissionsAsync(httpClient, url, logger, ct); + return [.. all]; } // Full comment tree for one submission, flattened. Unlike /comments/search it isn't capped at 100. diff --git a/src/WatchBack.Infrastructure/ThoughtProviders/RedditThoughtProvider.cs b/src/WatchBack.Infrastructure/ThoughtProviders/RedditThoughtProvider.cs index 0aaa397..4991598 100644 --- a/src/WatchBack.Infrastructure/ThoughtProviders/RedditThoughtProvider.cs +++ b/src/WatchBack.Infrastructure/ThoughtProviders/RedditThoughtProvider.cs @@ -118,7 +118,7 @@ public string GetCacheKey(MediaContext mediaContext) if (_options.UseArcticShift) { searches.AddRange(subredditsToSearch.Select(subreddit => - SearchArcticShiftAsync(subreddit, mediaContext, episode, specs, progress, ct))); + SearchArcticShiftAsync(subreddit, mediaContext, episode, progress, ct))); } SpecResult[] specResults = (await Task.WhenAll(searches)).SelectMany(r => r).ToArray(); @@ -402,50 +402,41 @@ private async Task SearchPullPushAsync( } } - // Arctic Shift needs a subreddit for any keyword search, so unlike PullPush there are no global - // searches. With a release date it lists the subreddit's posts around the air date and matches titles - // locally; without one it falls back to keyword searches. + // Arctic Shift needs a subreddit for any keyword search and times out on large ones, so unlike PullPush + // there are no keyword or global searches. It lists the subreddit's posts instead — around the air date + // when known, otherwise the newest — and matches titles locally. private async Task SearchArcticShiftAsync( - string subreddit, MediaContext mediaContext, EpisodeContext? episode, IReadOnlyList specs, + string subreddit, MediaContext mediaContext, EpisodeContext? episode, IProgress? progress, CancellationToken ct) { try { + PullPushSubmissionDto[]? posts; if (mediaContext.ReleaseDate is { } release) { // Episode threads go up around airing; movie discussion lingers far longer. (DateTimeOffset after, DateTimeOffset before) = episode is null ? (release.AddDays(-14), release.AddDays(90)) : (release.AddDays(-2), release.AddDays(10)); - - PullPushSubmissionDto[]? window = - await ArcticShiftSource.SearchWindowAsync(httpClient, subreddit, after, before, logger, ct); - if (window is null) - { - return [new SpecResult([], false, false, RedditSource.ArcticShift)]; - } - - (PullPushSubmissionDto[] byCode, PullPushSubmissionDto[] byTitle) = - SplitWindowMatches(window, mediaContext, episode); - return - [ - new SpecResult(byCode, false, true, RedditSource.ArcticShift), - new SpecResult(byTitle, true, true, RedditSource.ArcticShift) - ]; + posts = await ArcticShiftSource.SearchWindowAsync(httpClient, subreddit, after, before, logger, ct); + } + else + { + posts = await ArcticShiftSource.SearchRecentAsync(httpClient, subreddit, logger, ct); } - List<(string Title, bool Bypass)> queries = episode is null - ? [(IThoughtProvider.BuildTextQuery(mediaContext), true)] - : specs.Where(s => string.Equals(s.Subreddit, subreddit, StringComparison.OrdinalIgnoreCase)) - .Select(s => (s.Title, s.BypassFilter)).ToList(); - - SpecResult[] results = await Task.WhenAll(queries.Select(async q => + if (posts is null) { - PullPushSubmissionDto[]? subs = - await ArcticShiftSource.SearchTitleAsync(httpClient, subreddit, q.Title, null, logger, ct); - return new SpecResult(subs ?? [], q.Bypass, subs is not null, RedditSource.ArcticShift); - })); - return results; + return [new SpecResult([], false, false, RedditSource.ArcticShift)]; + } + + (PullPushSubmissionDto[] byCode, PullPushSubmissionDto[] byTitle) = + SplitWindowMatches(posts, mediaContext, episode); + return + [ + new SpecResult(byCode, false, true, RedditSource.ArcticShift), + new SpecResult(byTitle, true, true, RedditSource.ArcticShift) + ]; } finally { @@ -453,7 +444,7 @@ private async Task SearchArcticShiftAsync( } } - // Local equivalent of PullPush's server-side scoping for a window of a subreddit's posts: + // Local equivalent of PullPush's server-side scoping for a listing of a subreddit's posts: // episode-code titles are matched by MatchesEpisode afterwards; titles naming the episode (or, for // movies, the film) are returned separately so the caller can bypass the code check for them. private static (PullPushSubmissionDto[] ByCode, PullPushSubmissionDto[] ByTitle) SplitWindowMatches( diff --git a/tests/WatchBack.Infrastructure.Tests/RedditArcticShiftTests.cs b/tests/WatchBack.Infrastructure.Tests/RedditArcticShiftTests.cs index 9eca076..8003566 100644 --- a/tests/WatchBack.Infrastructure.Tests/RedditArcticShiftTests.cs +++ b/tests/WatchBack.Infrastructure.Tests/RedditArcticShiftTests.cs @@ -275,4 +275,27 @@ public async Task GetThoughtsAsync_GenericEpisodeTitle_DoesNotMatchLaterEpisodeT commentThreads.Should().ContainSingle().Which.Should().Contain("t3_e1x") .And.NotContain("e12", "an \"Episode 12\" thread is a later episode"); } + + // Prefetch targets the next episode, which has no release date. Keyword (title=) searches time out on + // large subreddits, so the newest posts are listed and matched locally instead. + [Fact] + public async Task GetThoughtsAsync_NoReleaseDate_ListsNewestPostsInsteadOfKeywordSearch() + { + List searches = []; + RedditThoughtProvider provider = CreateProvider(req => + { + if (req.RequestUri!.AbsolutePath.EndsWith("/posts/search", StringComparison.Ordinal)) + { + searches.Add(req.RequestUri); + } + + return ArcticShiftOk(req); + }, TooManyRequests); + EpisodeContext undated = new("Love Is Blind", null, "", 11, 1); + + ThoughtResult? result = await provider.GetThoughtsAsync(undated); + + searches.Should().ContainSingle().Which.Query.Should().Contain("sort=desc").And.NotContain("title="); + result!.PostTitle.Should().Be("Season 11 Episode 1 Discussion"); + } }