Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 57 additions & 29 deletions Jobs/XkcdJob.cs
Original file line number Diff line number Diff line change
Expand Up @@ -22,33 +22,31 @@ public class XkcdJob(DB db, DiscordWebhookService discordWebhook, LogsService lo
// Public square avatar for the xkcd identity (data: URLs are rejected by Discord's avatar_url).
private const string XkcdAvatarUrl = "https://pbs.twimg.com/profile_images/1488600831377252354/hEpPeSu0_400x400.jpg";

private record XkcdItem(string Title, string Link);
internal record XkcdItem(string Title, string Link);

internal static bool ShouldFetchFeed(int subscriptionCount) => subscriptionCount > 0;

public async Task Execute(IJobExecutionContext context)
{
CancellationToken cancellationToken = context.CancellationToken;
List<XkcdSubscription> subscriptions = await db.XkcdSubscriptions
.Include(s => s.Webhook)
.ToListAsync();
.ToListAsync(cancellationToken);

// Leave the feed unseeded until there is somewhere to post the initial comic.
if (!ShouldFetchFeed(subscriptions.Count))
return;

bool hasSeen = await db.XkcdSeen.AnyAsync();
bool hasSeen = await db.XkcdSeen.AnyAsync(cancellationToken);

List<XkcdItem> items;
try
{
string rss = await HttpClient.GetStringAsync("https://xkcd.com/rss.xml");
XDocument doc = XDocument.Parse(rss);
items = doc.Descendants("item")
.Select(x => new XkcdItem(
x.Element("title")?.Value ?? string.Empty,
x.Element("link")?.Value ?? string.Empty))
.Where(i => !string.IsNullOrEmpty(i.Link))
.ToList();
items = await FetchItemsAsync(HttpClient, cancellationToken);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
throw;
}
catch (Exception ex)
{
Expand All @@ -68,11 +66,14 @@ public async Task Execute(IJobExecutionContext context)
string latestLink = await db.XkcdDeliveryRetries
.OrderBy(r => r.LastAttemptAt)
.Select(r => r.Link)
.FirstOrDefaultAsync() ?? items[0].Link;
bool delivered = await DispatchAsync(latestLink, subscriptions, SendAsync);
.FirstOrDefaultAsync(cancellationToken) ?? items[0].Link;
bool delivered = await DispatchAsync(
latestLink,
subscriptions,
(subscription, link) => SendAsync(subscription, link, cancellationToken));
if (!delivered)
{
int attempts = await RecordFailedDeliveryAsync(db, latestLink, DateTime.UtcNow);
int attempts = await RecordFailedDeliveryAsync(db, latestLink, DateTime.UtcNow, cancellationToken);
if (ShouldRetryDelivery(attempts))
return;

Expand All @@ -81,30 +82,30 @@ public async Task Execute(IJobExecutionContext context)
LogSeverity.Warning);
}

await ClearDeliveryRetryAsync(latestLink);
await ClearDeliveryRetryAsync(latestLink, cancellationToken);

foreach (XkcdItem item in items)
db.XkcdSeen.Add(new XkcdSeen { Link = item.Link, SeenAt = DateTime.UtcNow });
if (items.All(item => item.Link != latestLink))
db.XkcdSeen.Add(new XkcdSeen { Link = latestLink, SeenAt = DateTime.UtcNow });

await db.SaveChangesAsync();
await db.SaveChangesAsync(cancellationToken);
return;
}

List<string> feedLinks = items.Select(i => i.Link).ToList();
HashSet<string> seen = (await db.XkcdSeen
.Where(x => feedLinks.Contains(x.Link))
.Select(x => x.Link)
.ToListAsync())
.ToListAsync(cancellationToken))
.ToHashSet();

// Retry pending comics first, including links that have rotated out of the RSS feed.
// RSS is newest-first; post other new comics oldest-first so they read chronologically.
List<string> pendingLinks = await db.XkcdDeliveryRetries
.OrderBy(r => r.LastAttemptAt)
.Select(r => r.Link)
.ToListAsync();
.ToListAsync(cancellationToken);
HashSet<string> pendingLinkSet = pendingLinks.ToHashSet();
List<string> deliveryLinks =
[
Expand All @@ -119,10 +120,13 @@ .. items

foreach (string link in deliveryLinks)
{
bool delivered = await DispatchAsync(link, subscriptions, SendAsync);
bool delivered = await DispatchAsync(
link,
subscriptions,
(subscription, entryLink) => SendAsync(subscription, entryLink, cancellationToken));
if (!delivered)
{
int attempts = await RecordFailedDeliveryAsync(db, link, DateTime.UtcNow);
int attempts = await RecordFailedDeliveryAsync(db, link, DateTime.UtcNow, cancellationToken);
if (ShouldRetryDelivery(attempts))
continue;

Expand All @@ -131,11 +135,23 @@ .. items
LogSeverity.Warning);
}

await ClearDeliveryRetryAsync(link);
await ClearDeliveryRetryAsync(link, cancellationToken);
db.XkcdSeen.Add(new XkcdSeen { Link = link, SeenAt = DateTime.UtcNow });
}

await db.SaveChangesAsync();
await db.SaveChangesAsync(cancellationToken);
}

internal static async Task<List<XkcdItem>> FetchItemsAsync(HttpClient httpClient, CancellationToken cancellationToken)
{
string rss = await httpClient.GetStringAsync("https://xkcd.com/rss.xml", cancellationToken);
XDocument doc = XDocument.Parse(rss);
return doc.Descendants("item")
.Select(x => new XkcdItem(
x.Element("title")?.Value ?? string.Empty,
x.Element("link")?.Value ?? string.Empty))
.Where(i => !string.IsNullOrEmpty(i.Link))
.ToList();
}

internal static async Task<bool> DispatchAsync(
Expand All @@ -155,9 +171,14 @@ internal static async Task<bool> DispatchAsync(

internal static bool ShouldRetryDelivery(int attemptCount) => attemptCount < MaxDeliveryAttempts;

internal static async Task<int> RecordFailedDeliveryAsync(DB db, string link, DateTime attemptedAt)
internal static async Task<int> RecordFailedDeliveryAsync(
DB db,
string link,
DateTime attemptedAt,
CancellationToken cancellationToken = default)
{
XkcdDeliveryRetry? retry = await db.XkcdDeliveryRetries.SingleOrDefaultAsync(r => r.Link == link);
XkcdDeliveryRetry? retry = await db.XkcdDeliveryRetries
.SingleOrDefaultAsync(r => r.Link == link, cancellationToken);
if (retry == null)
{
retry = new XkcdDeliveryRetry
Expand All @@ -174,26 +195,33 @@ internal static async Task<int> RecordFailedDeliveryAsync(DB db, string link, Da
retry.LastAttemptAt = attemptedAt;
}

await db.SaveChangesAsync();
await db.SaveChangesAsync(cancellationToken);
return retry.AttemptCount;
}

private async Task ClearDeliveryRetryAsync(string link)
private async Task ClearDeliveryRetryAsync(string link, CancellationToken cancellationToken)
{
XkcdDeliveryRetry? retry = await db.XkcdDeliveryRetries.SingleOrDefaultAsync(r => r.Link == link);
XkcdDeliveryRetry? retry = await db.XkcdDeliveryRetries
.SingleOrDefaultAsync(r => r.Link == link, cancellationToken);
if (retry != null)
db.XkcdDeliveryRetries.Remove(retry);
}

private async Task<bool> SendAsync(XkcdSubscription sub, string link)
private async Task<bool> SendAsync(XkcdSubscription sub, string link, CancellationToken cancellationToken)
{
if (sub.Webhook == null)
{
logsService.Log($"XkcdJob: no webhook available in channel {sub.ChannelDiscordId}", LogSeverity.Warning);
return false;
}

bool ok = await discordWebhook.SendAsync(sub.Webhook.WebhookId, sub.Webhook.Token, link, XkcdUsername, XkcdAvatarUrl);
bool ok = await discordWebhook.SendAsync(
sub.Webhook.WebhookId,
sub.Webhook.Token,
link,
XkcdUsername,
XkcdAvatarUrl,
cancellationToken);
if (!ok)
logsService.Log($"XkcdJob: failed to post comic to channel {sub.ChannelDiscordId}", LogSeverity.Warning);

Expand Down
11 changes: 11 additions & 0 deletions Morpheus.Tests/XkcdJobTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,17 @@ public async Task DispatchAsync_WhenOneDeliveryFails_ReportsFailureAndAttemptsEv
Assert.Equal([1UL, 2UL], attemptedChannels);
}

[Fact]
public async Task FetchItemsAsync_WhenCallerCancels_PropagatesCancellation()
{
using HttpClient httpClient = new();
using CancellationTokenSource cancellation = new();
await cancellation.CancelAsync();

await Assert.ThrowsAnyAsync<OperationCanceledException>(
() => XkcdJob.FetchItemsAsync(httpClient, cancellation.Token));
}

[Fact]
public async Task RecordFailedDeliveryAsync_PersistsOneWeekOfHourlyAttempts()
{
Expand Down