diff --git a/docker/all-in-one/Dockerfile b/docker/all-in-one/Dockerfile index 303c77e5..4423a1ae 100644 --- a/docker/all-in-one/Dockerfile +++ b/docker/all-in-one/Dockerfile @@ -185,6 +185,8 @@ COPY --from=dotnet-build /publish/worker /app/worker COPY docker/all-in-one/nginx.conf /etc/nginx/conf.d/default.conf # Same security-header snippet the frontend image uses, so the two never drift apart. COPY frontend/nginx-security-headers.conf /etc/nginx/snippets/garagestack-security-headers.conf +# And the same locations, behind this image's own upstream (docker/all-in-one/nginx.conf). +COPY frontend/nginx-app.conf /etc/nginx/snippets/garagestack-app.conf COPY docker/all-in-one/mosquitto.conf /etc/mosquitto/conf.d/garagestack.conf COPY docker/all-in-one/supervisord.conf /etc/supervisor/supervisord.conf COPY --chmod=755 \ diff --git a/docker/all-in-one/nginx.conf b/docker/all-in-one/nginx.conf index b5e1149c..255e925b 100644 --- a/docker/all-in-one/nginx.conf +++ b/docker/all-in-one/nginx.conf @@ -17,76 +17,9 @@ server { # nginx does not inherit add_header into such a location. include /etc/nginx/snippets/garagestack-security-headers.conf; - gzip on; - gzip_types text/plain text/css application/json application/manifest+json application/javascript text/xml application/xml image/svg+xml; + # The API process inside this container. + set $api_upstream http://127.0.0.1:9000; - # GarageStack API -- .NET process listens on localhost:9000 - location /api/ { - proxy_pass http://127.0.0.1:9000/api/; - proxy_http_version 1.1; - # $http_host, not $host: $host drops the port, and a redirect URI missing the port is - # rejected by the identity provider. - proxy_set_header Host $http_host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $garagestack_forwarded_proto; - } - - # SignalR WebSocket hubs - location /hubs/ { - proxy_pass http://127.0.0.1:9000/hubs/; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_set_header Host $http_host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $garagestack_forwarded_proto; - proxy_read_timeout 86400; - } - - # SPA -- all unmatched paths fall back to index.html - location / { - include /etc/nginx/snippets/garagestack-security-headers.conf; - add_header Cache-Control "no-store" always; - try_files $uri $uri/ /index.html; - } - - # SW and manifest must never be HTTP-cached: the browser needs to fetch - # them fresh on every load to detect app updates. Long-caching these causes - # the SW to keep precaching stale asset hashes after a new deployment. - location = /sw.js { - include /etc/nginx/snippets/garagestack-security-headers.conf; - add_header Cache-Control "no-store, no-cache, must-revalidate" always; - try_files $uri =404; - } - location = /registerSW.js { - include /etc/nginx/snippets/garagestack-security-headers.conf; - add_header Cache-Control "no-store, no-cache, must-revalidate" always; - try_files $uri =404; - } - location = /manifest.webmanifest { - include /etc/nginx/snippets/garagestack-security-headers.conf; - add_header Cache-Control "no-store, no-cache, must-revalidate" always; - # mime.types has no entry for .webmanifest, so it went out as application/octet-stream. - # With the types emptied here, default_type names it instead. - types { } - default_type application/manifest+json; - try_files $uri =404; - } - - # Vite's build output: every file name carries a content hash, so a release brings new URLs - # and these can be cached for good. - location ^~ /assets/ { - include /etc/nginx/snippets/garagestack-security-headers.conf; - expires 1y; - add_header Cache-Control "public, immutable"; - } - - # The icons and screenshots from public/ keep their names across releases, so a changed - # one has to reach browsers within a day rather than a year. - location ~* \.(js|css|png|jpg|jpeg|gif|ico|svg|webp|woff2?)$ { - include /etc/nginx/snippets/garagestack-security-headers.conf; - expires 1d; - } + # The locations, shared with the frontend image (frontend/nginx-app.conf). + include /etc/nginx/snippets/garagestack-app.conf; } diff --git a/documentation/ARCHITECTURE.md b/documentation/ARCHITECTURE.md index c4480b89..e1fd9728 100644 --- a/documentation/ARCHITECTURE.md +++ b/documentation/ARCHITECTURE.md @@ -177,6 +177,12 @@ process-local one doesn't already provide. The caches, each in memory or in Post widget, and every SignalR broadcast) uses a short-TTL `IMemoryCache` entry, invalidated immediately on every telemetry write so it can never serve data older than the write that triggered a SignalR broadcast. +- `ITelemetryRepository.GetDailyStatsAsync` (the statistics page, the tyre history and the + slow-puncture check) keeps each finished day in `IMemoryCache`, per vehicle and time zone, and + reads only the span from the first day it lacks to the last: a page opened again reads today + rather than the whole period. A finished day cannot change, since every row is stamped when it + arrives; a day counts as finished an hour after its local midnight, which covers the merge + window folding a message just after midnight into a row stamped before it. - Map POI tiles (charging stations, fuel stations, service areas, speed cameras) are cached in Postgres itself (`PoiCacheTile`/`PoiItem`), not in memory, since that data needs to survive process restarts and be queried by bounding box - a job a plain in-memory cache isn't suited for anyway. The diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 19d175ec..ba87df38 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -41,6 +41,7 @@ RUN echo "os patch date: ${OS_PATCH_DATE}" && apk upgrade --no-cache COPY --from=build /app/dist /usr/share/nginx/html COPY nginx.conf /etc/nginx/conf.d/default.conf COPY nginx-security-headers.conf /etc/nginx/snippets/garagestack-security-headers.conf +COPY nginx-app.conf /etc/nginx/snippets/garagestack-app.conf EXPOSE 80 HEALTHCHECK --interval=10s --timeout=5s --start-period=5s --retries=5 \ CMD wget --quiet --tries=1 --spider http://127.0.0.1/ || exit 1 diff --git a/frontend/nginx-app.conf b/frontend/nginx-app.conf new file mode 100644 index 00000000..9ade201a --- /dev/null +++ b/frontend/nginx-app.conf @@ -0,0 +1,85 @@ +# Everything inside the server block that the frontend image (frontend/nginx.conf) and the +# all-in-one image (docker/all-in-one/nginx.conf) share. Each includes it from its own server +# block, after setting $api_upstream to where its API listens: proxy_pass with a variable and no +# path passes the request URI on unchanged. +# +# The security headers are included again in every location that sets its own add_header, +# because nginx does not inherit add_header into such a location. + +# Text is sent compressed. The build writes a .gz beside every larger file (see +# frontend/scripts/precompress.mjs), compressed once at the highest level, and gzip_static +# serves that instead of compressing each response again; anything without one, the API's +# JSON included, is compressed on the fly. +gzip on; +gzip_static on; +gzip_vary on; +gzip_min_length 1024; +gzip_types text/plain text/css application/json application/manifest+json application/javascript text/xml application/xml image/svg+xml; + +location /api/ { + proxy_pass $api_upstream; + proxy_http_version 1.1; + # $http_host, not $host: $host drops the port, and a redirect URI missing the port is + # rejected by the identity provider. + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $garagestack_forwarded_proto; +} + +# SignalR WebSocket hubs +location /hubs/ { + proxy_pass $api_upstream; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $garagestack_forwarded_proto; + proxy_read_timeout 86400; +} + +location / { + include /etc/nginx/snippets/garagestack-security-headers.conf; + add_header Cache-Control "no-store" always; + try_files $uri $uri/ /index.html; +} + +# SW and manifest must never be HTTP-cached: the browser needs to fetch +# them fresh on every load to detect app updates. Long-caching these causes +# the SW to keep precaching stale asset hashes after a new deployment. +location = /sw.js { + include /etc/nginx/snippets/garagestack-security-headers.conf; + add_header Cache-Control "no-store, no-cache, must-revalidate" always; + try_files $uri =404; +} +location = /registerSW.js { + include /etc/nginx/snippets/garagestack-security-headers.conf; + add_header Cache-Control "no-store, no-cache, must-revalidate" always; + try_files $uri =404; +} +location = /manifest.webmanifest { + include /etc/nginx/snippets/garagestack-security-headers.conf; + add_header Cache-Control "no-store, no-cache, must-revalidate" always; + # mime.types has no entry for .webmanifest, so it went out as application/octet-stream. + # With the types emptied here, default_type names it instead. + types { } + default_type application/manifest+json; + try_files $uri =404; +} + +# Vite's build output: every file name carries a content hash, so a release brings new URLs +# and these can be cached for good. +location ^~ /assets/ { + include /etc/nginx/snippets/garagestack-security-headers.conf; + expires 1y; + add_header Cache-Control "public, immutable"; +} + +# The icons and screenshots from public/ keep their names across releases, so a changed +# one has to reach browsers within a day rather than a year. +location ~* \.(js|css|png|jpg|jpeg|gif|ico|svg|webp|woff2?)$ { + include /etc/nginx/snippets/garagestack-security-headers.conf; + expires 1d; +} diff --git a/frontend/nginx.conf b/frontend/nginx.conf index cb85c098..8bd59fba 100644 --- a/frontend/nginx.conf +++ b/frontend/nginx.conf @@ -24,76 +24,9 @@ server { # file again, because nginx does not inherit add_header into such a location. include /etc/nginx/snippets/garagestack-security-headers.conf; - gzip on; - gzip_types text/plain text/css application/json application/manifest+json application/javascript text/xml application/xml image/svg+xml; + # Where the API container answers. Through a variable, so nginx resolves the name with the + # resolver above instead of once at startup. + set $api_upstream http://api:8080; - location /api/ { - set $api_upstream http://api:8080; - proxy_pass $api_upstream; - proxy_http_version 1.1; - # $http_host, not $host: $host drops the port, and a redirect URI missing the port is - # rejected by the identity provider. - proxy_set_header Host $http_host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $garagestack_forwarded_proto; - } - - # SignalR WebSocket hubs - location /hubs/ { - set $api_upstream http://api:8080; - proxy_pass $api_upstream; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_set_header Host $http_host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $garagestack_forwarded_proto; - proxy_read_timeout 86400; - } - - location / { - include /etc/nginx/snippets/garagestack-security-headers.conf; - add_header Cache-Control "no-store" always; - try_files $uri $uri/ /index.html; - } - - # SW and manifest must never be HTTP-cached: the browser needs to fetch - # them fresh on every load to detect app updates. Long-caching these causes - # the SW to keep precaching stale asset hashes after a new deployment. - location = /sw.js { - include /etc/nginx/snippets/garagestack-security-headers.conf; - add_header Cache-Control "no-store, no-cache, must-revalidate" always; - try_files $uri =404; - } - location = /registerSW.js { - include /etc/nginx/snippets/garagestack-security-headers.conf; - add_header Cache-Control "no-store, no-cache, must-revalidate" always; - try_files $uri =404; - } - location = /manifest.webmanifest { - include /etc/nginx/snippets/garagestack-security-headers.conf; - add_header Cache-Control "no-store, no-cache, must-revalidate" always; - # mime.types has no entry for .webmanifest, so it went out as application/octet-stream. - # With the types emptied here, default_type names it instead. - types { } - default_type application/manifest+json; - try_files $uri =404; - } - - # Vite's build output: every file name carries a content hash, so a release brings new URLs - # and these can be cached for good. - location ^~ /assets/ { - include /etc/nginx/snippets/garagestack-security-headers.conf; - expires 1y; - add_header Cache-Control "public, immutable"; - } - - # The icons and screenshots from public/ keep their names across releases, so a changed - # one has to reach browsers within a day rather than a year. - location ~* \.(js|css|png|jpg|jpeg|gif|ico|svg|webp|woff2?)$ { - include /etc/nginx/snippets/garagestack-security-headers.conf; - expires 1d; - } + include /etc/nginx/snippets/garagestack-app.conf; } diff --git a/frontend/package.json b/frontend/package.json index 87435a54..e2533eac 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -10,7 +10,7 @@ "test:unit": "vitest", "test:coverage": "vitest run --coverage", "test:e2e": "playwright test", - "build-only": "vite build", + "build-only": "vite build && node scripts/precompress.mjs dist", "type-check": "vue-tsc --build", "lint": "pnpm run lint:oxlint && pnpm run lint:eslint && pnpm run lint:css", "lint:oxlint": "oxlint . --fix", diff --git a/frontend/scripts/precompress.mjs b/frontend/scripts/precompress.mjs new file mode 100644 index 00000000..d24149d7 --- /dev/null +++ b/frontend/scripts/precompress.mjs @@ -0,0 +1,37 @@ +// Writes a gzip copy beside every compressible file of the build, for nginx's gzip_static +// (frontend/nginx-app.conf): compressed once here at the highest level instead of at nginx's +// default level 1 on every response. Usage: node scripts/precompress.mjs [dir], dist by default. +import { readdir, readFile, stat, writeFile } from 'node:fs/promises' +import { join } from 'node:path' +import { constants, gzipSync } from 'node:zlib' + +const COMPRESSIBLE = /\.(?:js|mjs|css|html|svg|json|webmanifest|txt)$/ +// Below nginx's gzip_min_length the header costs more than the bytes it saves. +const MIN_BYTES = 1024 + +async function* files(dir) { + for (const entry of await readdir(dir, { withFileTypes: true })) { + const path = join(dir, entry.name) + if (entry.isDirectory()) yield* files(path) + else if (entry.isFile()) yield path + } +} + +const root = process.argv[2] ?? 'dist' +let written = 0 +let before = 0 +let after = 0 +for await (const path of files(root)) { + if (!COMPRESSIBLE.test(path) || (await stat(path)).size < MIN_BYTES) continue + const source = await readFile(path) + const compressed = gzipSync(source, { level: constants.Z_BEST_COMPRESSION }) + // A file that does not shrink is better sent as it is. + if (compressed.length >= source.length) continue + await writeFile(`${path}.gz`, compressed) + written += 1 + before += source.length + after += compressed.length +} + +const kib = (bytes) => `${(bytes / 1024).toFixed(0)} KiB` +console.log(`precompress: ${written} files, ${kib(before)} -> ${kib(after)} gzipped`) diff --git a/src/GarageStack.Core/Helpers/UpstreamRateGate.cs b/src/GarageStack.Core/Helpers/UpstreamRateGate.cs index 69db8e19..47aab420 100644 --- a/src/GarageStack.Core/Helpers/UpstreamRateGate.cs +++ b/src/GarageStack.Core/Helpers/UpstreamRateGate.cs @@ -19,8 +19,8 @@ public enum GateRefusal /// The politeness rules every third-party API client (Open Charge Map, Overpass, Nominatim, /// Valhalla, the RDW) needs: one request in flight at a time, a minimum interval between requests, and a /// backoff window after the upstream answers 429/503/504, as long as its Retry-After asks. A client takes the gate with -/// , calls and -/// around its request, and disposes the entry; the backoff can be read outside the gate so a +/// , sends its request through , which keeps the +/// interval and reads the upstream's answer for a "slow down", and disposes the entry; the backoff can be read outside the gate so a /// foreground caller fails fast without waiting. /// public sealed class UpstreamRateGate @@ -71,11 +71,31 @@ public async Task TryEnterAsync(TimeSpan? waitAtMost, bool failWhenBa } /// - /// Call while holding the gate. Delays until has passed since - /// the previous request and, when is set, until any active - /// backoff window has ended. + /// Call while holding the gate. Sends one request: waits until + /// has passed since the previous one and, with , until any + /// backoff window has closed, then sends it. When the upstream answers that it is busy + /// (), opens the backoff window its Retry-After asks for, or + /// without one, and says how long in the answer. /// - public async Task ThrottleAsync(TimeSpan minInterval, bool honourBackoff, CancellationToken ct) + public async Task SendAsync( + Func> send, + TimeSpan minInterval, + bool honourBackoff, + TimeSpan fallbackBackoff, + CancellationToken ct) + { + await ThrottleAsync(minInterval, honourBackoff, ct); + _lastRequestAt = DateTimeOffset.UtcNow; + var response = await send(ct); + var backoff = IsThrottlingStatus((int)response.StatusCode) + ? SetBackoff(response, fallbackBackoff) + : (TimeSpan?)null; + return new UpstreamAnswer(response, backoff); + } + + // Delays until minInterval has passed since the previous request and, with honourBackoff, + // until any active backoff window has ended. + private async Task ThrottleAsync(TimeSpan minInterval, bool honourBackoff, CancellationToken ct) { var nextAllowed = _lastRequestAt + minInterval; if (honourBackoff && BackoffUntil > nextAllowed) @@ -86,9 +106,6 @@ public async Task ThrottleAsync(TimeSpan minInterval, bool honourBackoff, Cancel await Task.Delay(wait, ct); } - /// Call while holding the gate, right before the HTTP request goes out. - public void MarkRequestSent() => _lastRequestAt = DateTimeOffset.UtcNow; - // The bounds on a Retry-After an upstream sends: a second at the least, so a "0" cannot // turn a refusal into an immediate retry, and an hour at the most, so a mistaken header // cannot switch a feature off for the day. @@ -103,7 +120,7 @@ public void SetBackoff(TimeSpan duration) => /// Retry-After header says, or when it sends none. /// /// How long the window is, for the log line. - public TimeSpan SetBackoff(HttpResponseMessage response, TimeSpan fallback) + internal TimeSpan SetBackoff(HttpResponseMessage response, TimeSpan fallback) { var duration = BackoffFor(response.Headers.RetryAfter, fallback, DateTimeOffset.UtcNow); SetBackoff(duration); @@ -124,7 +141,22 @@ internal static TimeSpan BackoffFor(RetryConditionHeaderValue? retryAfter, TimeS } /// True for the upstream status codes that mean "slow down", not "your request is wrong". - public static bool IsThrottlingStatus(int statusCode) => statusCode is 429 or 503 or 504; + private static bool IsThrottlingStatus(int statusCode) => statusCode is 429 or 503 or 504; +} + +/// +/// What the upstream answered one request through , and +/// whether that answer asked to slow down: is then how long the gate now +/// waits. Disposing it disposes the response. +/// +public sealed class UpstreamAnswer(HttpResponseMessage response, TimeSpan? backoff) : IDisposable +{ + public HttpResponseMessage Response { get; } = response; + + /// How long the gate backs off, when the upstream said it was busy; null otherwise. + public TimeSpan? Backoff { get; } = backoff; + + public void Dispose() => Response.Dispose(); } /// diff --git a/src/GarageStack.Data/Extensions/ServiceCollectionExtensions.cs b/src/GarageStack.Data/Extensions/ServiceCollectionExtensions.cs index 50f763a2..064a69ea 100644 --- a/src/GarageStack.Data/Extensions/ServiceCollectionExtensions.cs +++ b/src/GarageStack.Data/Extensions/ServiceCollectionExtensions.cs @@ -19,22 +19,11 @@ public static IServiceCollection AddGarageStackData(this IServiceCollection serv // before MigrateAsync fails on a real database. services.AddDbContext(opts => opts.UseNpgsql(connectionString)); - services.AddMemoryCache(); + services.AddSharedRepositories(); services.AddScoped(); services.AddScoped(); services.AddScoped(); - services.AddScoped(); - services.AddScoped(); - services.AddScoped(); services.AddScoped(); - services.AddScoped(); - services.AddScoped(); - services.AddScoped(); - services.AddScoped(); - services.AddScoped(); - services.AddScoped(); - services.AddScoped(); - services.AddGarageStackPoiClients(); services.AddGarageStackRdw(); return services; @@ -45,22 +34,11 @@ public static IServiceCollection AddDemoServices(this IServiceCollection service services.AddDbContext(opts => opts.UseInMemoryDatabase("garagestack-demo")); - services.AddMemoryCache(); + // Every repository but these three is the real one, over the in-memory database. + services.AddSharedRepositories(); services.AddScoped(); services.AddSingleton(); services.AddSingleton(); - services.AddScoped(); - services.AddScoped(); - services.AddScoped(); - // The real one: demo settings live in the in-memory database like everything else there. - services.AddScoped(); - services.AddScoped(); - services.AddScoped(); - services.AddScoped(); - services.AddScoped(); - services.AddScoped(); - services.AddScoped(); - services.AddGarageStackPoiClients(); // The RDW card is part of what the demo shows, answered by a stand-in so no plate leaves: // on unless RDW_ENABLED says false. services.AddSingleton(sp => new RdwOptions @@ -73,6 +51,24 @@ public static IServiceCollection AddDemoServices(this IServiceCollection service return services; } + // The repositories demo mode shares with a real deployment, and the upstream clients behind + // them: the same classes over whichever database the caller registered. + private static IServiceCollection AddSharedRepositories(this IServiceCollection services) + { + services.AddMemoryCache(); + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + return services.AddGarageStackPoiClients(); + } + // The RDW lookups. Switched off (RDW_ENABLED unset), the client is still there for the API to // answer "off" with, but it refuses before sending anything, and nothing calls it anyway. private static IServiceCollection AddGarageStackRdw(this IServiceCollection services) diff --git a/src/GarageStack.Data/Repositories/TelemetryRepository.cs b/src/GarageStack.Data/Repositories/TelemetryRepository.cs index 8e9af15e..d5dfc2cd 100644 --- a/src/GarageStack.Data/Repositories/TelemetryRepository.cs +++ b/src/GarageStack.Data/Repositories/TelemetryRepository.cs @@ -91,20 +91,32 @@ public async Task AddAsync(TelemetrySnapshot snapshot, CancellationToken c public async Task MergeIntoAsync(long rowId, TelemetrySnapshot patch, CancellationToken ct = default) { - var existing = await db.TelemetrySnapshots.FindAsync([rowId], ct); - if (existing is null) + // Every MQTT message of a poll lands here, so the row is not read first: a stand-in with + // only its key is attached, the patch's fields are set on it, and the UPDATE writes exactly + // those (last write wins per field). Every telemetry field is nullable, so a field the + // stand-in leaves empty is one the patch did not carry, and stays as the row has it. + var row = db.TelemetrySnapshots.Local.FirstOrDefault(s => s.Id == rowId); + if (row is null) { + row = new TelemetrySnapshot { Id = rowId, VehicleId = patch.VehicleId }; + db.TelemetrySnapshots.Attach(row); + } + TelemetryFields.ApplyNonNullFields(row, patch); + + try + { + await db.SaveChangesAsync(ct); + } + catch (DbUpdateConcurrencyException) + { + // The row is gone (nothing deletes recent telemetry, but a merge must never lose a + // message): the patch becomes a row of its own. + db.Entry(row).State = EntityState.Detached; db.TelemetrySnapshots.Add(patch); await db.SaveChangesAsync(ct); - await NotifyUpdatedAsync(patch.VehicleId, ct); - return; } - // Last-write-wins per field: overwrite with any non-null value from the patch. - TelemetryFields.ApplyNonNullFields(existing, patch); - - await db.SaveChangesAsync(ct); - await NotifyUpdatedAsync(existing.VehicleId, ct); + await NotifyUpdatedAsync(patch.VehicleId, ct); } private async Task NotifyUpdatedAsync(int vehicleId, CancellationToken ct) @@ -220,8 +232,57 @@ private async Task NotifyUpdatedAsync(int vehicleId, CancellationToken ct) return merged; } + // A finished day's statistics cannot change: every row is stamped when it arrives, so no row + // lands on a day that has ended. The margin covers the merge window, which folds messages + // arriving just after midnight into a row stamped before it; compaction folds old rows but + // keeps each day's figures. The expiry only bounds how long an idle period stays in memory. + private static readonly TimeSpan DayFinishedMargin = TimeSpan.FromHours(1); + private static readonly TimeSpan DailyStatCacheTtl = TimeSpan.FromDays(7); + + private static string DailyStatCacheKey(int vehicleId, TimeZoneInfo zone, DateOnly date) => + $"daily-stat/{vehicleId}/{zone.Id}/{date:yyyy-MM-dd}"; + public async Task> GetDailyStatsAsync( int vehicleId, DateOnly from, DateOnly to, TimeZoneInfo zone, DateOnly today, CancellationToken ct = default) + { + if (cache is null) return await ReadDailyStatsAsync(vehicleId, from, to, zone, today, ct); + + // Finished days come from the cache, and only the span from the first day it lacks to the + // last is read: a statistics page opened again reads today (and, for an hour after + // midnight, yesterday) rather than the whole period. + var last = to < today ? to : today; + var finishedBefore = UtcNow - DayFinishedMargin; + bool IsFinished(DateOnly date) => ZoneTime.StartOfDay(date.AddDays(1), zone) <= finishedBefore; + + var byDate = new SortedDictionary(); + DateOnly? firstMissing = null, lastMissing = null; + for (var date = from; date <= last; date = date.AddDays(1)) + { + if (IsFinished(date) && cache.TryGetValue(DailyStatCacheKey(vehicleId, zone, date), out DailyStat? hit) && hit is not null) + { + byDate[date] = hit; + continue; + } + firstMissing ??= date; + lastMissing = date; + } + + if (firstMissing is { } readFrom && lastMissing is { } readTo) + { + // A day the cache held inside the span is read again too; the fresh answer wins. + foreach (var day in await ReadDailyStatsAsync(vehicleId, readFrom, readTo, zone, today, ct)) + { + if (IsFinished(day.Date)) + cache.Set(DailyStatCacheKey(vehicleId, zone, day.Date), day, DailyStatCacheTtl); + byDate[day.Date] = day; + } + } + + return [.. byDate.Values]; + } + + private async Task> ReadDailyStatsAsync( + int vehicleId, DateOnly from, DateOnly to, TimeZoneInfo zone, DateOnly today, CancellationToken ct) { var builder = new DailyStatsBuilder(zone, from, to, today); var (start, end) = DailyStatsBuilder.UtcRange(from, to < today ? to : today, zone); diff --git a/src/GarageStack.Data/Services/NominatimApiClient.cs b/src/GarageStack.Data/Services/NominatimApiClient.cs index 81fdfb55..6d92191d 100644 --- a/src/GarageStack.Data/Services/NominatimApiClient.cs +++ b/src/GarageStack.Data/Services/NominatimApiClient.cs @@ -60,16 +60,14 @@ public sealed class NominatimApiClient( try { - await _gate.ThrottleAsync(MinInterval, honourBackoff: true, ct); - var url = BuildReverseUrl(lat, lng, precision, language); var client = httpClientFactory.CreateClient(HttpClientName); - _gate.MarkRequestSent(); - using var response = await client.GetAsync(url, ct); + using var answer = await _gate.SendAsync( + token => client.GetAsync(url, token), MinInterval, honourBackoff: true, RetryAfter429, ct); + var response = answer.Response; - if (UpstreamRateGate.IsThrottlingStatus((int)response.StatusCode)) + if (answer.Backoff is { } backoff) { - var backoff = _gate.SetBackoff(response, RetryAfter429); logger.LogDebug("Nominatim {Status}, backing off {Seconds}s", (int)response.StatusCode, (int)backoff.TotalSeconds); return null; diff --git a/src/GarageStack.Data/Services/OcmApiClient.cs b/src/GarageStack.Data/Services/OcmApiClient.cs index 3f001edf..16e4c175 100644 --- a/src/GarageStack.Data/Services/OcmApiClient.cs +++ b/src/GarageStack.Data/Services/OcmApiClient.cs @@ -55,8 +55,6 @@ public sealed class OcmApiClient( return null; } - await _gate.ThrottleAsync(MinInterval, honourBackoff: false, ct); - // currenttypeid=30 = DC only; excludes AC slow chargers (Type 1/2, Schuko, etc.) // distance=50 covers the full 0.5°×0.5° tile diagonal at European latitudes (~33 km) var url = BaseUrl + @@ -68,12 +66,12 @@ public sealed class OcmApiClient( request.Headers.Add("X-API-Key", apiKey); var client = httpClientFactory.CreateClient(HttpClientName); - _gate.MarkRequestSent(); - using var response = await client.SendAsync(request, ct); + using var answer = await _gate.SendAsync( + token => client.SendAsync(request, token), MinInterval, honourBackoff: false, RetryAfter429, ct); + var response = answer.Response; - if (UpstreamRateGate.IsThrottlingStatus((int)response.StatusCode)) + if (answer.Backoff is { } backoff) { - var backoff = _gate.SetBackoff(response, RetryAfter429); logger.LogWarning("OCM rate-limited ({Status}); pausing charging station requests for {Seconds}s", (int)response.StatusCode, (int)backoff.TotalSeconds); return null; diff --git a/src/GarageStack.Data/Services/OverpassApiClient.cs b/src/GarageStack.Data/Services/OverpassApiClient.cs index 8d821d7a..4593b9da 100644 --- a/src/GarageStack.Data/Services/OverpassApiClient.cs +++ b/src/GarageStack.Data/Services/OverpassApiClient.cs @@ -87,31 +87,29 @@ public async Task> FetchBackgroundAsync(string poiType, i return null; } - await _gate.ThrottleAsync( + var client = httpClientFactory.CreateClient(HttpClientName); + using var content = new FormUrlEncodedContent([new KeyValuePair("data", query)]); + // A foreground 429 backs off briefly, so later pans don't retry the same rate-limited + // tile and keep getting 429 forever ("area stays blank"); a background one for longer, + // and the foreground path skips Overpass for as long. Either way as long as Overpass + // asks, when it says. + using var answer = await _gate.SendAsync( + token => client.PostAsync(options.BaseUrl, content, token), foreground ? ForegroundMinInterval : BackgroundMinInterval, honourBackoff: !foreground, + foreground ? ForegroundRetryAfter429 : BackgroundRetryAfter429, ct); + var response = answer.Response; - var client = httpClientFactory.CreateClient(HttpClientName); - using var content = new FormUrlEncodedContent([new KeyValuePair("data", query)]); - _gate.MarkRequestSent(); - using var response = await client.PostAsync(options.BaseUrl, content, ct); - - if (UpstreamRateGate.IsThrottlingStatus((int)response.StatusCode)) + if (answer.Backoff is { } backoff) { if (foreground) { - // Short backoff so subsequent pans don't immediately retry the same - // rate-limited tile and keep getting 429 forever ("area stays blank"). - var backoff = _gate.SetBackoff(response, ForegroundRetryAfter429); logger.LogDebug("Overpass {Status} for {PoiType} ({CellLat},{CellLng}) on foreground path, backing off {Seconds}s", (int)response.StatusCode, poiType, cellLat, cellLng, (int)backoff.TotalSeconds); return null; } - // Background: record backoff window so the foreground path skips Overpass for - // as long as Overpass asked, or the next BackgroundRetryAfter429 seconds. - _gate.SetBackoff(response, BackgroundRetryAfter429); throw new HttpRequestException( $"Overpass rate-limited ({(int)response.StatusCode}) for {poiType} ({cellLat},{cellLng})"); } diff --git a/src/GarageStack.Data/Services/RdwApiClient.cs b/src/GarageStack.Data/Services/RdwApiClient.cs index b9f64e12..05a34e67 100644 --- a/src/GarageStack.Data/Services/RdwApiClient.cs +++ b/src/GarageStack.Data/Services/RdwApiClient.cs @@ -90,21 +90,19 @@ private async Task> GetAsync(string dataset, string query, C if (!entry.Entered) throw new RdwUnavailableException("Another RDW lookup is still running"); - await _gate.ThrottleAsync(MinInterval, honourBackoff: false, ct); - using var request = new HttpRequestMessage(HttpMethod.Get, $"{RdwOptions.BaseUrl}/resource/{dataset}.json?{query}"); if (options.AppToken is { } token) request.Headers.Add("X-App-Token", token); var client = httpClientFactory.CreateClient(HttpClientName); - _gate.MarkRequestSent(); try { - using var response = await client.SendAsync(request, ct); + using var answer = await _gate.SendAsync( + token => client.SendAsync(request, token), MinInterval, honourBackoff: false, RetryAfterThrottle, ct); + var response = answer.Response; - if (UpstreamRateGate.IsThrottlingStatus((int)response.StatusCode)) + if (answer.Backoff is { } backoff) { - var backoff = _gate.SetBackoff(response, RetryAfterThrottle); logger.LogWarning("RDW rate-limited ({Status}); pausing lookups for {Seconds}s", (int)response.StatusCode, (int)backoff.TotalSeconds); throw new RdwUnavailableException("The RDW asked to slow down; trying again later"); diff --git a/src/GarageStack.Data/Services/ValhallaApiClient.cs b/src/GarageStack.Data/Services/ValhallaApiClient.cs index f656d13e..119d01a7 100644 --- a/src/GarageStack.Data/Services/ValhallaApiClient.cs +++ b/src/GarageStack.Data/Services/ValhallaApiClient.cs @@ -66,15 +66,15 @@ public async Task MatchAsync(IReadOnlyList points, Ca try { - await _gate.ThrottleAsync(MinInterval, honourBackoff: true, ct); - var client = httpClientFactory.CreateClient(HttpClientName); - _gate.MarkRequestSent(); - using var response = await client.PostAsJsonAsync($"{BaseUrl}/trace_attributes", BuildRequest(points), ct); + var body = BuildRequest(points); + using var answer = await _gate.SendAsync( + token => client.PostAsJsonAsync($"{BaseUrl}/trace_attributes", body, token), + MinInterval, honourBackoff: true, RetryAfter429, ct); + var response = answer.Response; - if (UpstreamRateGate.IsThrottlingStatus((int)response.StatusCode)) + if (answer.Backoff is { } backoff) { - var backoff = _gate.SetBackoff(response, RetryAfter429); logger.LogDebug("Valhalla {Status}, backing off {Seconds}s", (int)response.StatusCode, (int)backoff.TotalSeconds); return TraceMatch.Unavailable; diff --git a/src/GarageStack.Tests/DailyStatsTests.cs b/src/GarageStack.Tests/DailyStatsTests.cs index ba0a5705..b42c4692 100644 --- a/src/GarageStack.Tests/DailyStatsTests.cs +++ b/src/GarageStack.Tests/DailyStatsTests.cs @@ -3,6 +3,7 @@ using GarageStack.Data; using GarageStack.Data.Repositories; using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Caching.Memory; namespace GarageStack.Tests; @@ -135,3 +136,106 @@ public async Task GetDailyStats_FoldsTheStoredReadingsIntoDays() Assert.Null(stats[0].EvSocPercent); } } + +public class TelemetryRepositoryDailyStatsCacheTests +{ + private static readonly TimeZoneInfo Amsterdam = TimeZoneInfo.FindSystemTimeZoneById("Europe/Amsterdam"); + + // 14:00 on 12 October in Amsterdam (UTC+2). + private static readonly DateTimeOffset Now = new(2026, 10, 12, 12, 0, 0, TimeSpan.Zero); + private static readonly DateOnly Today = new(2026, 10, 12); + + private sealed record Setup(AppDbContext Db, TelemetryRepository Repo, int VehicleId, FixedTimeProvider Clock) : IAsyncDisposable + { + public async Task AddReadingAsync(DateTime utc, double soc, CancellationToken ct) + { + Db.TelemetrySnapshots.Add(new TelemetrySnapshot { VehicleId = VehicleId, RecordedAt = utc, EvSocPercent = soc }); + await Db.SaveChangesAsync(ct); + } + + public Task> StatsAsync(DateOnly from, DateOnly to, CancellationToken ct) => + Repo.GetDailyStatsAsync(VehicleId, from, to, Amsterdam, ZoneTime.DateOf(Clock.GetUtcNow().UtcDateTime, Amsterdam), ct); + + public ValueTask DisposeAsync() => Db.DisposeAsync(); + } + + private static async Task SetupAsync(DateTimeOffset now) + { + var db = new AppDbContext(new DbContextOptionsBuilder() + .UseInMemoryDatabase(Guid.NewGuid().ToString()) + .Options); + var vehicle = new Vehicle { Vin = "FAKEVN00000000001" }; + db.Vehicles.Add(vehicle); + await db.SaveChangesAsync(); + var clock = new FixedTimeProvider(now); + var repo = new TelemetryRepository(db, cache: new MemoryCache(new MemoryCacheOptions()), time: clock); + return new Setup(db, repo, vehicle.Id, clock); + } + + [Fact] + public async Task AFinishedDay_IsReadOnce() + { + var ct = TestContext.Current.CancellationToken; + await using var setup = await SetupAsync(Now); + var yesterday = new DateTime(2026, 10, 11, 8, 0, 0, DateTimeKind.Utc); + await setup.AddReadingAsync(yesterday, 40, ct); + + var first = await setup.StatsAsync(Today.AddDays(-1), Today, ct); + // Nothing lands on a finished day in practice; here it shows the answer came from the cache. + await setup.AddReadingAsync(yesterday.AddHours(1), 80, ct); + var second = await setup.StatsAsync(Today.AddDays(-1), Today, ct); + + Assert.Equal(40, first[0].EvSocPercent); + Assert.Equal(40, second[0].EvSocPercent); + } + + [Fact] + public async Task Today_IsReadEveryTime() + { + var ct = TestContext.Current.CancellationToken; + await using var setup = await SetupAsync(Now); + var morning = new DateTime(2026, 10, 12, 7, 0, 0, DateTimeKind.Utc); + await setup.AddReadingAsync(morning, 40, ct); + + await setup.StatsAsync(Today, Today, ct); + await setup.AddReadingAsync(morning.AddHours(1), 80, ct); + var stats = await setup.StatsAsync(Today, Today, ct); + + Assert.Equal(60, Assert.Single(stats).EvSocPercent); + } + + [Fact] + public async Task ADayEndedWithinTheHour_IsReadAgain() + { + var ct = TestContext.Current.CancellationToken; + // 00:30 on 12 October in Amsterdam: yesterday ended half an hour ago. + await using var setup = await SetupAsync(new DateTimeOffset(2026, 10, 11, 22, 30, 0, TimeSpan.Zero)); + var lateEvening = new DateTime(2026, 10, 11, 21, 0, 0, DateTimeKind.Utc); + await setup.AddReadingAsync(lateEvening, 40, ct); + + await setup.StatsAsync(Today.AddDays(-1), Today, ct); + // A message folded into the last row before midnight. + await setup.AddReadingAsync(lateEvening.AddMinutes(59), 80, ct); + var stats = await setup.StatsAsync(Today.AddDays(-1), Today, ct); + + Assert.Equal(60, stats[0].EvSocPercent); + } + + [Fact] + public async Task APeriodWithCachedAndUncachedDays_AnswersAsAColdRead() + { + var ct = TestContext.Current.CancellationToken; + await using var setup = await SetupAsync(Now); + for (var day = 1; day <= 10; day += 3) + await setup.AddReadingAsync(new DateTime(2026, 10, day, 9, 0, 0, DateTimeKind.Utc), day * 5, ct); + var from = new DateOnly(2026, 10, 1); + + // The middle of the period is cached first, then the whole of it is asked for. + await setup.StatsAsync(new DateOnly(2026, 10, 4), new DateOnly(2026, 10, 7), ct); + var warm = await setup.StatsAsync(from, Today, ct); + var cold = await new TelemetryRepository(setup.Db).GetDailyStatsAsync(setup.VehicleId, from, Today, Amsterdam, Today, ct); + + Assert.Equal(cold, warm); + Assert.Equal(12, warm.Count); + } +} diff --git a/src/GarageStack.Tests/TelemetryRepositoryTests.cs b/src/GarageStack.Tests/TelemetryRepositoryTests.cs index 2e7e5323..5c064856 100644 --- a/src/GarageStack.Tests/TelemetryRepositoryTests.cs +++ b/src/GarageStack.Tests/TelemetryRepositoryTests.cs @@ -531,6 +531,61 @@ public class TelemetryRepositoryMergeTests return (db, new TelemetryRepository(db), vehicle); } + // The Worker merges every message in a scope of its own, so the row is never tracked yet. + [Fact] + public async Task MergeIntoAsync_FromAFreshContext_WritesThePatchAndKeepsTheRest() + { + var ct = TestContext.Current.CancellationToken; + var options = new DbContextOptionsBuilder() + .UseInMemoryDatabase(Guid.NewGuid().ToString()) + .Options; + long id; + int vehicleId; + var recordedAt = new DateTime(2026, 10, 19, 8, 0, 0, DateTimeKind.Utc); + await using (var first = new AppDbContext(options)) + { + var vehicle = new Vehicle { Vin = "FAKEVN00000000001" }; + first.Vehicles.Add(vehicle); + await first.SaveChangesAsync(ct); + vehicleId = vehicle.Id; + id = await new TelemetryRepository(first).AddAsync( + new TelemetrySnapshot { VehicleId = vehicleId, RecordedAt = recordedAt, FuelLevelPercent = 60, BatteryVoltage = 12.6 }, ct); + } + + await using (var second = new AppDbContext(options)) + { + await new TelemetryRepository(second).MergeIntoAsync( + id, new TelemetrySnapshot { VehicleId = vehicleId, FuelLevelPercent = 55, EvSocPercent = 80 }, ct); + } + + await using var check = new AppDbContext(options); + var row = Assert.Single(await check.TelemetrySnapshots.ToListAsync(ct)); + Assert.Equal(55, row.FuelLevelPercent); + Assert.Equal(80, row.EvSocPercent); + Assert.Equal(12.6, row.BatteryVoltage); + Assert.Equal(recordedAt, row.RecordedAt); + } + + [Fact] + public async Task MergeIntoAsync_FromAFreshContext_AddsThePatch_WhenTheRowIsGone() + { + var ct = TestContext.Current.CancellationToken; + var options = new DbContextOptionsBuilder() + .UseInMemoryDatabase(Guid.NewGuid().ToString()) + .Options; + await using var db = new AppDbContext(options); + var vehicle = new Vehicle { Vin = "FAKEVN00000000001" }; + db.Vehicles.Add(vehicle); + await db.SaveChangesAsync(ct); + + await using (var merging = new AppDbContext(options)) + await new TelemetryRepository(merging).MergeIntoAsync( + 99999, new TelemetrySnapshot { VehicleId = vehicle.Id, FuelLevelPercent = 70 }, ct); + + var row = Assert.Single(await db.TelemetrySnapshots.AsNoTracking().ToListAsync(ct)); + Assert.Equal(70, row.FuelLevelPercent); + } + [Fact] public async Task AddAsync_ReturnsNewRowId() { diff --git a/src/GarageStack.Tests/UpstreamRateGateTests.cs b/src/GarageStack.Tests/UpstreamRateGateTests.cs index 71574a2f..b93638bd 100644 --- a/src/GarageStack.Tests/UpstreamRateGateTests.cs +++ b/src/GarageStack.Tests/UpstreamRateGateTests.cs @@ -128,4 +128,66 @@ public void SetBackoff_FromAResponse_OpensTheWindowItsRetryAfterAsks() Assert.Equal(TimeSpan.FromSeconds(300), wait); Assert.InRange(gate.BackoffUntil, before.AddSeconds(300), DateTimeOffset.UtcNow.AddSeconds(300)); } + + private static Func> Answering(HttpStatusCode status, TimeSpan? retryAfter = null) => + _ => + { + var response = new HttpResponseMessage(status); + if (retryAfter is { } wait) response.Headers.RetryAfter = new RetryConditionHeaderValue(wait); + return Task.FromResult(response); + }; + + [Fact] + public async Task SendAsync_PassesAnOrdinaryAnswerThrough_WithoutBackingOff() + { + var ct = TestContext.Current.CancellationToken; + var gate = new UpstreamRateGate(); + + using var answer = await gate.SendAsync(Answering(HttpStatusCode.OK), TimeSpan.Zero, honourBackoff: true, Fallback, ct); + + Assert.Equal(HttpStatusCode.OK, answer.Response.StatusCode); + Assert.Null(answer.Backoff); + Assert.False(gate.IsBackingOff); + } + + [Theory] + [InlineData(HttpStatusCode.TooManyRequests)] + [InlineData(HttpStatusCode.ServiceUnavailable)] + [InlineData(HttpStatusCode.GatewayTimeout)] + public async Task SendAsync_BacksOffAsLongAsABusyUpstreamAsks(HttpStatusCode status) + { + var ct = TestContext.Current.CancellationToken; + var gate = new UpstreamRateGate(); + + using var answer = await gate.SendAsync(Answering(status, TimeSpan.FromSeconds(300)), TimeSpan.Zero, honourBackoff: true, Fallback, ct); + + Assert.Equal(TimeSpan.FromSeconds(300), answer.Backoff); + Assert.True(gate.IsBackingOff); + } + + [Fact] + public async Task SendAsync_BacksOffForTheFallback_WhenABusyUpstreamSaysNothing() + { + var ct = TestContext.Current.CancellationToken; + var gate = new UpstreamRateGate(); + + using var answer = await gate.SendAsync(Answering(HttpStatusCode.TooManyRequests), TimeSpan.Zero, honourBackoff: true, Fallback, ct); + + Assert.Equal(Fallback, answer.Backoff); + } + + [Fact] + public async Task SendAsync_KeepsTheIntervalBetweenRequests() + { + var ct = TestContext.Current.CancellationToken; + var gate = new UpstreamRateGate(); + var interval = TimeSpan.FromMilliseconds(200); + + using (await gate.SendAsync(Answering(HttpStatusCode.OK), interval, honourBackoff: false, Fallback, ct)) { } + var started = DateTimeOffset.UtcNow; + using (await gate.SendAsync(Answering(HttpStatusCode.OK), interval, honourBackoff: false, Fallback, ct)) { } + + // A little short of the interval, for the clock's resolution. + Assert.True(DateTimeOffset.UtcNow - started >= interval - TimeSpan.FromMilliseconds(20)); + } }