diff --git a/api/api/query_logs.go b/api/api/query_logs.go index f79a7b4d..b1860a13 100644 --- a/api/api/query_logs.go +++ b/api/api/query_logs.go @@ -19,7 +19,7 @@ import ( // @Param id path string true "Profile ID" // @Param page query int false "specify page number" default(1) // @Param limit query int false "specify logs limit by page" default(100) -// @Param status query string false "specify status for query" default("all") +// @Param status query string false "specify status for query" Enums(all,blocked,processed,unanswered) default("all") // @Param timespan query string false "specify timespan for query" default("LAST_1_HOUR") // @Param device_id query string false "specify device ID for filtering" // @Param search query string false "substring (case-insensitive) match against stored domain; free-form (short inputs may scan more)" @@ -36,7 +36,7 @@ func (s *APIServer) getProfileQueryLogs() fiber.Handler { Page: c.QueryInt("page", 1), Limit: c.QueryInt("limit", 25), Timespan: c.Query("timespan", model.LAST_1_HOUR), - Status: c.Query("status", "all"), + Status: c.Query("status", model.QueryLogStatusAll), DeviceId: c.Query("device_id", ""), Search: c.Query("search", ""), SortBy: c.Query("sort_by", "created"), diff --git a/api/api/query_logs_test.go b/api/api/query_logs_test.go index 6c46ebc2..fa657a35 100644 --- a/api/api/query_logs_test.go +++ b/api/api/query_logs_test.go @@ -65,6 +65,32 @@ func (s *QueryLogsAPIShortSuite) auth(req *http.Request) { s.db.On("GetSession", mock.Anything, qlSessTok).Return(model.Session{AccountID: qlAccID}, true, nil) } +// tableRef: query-log-outcomes-behaviour.md #C5 +func (s *QueryLogsAPIShortSuite) TestGetLogsUnansweredStatusAccepted() { + logs := []model.QueryLog{{ProfileID: qlProfile, Status: "processed", Outcome: "timeout", Timestamp: time.Now(), DNSRequest: model.DNSRequest{Domain: "example.com"}}} + s.svc.On("GetProfileQueryLogs", mock.Anything, qlAccID, qlProfile, "unanswered", "LAST_1_HOUR", "", "", "created", 1, 25).Return(logs, nil) + req := httptest.NewRequest(http.MethodGet, "/api/v1/profiles/"+qlProfile+"/logs?page=1&limit=25&status=unanswered×pan=LAST_1_HOUR", nil) + s.auth(req) + resp, err := s.server().App.Test(req, -1) + require.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusOK, resp.StatusCode) + var out []model.QueryLog + require.NoError(s.T(), json.NewDecoder(resp.Body).Decode(&out)) + assert.Len(s.T(), out, 1) +} + +// specRef: query-log-outcomes-behaviour.md #C4 — the row status "unavailable" +// is not a filter value; those rows are reached through "unanswered". +func (s *QueryLogsAPIShortSuite) TestGetLogsUnknownStatusRejected() { + for _, status := range []string{"dropped", "unavailable"} { + req := httptest.NewRequest(http.MethodGet, "/api/v1/profiles/"+qlProfile+"/logs?page=1&limit=25&status="+status+"×pan=LAST_1_HOUR", nil) + s.auth(req) + resp, err := s.server().App.Test(req, -1) + require.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusBadRequest, resp.StatusCode, status) + } +} + func (s *QueryLogsAPIShortSuite) TestGetLogsSuccess() { logs := []model.QueryLog{{ProfileID: qlProfile, Status: "processed", Timestamp: time.Now(), DNSRequest: model.DNSRequest{Domain: "example.com"}}} s.svc.On("GetProfileQueryLogs", mock.Anything, qlAccID, qlProfile, "processed", "LAST_1_HOUR", "", "", "created", 1, 25).Return(logs, nil) diff --git a/api/api/requests/query_logs.go b/api/api/requests/query_logs.go index 87e50033..fb1addea 100644 --- a/api/api/requests/query_logs.go +++ b/api/api/requests/query_logs.go @@ -11,7 +11,7 @@ type QueryLogsQueryParams struct { Page int `json:"page" validate:"required,numeric,min=1"` Limit int `json:"limit" validate:"required,oneof=10 25 50 100"` Timespan string `json:"timespan" validate:"oneof=LAST_1_HOUR LAST_12_HOURS LAST_1_DAY LAST_7_DAYS LAST_MONTH"` - Status string `json:"status" validate:"omitempty,oneof=all blocked processed"` + Status string `json:"status" validate:"omitempty,oneof=all blocked processed unanswered"` DeviceId string `json:"device_id" validate:"omitempty"` Search string `json:"search" validate:"omitempty,max=256"` SortBy string `json:"sort_by" validate:"oneof=created domain client_ip"` diff --git a/api/db/mongodb/migrations/026_statistics_clear_legacy.down.json b/api/db/mongodb/migrations/026_statistics_clear_legacy.down.json new file mode 100644 index 00000000..fe51488c --- /dev/null +++ b/api/db/mongodb/migrations/026_statistics_clear_legacy.down.json @@ -0,0 +1 @@ +[] diff --git a/api/db/mongodb/migrations/026_statistics_clear_legacy.up.json b/api/db/mongodb/migrations/026_statistics_clear_legacy.up.json new file mode 100644 index 00000000..c71596d5 --- /dev/null +++ b/api/db/mongodb/migrations/026_statistics_clear_legacy.up.json @@ -0,0 +1,12 @@ +[ + { + "delete": "statistics", + "deletes": [ + { + "q": {}, + "limit": 0 + } + ], + "writeConcern": { "w": "majority" } + } +] diff --git a/api/db/mongodb/migrations/README.md b/api/db/mongodb/migrations/README.md index 975c7313..8ea86b25 100644 --- a/api/db/mongodb/migrations/README.md +++ b/api/db/mongodb/migrations/README.md @@ -14,6 +14,18 @@ the target environment: case-collision groups must be zero, and emails must be A without surrounding whitespace (`$toLower` is ASCII-only; the migration does not trim). Audit queries are in the PR that introduced the migration. +### Migration 026 (statistics clear) + +Clears the legacy per-profile `statistics` time-series collection. Query statistics are +now service-wide and live in `service_statistics` (a regular collection the proxy upserts +into, one document per PoP and hour, no TTL); +per-profile statistics return with the Analytics page in a new shape. `delete` with an empty +filter, not `drop`, so a fresh database without the collection succeeds; an empty-filter +delete on a time-series collection needs MongoDB ≥ 7.0. Idempotent; the down migration is +a no-op. Deploy note: proxies still running the previous release between the DCN and DFN +restarts may recreate `statistics` as a plain collection; after the DFN restart, drop it if +`db.statistics.countDocuments({})` is non-zero. + ### Query logs collections Note: Query logs time-series collections are created by the proxy service. Their only index is the `{profile_id, timestamp}` meta+time index MongoDB creates automatically on time-series creation (≥6.3) — no code creates query-log indexes explicitly (verified against prod, moddns-shadow#688). diff --git a/api/db/mongodb/query_logs.go b/api/db/mongodb/query_logs.go index 32f7dc21..2ca65a95 100644 --- a/api/db/mongodb/query_logs.go +++ b/api/db/mongodb/query_logs.go @@ -78,7 +78,11 @@ func (r *QueryLogsRepository) GetQueryLogs(ctx context.Context, profileId string }, }) } - if status != "all" { + switch status { + case model.QueryLogStatusAll: + case model.QueryLogStatusUnanswered: + matchFilter = append(matchFilter, unansweredFilter()) + default: matchFilter = append(matchFilter, bson.E{ Key: "status", Value: status, @@ -247,3 +251,9 @@ func (r *QueryLogsRepository) getCollObject(retention model.Retention) *mongo.Co return r.queryLogsCollOneHour } } + +// unansweredFilter selects the "No answer" class by outcome +// (query-log-outcomes-behaviour.md C5). +func unansweredFilter() bson.E { + return bson.E{Key: "outcome", Value: bson.D{{Key: "$in", Value: model.UnansweredOutcomes}}} +} diff --git a/api/docs/docs.go b/api/docs/docs.go index 34546f3a..3a3aa353 100644 --- a/api/docs/docs.go +++ b/api/docs/docs.go @@ -1790,6 +1790,12 @@ const docTemplate = `{ "in": "query" }, { + "enum": [ + "all", + "blocked", + "processed", + "unanswered" + ], "type": "string", "default": "\"all\"", "description": "specify status for query", diff --git a/api/docs/swagger.json b/api/docs/swagger.json index 30116cc5..876ef020 100644 --- a/api/docs/swagger.json +++ b/api/docs/swagger.json @@ -1782,6 +1782,12 @@ "in": "query" }, { + "enum": [ + "all", + "blocked", + "processed", + "unanswered" + ], "type": "string", "default": "\"all\"", "description": "specify status for query", diff --git a/api/docs/swagger.yaml b/api/docs/swagger.yaml index bbba5650..89b49579 100644 --- a/api/docs/swagger.yaml +++ b/api/docs/swagger.yaml @@ -2567,6 +2567,11 @@ paths: type: integer - default: '"all"' description: specify status for query + enum: + - all + - blocked + - processed + - unanswered in: query name: status type: string diff --git a/api/model/query_log_outcome.go b/api/model/query_log_outcome.go new file mode 100644 index 00000000..636946a6 --- /dev/null +++ b/api/model/query_log_outcome.go @@ -0,0 +1,41 @@ +package model + +// Values of the logs endpoint's status filter. Blocked and processed match +// QueryLog.Status as written by the proxy; unanswered is a pseudo-status +// resolved by outcome (query-log-outcomes-behaviour.md C5). The proxy also +// writes status "unavailable" (O11); those rows are reached through +// unanswered, never selected by status. +const ( + QueryLogStatusAll = "all" + QueryLogStatusBlocked = "blocked" + QueryLogStatusProcessed = "processed" + QueryLogStatusUnanswered = "unanswered" +) + +// Resolution-outcome tokens written by the proxy into QueryLog.Outcome. +// Decision table: docs/specs/query-log-outcomes-behaviour.md (rows O1–O11). +const ( + OutcomeResolved = "resolved" + OutcomeNoData = "nodata" + OutcomeNXDomain = "nxdomain" + OutcomeBlocked = "blocked" + OutcomeServfailDNSSEC = "servfail_dnssec" + OutcomeServfailUpstream = "servfail_upstream" + OutcomeTimeout = "timeout" + OutcomeNetworkError = "network_error" + OutcomeRefused = "refused" + OutcomeFilterUnavailable = "filter_unavailable" +) + +// UnansweredOutcomes is the "No answer" class for the logs status filter, +// the same set the collapsed-row label uses (query-log-outcomes-behaviour.md +// C3, C5): the service could not answer. Deliberate verdicts — blocked, and +// DNSSEC validation failures — are not failures and stay out. Every stored +// row carries an outcome: the field predates the longest retention window. +var UnansweredOutcomes = []string{ + OutcomeServfailUpstream, + OutcomeTimeout, + OutcomeNetworkError, + OutcomeRefused, + OutcomeFilterUnavailable, +} diff --git a/api/service/query_logs/service.go b/api/service/query_logs/service.go index 82e4fe9b..af118433 100644 --- a/api/service/query_logs/service.go +++ b/api/service/query_logs/service.go @@ -7,12 +7,6 @@ import ( "github.com/ivpn/dns/api/model" ) -const ( - STATUS_ALL = "all" - STATUS_BLOCKED = "blocked" - STATUS_PROCESSED = "processed" -) - type QueryLogsService struct { QueryLogsRepository repository.QueryLogsRepository } @@ -37,7 +31,7 @@ func (q *QueryLogsService) GetProfileQueryLogs(ctx context.Context, profileId st } func (q *QueryLogsService) DownloadProfileQueryLogs(ctx context.Context, profileId string, retention model.Retention, page, limit int) ([]model.QueryLog, error) { - logs, err := q.QueryLogsRepository.GetQueryLogs(ctx, profileId, retention, STATUS_ALL, 0, "", "", "created", page, limit) + logs, err := q.QueryLogsRepository.GetQueryLogs(ctx, profileId, retention, model.QueryLogStatusAll, 0, "", "", "created", page, limit) if err != nil { return nil, err } diff --git a/api/service/query_logs/service_test.go b/api/service/query_logs/service_test.go index c7896e98..8d9e3e82 100644 --- a/api/service/query_logs/service_test.go +++ b/api/service/query_logs/service_test.go @@ -191,6 +191,13 @@ func (s *QueryLogsServiceSuite) seedQueryLogs(ctx context.Context) { bson.D{{Key: "timestamp", Value: now.Add(-3 * time.Hour)}, {Key: "profile_id", Value: s.profileID}, {Key: "device_id", Value: "phone"}, {Key: "status", Value: "processed"}, {Key: "reasons", Value: bson.A{}}, {Key: "dns_request", Value: bson.D{{Key: "domain", Value: "sub.example.com"}, {Key: "query_type", Value: "AAAA"}, {Key: "response_code", Value: "NOERROR"}, {Key: "dnssec", Value: true}}}, {Key: "client_ip", Value: "1.2.3.5"}, {Key: "protocol", Value: "udp"}}, bson.D{{Key: "timestamp", Value: now.Add(-25 * time.Hour)}, {Key: "profile_id", Value: s.profileID}, {Key: "device_id", Value: "laptop"}, {Key: "status", Value: "blocked"}, {Key: "reasons", Value: bson.A{"tracker"}}, {Key: "dns_request", Value: bson.D{{Key: "domain", Value: "old.example.com"}, {Key: "query_type", Value: "A"}, {Key: "response_code", Value: "NOERROR"}, {Key: "dnssec", Value: false}}}, {Key: "client_ip", Value: "1.2.3.6"}, {Key: "protocol", Value: "udp"}}, // outside 1d timespan bson.D{{Key: "timestamp", Value: now.Add(-1 * time.Hour)}, {Key: "profile_id", Value: s.profileID}, {Key: "device_id", Value: "tablet"}, {Key: "status", Value: "processed"}, {Key: "reasons", Value: bson.A{}}, {Key: "dns_request", Value: bson.D{{Key: "domain", Value: "example.org"}, {Key: "query_type", Value: "A"}, {Key: "response_code", Value: "NXDOMAIN"}, {Key: "dnssec", Value: false}}}, {Key: "client_ip", Value: "1.2.3.7"}, {Key: "protocol", Value: "udp"}}, + // specRef: query-log-outcomes-behaviour.md #O11 — answered SERVFAIL by the proxy, neither blocked nor processed + bson.D{{Key: "timestamp", Value: now.Add(-4 * time.Hour)}, {Key: "profile_id", Value: s.profileID}, {Key: "device_id", Value: "laptop"}, {Key: "status", Value: "unavailable"}, {Key: "reasons", Value: bson.A{}}, {Key: "outcome", Value: "filter_unavailable"}, {Key: "dns_request", Value: bson.D{{Key: "domain", Value: "unavailable.example.net"}, {Key: "query_type", Value: "A"}, {Key: "response_code", Value: "SERVFAIL"}, {Key: "dnssec", Value: false}}}, {Key: "client_ip", Value: "1.2.3.8"}, {Key: "protocol", Value: "udp"}}, + // "No answer" class (C5): the timeout row matches; the DNSSEC verdict and + // the outcome-less REFUSED row stand in for rows the filter must NOT match. + bson.D{{Key: "timestamp", Value: now.Add(-5 * time.Hour)}, {Key: "profile_id", Value: s.profileID}, {Key: "device_id", Value: "laptop"}, {Key: "status", Value: "processed"}, {Key: "reasons", Value: bson.A{}}, {Key: "outcome", Value: "timeout"}, {Key: "dns_request", Value: bson.D{{Key: "domain", Value: "timeout.unanswered.test"}, {Key: "query_type", Value: "A"}, {Key: "response_code", Value: "SERVFAIL"}, {Key: "dnssec", Value: false}}}, {Key: "client_ip", Value: "1.2.3.8"}, {Key: "protocol", Value: "udp"}}, + bson.D{{Key: "timestamp", Value: now.Add(-6 * time.Hour)}, {Key: "profile_id", Value: s.profileID}, {Key: "device_id", Value: "laptop"}, {Key: "status", Value: "processed"}, {Key: "reasons", Value: bson.A{"dnssec_failed"}}, {Key: "outcome", Value: "servfail_dnssec"}, {Key: "dns_request", Value: bson.D{{Key: "domain", Value: "dnssec.unanswered.test"}, {Key: "query_type", Value: "A"}, {Key: "response_code", Value: "SERVFAIL"}, {Key: "dnssec", Value: false}}}, {Key: "client_ip", Value: "1.2.3.8"}, {Key: "protocol", Value: "udp"}}, + bson.D{{Key: "timestamp", Value: now.Add(-7 * time.Hour)}, {Key: "profile_id", Value: s.profileID}, {Key: "device_id", Value: "laptop"}, {Key: "status", Value: "processed"}, {Key: "reasons", Value: bson.A{}}, {Key: "dns_request", Value: bson.D{{Key: "domain", Value: "legacy.unanswered.test"}, {Key: "query_type", Value: "A"}, {Key: "response_code", Value: "REFUSED"}, {Key: "dnssec", Value: false}}}, {Key: "client_ip", Value: "1.2.3.8"}, {Key: "protocol", Value: "udp"}}, // Another profile for isolation bson.D{{Key: "timestamp", Value: now.Add(-2 * time.Hour)}, {Key: "profile_id", Value: "other-profile"}, {Key: "device_id", Value: "laptop"}, {Key: "status", Value: "blocked"}, {Key: "reasons", Value: bson.A{"malware"}}, {Key: "dns_request", Value: bson.D{{Key: "domain", Value: "example.com"}, {Key: "query_type", Value: "A"}, {Key: "response_code", Value: "NOERROR"}, {Key: "dnssec", Value: false}}}, {Key: "client_ip", Value: "9.9.9.9"}, {Key: "protocol", Value: "udp"}}, } @@ -218,7 +225,11 @@ func (s *QueryLogsServiceSuite) TestGetProfileQueryLogs() { {"blocked search example within 1d", "blocked", "LAST_1_DAY", "", "example", "created", 0, 0, 1, "example"}, // Only sub.example.com matches processed status; example.com is blocked. Expect 1 result. {"processed search com within 1d", "processed", "LAST_1_DAY", "", "com", "created", 0, 0, 1, "com"}, - {"all no search within 1d", "all", "LAST_1_DAY", "", "", "created", 0, 0, 3, ""}, // excludes old.chatgpt.com outside 1d + {"all no search within 1d", "all", "LAST_1_DAY", "", "", "created", 0, 0, 7, ""}, // excludes old.example.com outside 1d + // tableRef: query-log-outcomes-behaviour.md #C5 — outcome-based class (C3 set); DNSSEC verdicts and rows without an outcome are not matched + {"unanswered selects every no-answer outcome", "unanswered", "LAST_1_DAY", "", "", "created", 0, 0, 2, ""}, + {"unanswered excludes resolved and blocked rows", "unanswered", "LAST_1_DAY", "", "example.com", "created", 0, 0, 0, ""}, + {"unanswered combines with device filter", "unanswered", "LAST_1_DAY", "tablet", "", "created", 0, 0, 0, ""}, {"device filtered processed", "processed", "LAST_1_DAY", "tablet", "", "created", 0, 0, 1, "example.org"}, {"pagination first page size 1", "processed", "LAST_1_DAY", "", "com", "created", 1, 1, 1, "com"}, {"search miss returns empty", "blocked", "LAST_1_DAY", "", "nomatch", "created", 0, 0, 0, ""}, @@ -245,23 +256,23 @@ func (s *QueryLogsServiceSuite) TestGetProfileQueryLogsSorting() { s.Run("domain ascending", func() { logs, err := s.service.GetProfileQueryLogs(ctx, s.profileID, retention, "all", "LAST_7_DAYS", "", "", "domain", 0, 0) s.Require().NoError(err) - s.Equal(4, len(logs)) + s.Equal(8, len(logs)) domains := []string{} for _, l := range logs { domains = append(domains, l.DNSRequest.Domain) } - s.Equal([]string{"example.com", "example.org", "old.example.com", "sub.example.com"}, domains) + s.Equal([]string{"dnssec.unanswered.test", "example.com", "example.org", "legacy.unanswered.test", "old.example.com", "sub.example.com", "timeout.unanswered.test", "unavailable.example.net"}, domains) }) s.Run("client ip ascending", func() { logs, err := s.service.GetProfileQueryLogs(ctx, s.profileID, retention, "all", "LAST_7_DAYS", "", "", "client_ip", 0, 0) s.Require().NoError(err) - s.Equal(4, len(logs)) + s.Equal(8, len(logs)) ips := []string{} for _, l := range logs { ips = append(ips, l.ClientIP) } - s.Equal([]string{"1.2.3.4", "1.2.3.5", "1.2.3.6", "1.2.3.7"}, ips) + s.Equal([]string{"1.2.3.4", "1.2.3.5", "1.2.3.6", "1.2.3.7", "1.2.3.8", "1.2.3.8", "1.2.3.8", "1.2.3.8"}, ips) }) } @@ -272,8 +283,8 @@ func (s *QueryLogsServiceSuite) TestDownloadProfileQueryLogs() { logs, err := s.service.DownloadProfileQueryLogs(ctx, s.profileID, retention, 0, 0) s.Require().NoError(err) - // Should include the document outside 1d window (old.chatgpt.com) but not other profile's logs. - s.Equal(4, len(logs), "download should return all 4 logs for profile") + // Should include the document outside 1d window (old.example.com) but not other profile's logs. + s.Equal(8, len(logs), "download should return all 8 logs for profile") foundOld := false for _, l := range logs { if l.DNSRequest.Domain == "old.example.com" { diff --git a/app/src/App.tsx b/app/src/App.tsx index 05b3ee80..c6af3e7c 100644 --- a/app/src/App.tsx +++ b/app/src/App.tsx @@ -362,7 +362,9 @@ async function profilesOnlyLoader() { // Unified base layout for public/protected wrappers function BaseLayout({ children, mode }: { children: React.ReactNode, mode: 'public' | 'app' }) { - const baseClasses = 'relative flex flex-col min-h-screen overflow-x-hidden bg-[var(--shadcn-ui-app-background)]'; + // overflow-x-clip, not -hidden: `hidden` computes overflow-y:auto and turns the + // wrapper into a scroll container nested inside the viewport scroller. + const baseClasses = 'relative flex flex-col min-h-screen overflow-x-clip bg-[var(--shadcn-ui-app-background)]'; if (mode === 'public') { return (
diff --git a/app/src/__tests__/e2e/functional/custom-rules-empty-groups.spec.ts b/app/src/__tests__/e2e/functional/custom-rules-empty-groups.spec.ts new file mode 100644 index 00000000..875bef2b --- /dev/null +++ b/app/src/__tests__/e2e/functional/custom-rules-empty-groups.spec.ts @@ -0,0 +1,73 @@ +import { test, expect, type Route } from '@playwright/test'; +import { registerMocks } from '../../mocks/registerMocks'; + +// Issue #196: group folders live in the profile's group registry, independently of +// the rules that point at them. The list must keep rendering those folders when the +// rule list is empty — including right after the last rule is deleted — instead of +// collapsing to the "no rules yet" empty state. +const PROFILE_ENDPOINT = /\/api\/v1\/profiles\/prof1(\/?|\?.*)$/i; +const RULE_DELETE_ENDPOINT = /\/api\/v1\/profiles\/prof1\/custom_rules\/r1(\/?|\?.*)$/i; + +const groups = { block: [{ name: 'Work', comment: '' }, { name: 'Ads', comment: '' }] }; + +const withoutRules = { + id: 'prof1', profile_id: 'prof1', name: 'Default', + settings: { custom_rule_groups: groups, custom_rules: [] }, +}; + +const withOneRule = { + ...withoutRules, + settings: { + ...withoutRules.settings, + custom_rules: [{ id: 'r1', action: 'block', value: 'work.example.com', group: 'Work', order: 0 }], + }, +}; + +test.describe('@functional custom rules keep empty groups visible', () => { + // eslint-disable-next-line no-empty-pattern + test.beforeEach(({}, testInfo) => { + test.skip(!/chromium-desktop/i.test(testInfo.project.name), 'group folders are exercised on desktop'); + }); + + test('renders registry groups when the profile has no custom rules', async ({ page }) => { + await registerMocks(page, { authenticated: true, customProfiles: [withoutRules] }); + await page.goto('/custom-rules'); + + await expect(page.getByRole('button', { name: 'Drag to reorder group Work' })).toBeVisible(); + await expect(page.getByRole('button', { name: 'Drag to reorder group Ads' })).toBeVisible(); + await expect(page.getByText('There are no denied domains yet.')).toHaveCount(0); + }); + + test('deleting the last rule keeps its group folder on screen', async ({ page }) => { + await registerMocks(page, { authenticated: true, customProfiles: [withOneRule] }); + + // Registered after registerMocks so these take precedence over the catch-all. + // The single-profile GET flips to the rule-less payload once the DELETE has landed, + // mirroring the server state the page refetches after a delete. + let deleted = false; + await page.route(RULE_DELETE_ENDPOINT, (r: Route) => { + if (r.request().method() !== 'DELETE') return r.fallback(); + deleted = true; + return r.fulfill({ status: 200, contentType: 'application/json', body: '' }); + }); + await page.route(PROFILE_ENDPOINT, (r: Route) => { + if (r.request().method() !== 'GET') return r.fallback(); + return r.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify(deleted ? withoutRules : withOneRule), + }); + }); + + await page.goto('/custom-rules'); + await expect(page.getByText('work.example.com')).toBeVisible(); + + await page.getByRole('button', { name: 'Delete rule' }).click(); + + await expect.poll(() => deleted).toBe(true); + await expect(page.getByText('work.example.com')).toHaveCount(0); + await expect(page.getByRole('button', { name: 'Drag to reorder group Work' })).toBeVisible(); + await expect(page.getByRole('button', { name: 'Drag to reorder group Ads' })).toBeVisible(); + await expect(page.getByText('There are no denied domains yet.')).toHaveCount(0); + }); +}); diff --git a/app/src/__tests__/e2e/functional/custom-rules-paste-target.spec.ts b/app/src/__tests__/e2e/functional/custom-rules-paste-target.spec.ts new file mode 100644 index 00000000..f8deedc9 --- /dev/null +++ b/app/src/__tests__/e2e/functional/custom-rules-paste-target.spec.ts @@ -0,0 +1,77 @@ +import { test, expect } from '@playwright/test'; +import { registerMocks } from '../../mocks/registerMocks'; + +// The browser only offers Copy/Paste in its context or long-press menu when the +// pointer lands on the itself. The empty add-rule box must therefore keep +// its input stretched across the visible field rather than collapsing to the +// width of the (empty) typed text behind the placeholder. +test.describe('@functional custom rules add box exposes a pasteable input', () => { + test('the empty add-rule field is hit by the input, not its wrapper', async ({ page }) => { + await registerMocks(page, { + authenticated: true, + customProfiles: [{ id: 'prof1', profile_id: 'prof1', name: 'Default', settings: { custom_rules: [] } }], + }); + await page.goto('/custom-rules'); + + const input = page.locator('input#react-select-rule-composer-denylist-input'); + await expect(input).toBeAttached(); + + const control = page.locator('.rule-composer__control').first(); + const placeholder = page.locator('.rule-composer__placeholder').first(); + await expect(placeholder).toBeVisible(); + // Centre the field so neither the sticky header nor the mobile bottom nav overlaps it. + await control.evaluate((el) => el.scrollIntoView({ block: 'center' })); + + const box = await placeholder.boundingBox(); + const controlBox = await control.boundingBox(); + if (!box || !controlBox) throw new Error('add-rule field did not render'); + + // Right-click / long-press where the placeholder text is drawn. + const hit = await page.evaluate( + ([x, y]) => { const el = document.elementFromPoint(x, y); return el ? el.tagName + '.' + el.className : null; }, + [box.x + box.width / 2, box.y + box.height / 2], + ); + expect(hit).toMatch(/^INPUT\b/); + + // And the input spans the free space of the field, not a few pixels. + const inputBox = await input.boundingBox(); + if (!inputBox) throw new Error('input has no box'); + expect(inputBox.width).toBeGreaterThan(controlBox.width * 0.5); + }); + + test('a real right-click on the empty field reaches the input (desktop)', async ({ page }) => { + test.skip(!/chromium-desktop/i.test(test.info().project.name), 'context menus are a desktop pointer path'); + + await registerMocks(page, { + authenticated: true, + customProfiles: [{ id: 'prof1', profile_id: 'prof1', name: 'Default', settings: { custom_rules: [] } }], + }); + await page.goto('/custom-rules'); + + const control = page.locator('.rule-composer__control').first(); + const placeholder = page.locator('.rule-composer__placeholder').first(); + await expect(placeholder).toBeVisible(); + await control.evaluate((el) => el.scrollIntoView({ block: 'center' })); + + // Record what the contextmenu event lands on and whether anything cancels it. + await page.evaluate(() => { + const w = window as Window & { __ctx?: { tag: string; id: string; prevented: boolean } }; + document.addEventListener( + 'contextmenu', + (e) => { + const t = e.target as HTMLElement; + w.__ctx = { tag: t.tagName, id: t.id, prevented: e.defaultPrevented }; + }, + { capture: false }, + ); + }); + + const box = await placeholder.boundingBox(); + if (!box) throw new Error('add-rule field did not render'); + await page.mouse.click(box.x + box.width / 2, box.y + box.height / 2, { button: 'right' }); + + const ctx = await page.evaluate(() => (window as Window & { __ctx?: unknown }).__ctx); + expect(ctx).toEqual({ tag: 'INPUT', id: 'react-select-rule-composer-denylist-input', prevented: false }); + await expect(page.locator('input#react-select-rule-composer-denylist-input')).toBeFocused(); + }); +}); diff --git a/app/src/__tests__/e2e/layout/faq-expand-scroll.spec.ts b/app/src/__tests__/e2e/layout/faq-expand-scroll.spec.ts new file mode 100644 index 00000000..fb7c815d --- /dev/null +++ b/app/src/__tests__/e2e/layout/faq-expand-scroll.spec.ts @@ -0,0 +1,89 @@ +import { test, expect, type Page } from '@playwright/test'; +import { registerMocks } from '../../mocks/registerMocks'; + +// Expanding an FAQ answer while scrolled must keep the viewport where it is. +// Mobile projects only: the report is a phone-viewport regression. + +const MOBILE_PROJECTS = ['chromium-mobile-dark', 'iphone15pro-dark']; +const LAST_QUESTION = 'Do you support 2FA?'; +// Height transition is 300ms; give layout time to settle before measuring. +const SETTLE_MS = 600; + +async function openFaqScrolledToLastQuestion(page: Page) { + await registerMocks(page); + await page.goto('/faq'); + await page.waitForLoadState('networkidle'); + const lastQuestion = page.getByRole('button', { name: LAST_QUESTION }); + await lastQuestion.scrollIntoViewIfNeeded(); + await page.waitForTimeout(SETTLE_MS); + const scrollYBefore = await page.evaluate(() => window.scrollY); + expect(scrollYBefore).toBeGreaterThan(500); + return { lastQuestion, scrollYBefore }; +} + +test.describe('FAQ expand keeps scroll position', () => { + // eslint-disable-next-line no-empty-pattern + test.beforeEach(async ({}, testInfo) => { + test.skip(!MOBILE_PROJECTS.includes(testInfo.project.name), 'mobile projects only'); + }); + + test('expanding the last answer does not move the viewport', async ({ page }) => { + const { lastQuestion, scrollYBefore } = await openFaqScrolledToLastQuestion(page); + await lastQuestion.click(); + await page.waitForTimeout(SETTLE_MS); + const scrollYAfter = await page.evaluate(() => window.scrollY); + expect(Math.abs(scrollYAfter - scrollYBefore)).toBeLessThan(8); + await expect(page.getByText('Two-Factor Authentication adds an additional layer')).toBeInViewport(); + }); + + test('expand all from a scrolled position does not jump to the top', async ({ page }) => { + const { scrollYBefore } = await openFaqScrolledToLastQuestion(page); + // The control sits at the top of the page; drive it without scrolling there. + await page.getByRole('button', { name: 'Expand All' }).dispatchEvent('click'); + await page.waitForTimeout(SETTLE_MS); + const scrollYAfter = await page.evaluate(() => window.scrollY); + expect(scrollYAfter).toBeGreaterThan(0); + // Content above grew, so the offset may rise; it must never fall back to the top. + expect(scrollYAfter).toBeGreaterThanOrEqual(scrollYBefore - 8); + }); + + test('no scrollable space below the footer links', async ({ page }) => { + await registerMocks(page); + await page.goto('/faq'); + await page.waitForLoadState('networkidle'); + // Collapsed answers must not push the document past the footer: absolutely + // positioned descendants need the clipped wrapper as their containing block. + const { scrollHeight, footerBottom } = await page.evaluate(() => { + const link = document.querySelector('a[title="Go to Terms of Service page"]') as HTMLElement; + const footer = link.closest('div')!.parentElement as HTMLElement; + return { + scrollHeight: document.documentElement.scrollHeight, + footerBottom: footer.getBoundingClientRect().bottom + window.scrollY, + }; + }); + // 32px is the page wrapper's bottom padding. + expect(scrollHeight - footerBottom).toBeLessThanOrEqual(40); + }); + + test('the document is the only vertical scroller', async ({ page }) => { + await registerMocks(page); + await page.goto('/faq'); + await page.waitForLoadState('networkidle'); + // A wrapper that clips or scrolls vertically AND holds more content than it + // shows is a nested scroller; those are what confuse mobile engines. + const nested = await page.evaluate(() => { + const found: string[] = []; + for (const el of document.querySelectorAll('body *')) { + const overflowY = getComputedStyle(el).overflowY; + if (overflowY === 'visible' || overflowY === 'clip') continue; + // Collapsed answers (height 0) and sr-only spans clip on purpose. + if (el.clientHeight <= 1) continue; + if (el.scrollHeight > el.clientHeight + 1) { + found.push(`${el.tagName.toLowerCase()}.${el.className.toString().slice(0, 60)} ${el.clientHeight}/${el.scrollHeight}`); + } + } + return found; + }); + expect(nested).toEqual([]); + }); +}); diff --git a/app/src/__tests__/unit/QueryLogCard.test.tsx b/app/src/__tests__/unit/QueryLogCard.test.tsx index 47a648b2..f02d5faf 100644 --- a/app/src/__tests__/unit/QueryLogCard.test.tsx +++ b/app/src/__tests__/unit/QueryLogCard.test.tsx @@ -574,3 +574,39 @@ describe('QueryLogCard quick rule button', () => { expect(onQuickRule).not.toHaveBeenCalled(); }); }); + +// specRef: query-log-outcomes-behaviour.md #C4 +describe('QueryLogCard unavailable status (settings store unreachable)', () => { + beforeEach(() => { + (window as unknown as { innerWidth: number }).innerWidth = 1280; + stubDesktopMatchMedia(true); + }); + + const unavailableLog: ModelQueryLog = { + profile_id: 'p5', + timestamp: new Date().toISOString(), + status: 'unavailable', + outcome: 'filter_unavailable', + protocol: 'dns', + device_id: 'desktop-device', + client_ip: '10.0.0.5', + dns_request: { domain: 'unavailable.example.com.', query_type: 'A', response_code: 'SERVFAIL' } + }; + + test('never shows the Blocked pill and reports No answer', () => { + render(); + const indicator = screen.getByTestId('querylog-status-indicator'); + expect(indicator).not.toHaveAttribute('data-state', 'blocked'); + expect(indicator).toHaveTextContent(/no answer/i); + }); + + test('quick rule keeps the processed affordance and defaults to denylist', () => { + const onQuickRule = vi.fn(); + render(); + const button = screen.getByTestId('logs-quick-rule-button'); + expect(button.className).toContain('slate-800'); + expect(button.className).not.toContain('rdns-600'); + fireEvent.click(button); + expect(onQuickRule).toHaveBeenCalledWith('unavailable.example.com', 'denylist'); + }); +}); diff --git a/app/src/__tests__/unit/lib/formatOutcome.test.ts b/app/src/__tests__/unit/lib/formatOutcome.test.ts index 88a50d80..c0347f47 100644 --- a/app/src/__tests__/unit/lib/formatOutcome.test.ts +++ b/app/src/__tests__/unit/lib/formatOutcome.test.ts @@ -21,22 +21,21 @@ describe('formatOutcome', () => { expect(formatOutcome('refused')).toBe('Refused'); }); - it('falls back to a response-code derived label for legacy entries', () => { - // tableRef: query-log-outcomes-behaviour O10 - expect(formatOutcome(undefined, 'NOERROR')).toBe('Resolved'); - expect(formatOutcome('', 'NXDOMAIN')).toBe('Domain not found'); - expect(formatOutcome(undefined, 'SERVFAIL')).toBe('Upstream failure'); - expect(formatOutcome(undefined, 'REFUSED')).toBe('Refused'); - expect(formatOutcome(undefined, undefined)).toBe('Unknown'); - expect(formatOutcome('', '')).toBe('Unknown'); + it('labels a filtering-unavailable SERVFAIL distinctly from upstream failures', () => { + // tableRef: query-log-outcomes-behaviour O11 — the proxy synthesized the + // SERVFAIL itself because a filter stage could not read the settings store. + expect(formatOutcome('filter_unavailable')).toBe('Filtering unavailable'); }); - it('shows an unmapped response code verbatim instead of Unknown', () => { - // tableRef: query-log-outcomes-behaviour OE4 — rare rcodes (FORMERR, - // NOTIMP, ...) surface as-is; "Unknown" is reserved for entries with - // neither outcome nor response code. + it('shows the response code verbatim when the outcome is absent, Unknown when both are', () => { + // tableRef: query-log-outcomes-behaviour O10, OE4 — an absent outcome is + // the proxy's defensive case (e.g. a FORMERR/NOTIMP rcode outside the + // table); the rcode is shown as-is, never mapped to an outcome label. expect(formatOutcome(undefined, 'FORMERR')).toBe('FORMERR'); expect(formatOutcome('', 'NOTIMP')).toBe('NOTIMP'); + expect(formatOutcome(undefined, 'SERVFAIL')).toBe('SERVFAIL'); + expect(formatOutcome(undefined, undefined)).toBe('Unknown'); + expect(formatOutcome('', '')).toBe('Unknown'); }); it('shows an unknown token verbatim rather than hiding it', () => { @@ -83,36 +82,31 @@ describe('outcomePairs', () => { ]); }); - it('falls back per member for legacy entries without outcome', () => { - // tableRef: query-log-outcomes-behaviour C1, O10 + it('renders filter_unavailable as a failure-class chip', () => { + // tableRef: query-log-outcomes-behaviour C1, O11 + expect(outcomePairs([member('A', 'filter_unavailable')])).toEqual([ + { queryType: 'A', label: 'Filtering unavailable', failure: true }, + ]); + }); + + it('a member without an outcome shows its rcode verbatim and is not failure-tinted', () => { + // tableRef: query-log-outcomes-behaviour C1, O10, OE4 const r = outcomePairs([ - member('A', undefined, 'NOERROR'), + member('A', undefined, 'FORMERR'), member('AAAA', 'timeout'), ]); expect(r).toEqual([ - { queryType: 'A', label: 'Resolved', failure: false }, + { queryType: 'A', label: 'FORMERR', failure: false }, { queryType: 'AAAA', label: 'Upstream timeout', failure: true }, ]); }); - - it('legacy blocked entries read the status, not the synthesized NOERROR rcode', () => { - // tableRef: query-log-outcomes-behaviour O10 — a blocked response is a - // synthesized NOERROR (0.0.0.0/::), so the rcode fallback alone would - // wrongly render "Resolved" under a red Blocked pill. - const legacyBlocked: ModelQueryLog = { - status: 'blocked', - dns_request: { query_type: 'A', response_code: 'NOERROR' }, - }; - expect(outcomePairs([legacyBlocked])).toEqual([ - { queryType: 'A', label: 'Blocked', failure: true }, - ]); - }); }); describe('hasUnansweredMember', () => { it('flags each unanswered outcome token', () => { // tableRef: query-log-outcomes-behaviour C3 — collapsed-card chip trigger set - for (const outcome of ['servfail_upstream', 'timeout', 'network_error', 'refused']) { + // (O11 filter_unavailable is a synthesized SERVFAIL — unanswered too) + for (const outcome of ['servfail_upstream', 'timeout', 'network_error', 'refused', 'filter_unavailable']) { expect(hasUnansweredMember([member('A', outcome)])).toBe(true); } }); @@ -134,8 +128,6 @@ describe('hasUnansweredMember', () => { it('does not flag blocked entries — the Blocked pill owns those', () => { // tableRef: query-log-outcomes-behaviour C3 expect(hasUnansweredMember([{ ...member('A', 'blocked'), status: 'blocked' }])).toBe(false); - // legacy blocked: no outcome, synthesized NOERROR - expect(hasUnansweredMember([{ ...member('A', undefined, 'NOERROR'), status: 'blocked' }])).toBe(false); }); it('flags a mixed group when any member went unanswered', () => { @@ -145,24 +137,12 @@ describe('hasUnansweredMember', () => { expect(hasUnansweredMember([member('A', 'resolved'), member('AAAA', 'nodata')])).toBe(false); }); - it('falls back to the response code for legacy entries', () => { - // tableRef: query-log-outcomes-behaviour C3, O10 — legacy SERVFAIL/REFUSED - // entries went unanswered too; NOERROR/NXDOMAIN did not. - expect(hasUnansweredMember([member('A', undefined, 'SERVFAIL')])).toBe(true); - expect(hasUnansweredMember([member('A', undefined, 'REFUSED')])).toBe(true); - expect(hasUnansweredMember([member('A', undefined, 'NOERROR')])).toBe(false); - expect(hasUnansweredMember([member('A', undefined, 'NXDOMAIN')])).toBe(false); + it('never infers "No answer" from the response code alone', () => { + // tableRef: query-log-outcomes-behaviour C3, O10 — an absent outcome is + // the defensive case, not a legacy row; the rcode is not a trigger. + expect(hasUnansweredMember([member('A', undefined, 'SERVFAIL')])).toBe(false); + expect(hasUnansweredMember([member('A', undefined, 'REFUSED')])).toBe(false); expect(hasUnansweredMember([member('A')])).toBe(false); }); - - it('legacy DNSSEC-failed SERVFAIL entries defer to the DNSSEC label', () => { - // tableRef: query-log-outcomes-behaviour C3 — pre-outcome entries carry the - // dnssec_failed reason (same signal as O5); the red DNSSEC label covers them. - const legacyDnssec: ModelQueryLog = { - ...member('A', undefined, 'SERVFAIL'), - reasons: ['dnssec_failed'], - }; - expect(hasUnansweredMember([legacyDnssec])).toBe(false); - }); }); diff --git a/app/src/api/client/api.ts b/app/src/api/client/api.ts index 9f11dad3..a1734cd0 100644 --- a/app/src/api/client/api.ts +++ b/app/src/api/client/api.ts @@ -6546,7 +6546,7 @@ export const QueryLogsApiAxiosParamCreator = function (configuration?: Configura * @param {string} id Profile ID * @param {number} [page] specify page number * @param {number} [limit] specify logs limit by page - * @param {string} [status] specify status for query + * @param {ApiV1ProfilesIdLogsGetStatusEnum} [status] specify status for query * @param {string} [timespan] specify timespan for query * @param {string} [deviceId] specify device ID for filtering * @param {string} [search] substring (case-insensitive) match against stored domain; free-form (short inputs may scan more) @@ -6554,7 +6554,7 @@ export const QueryLogsApiAxiosParamCreator = function (configuration?: Configura * @param {*} [options] Override http request option. * @throws {RequiredError} */ - apiV1ProfilesIdLogsGet: async (id: string, page?: number, limit?: number, status?: string, timespan?: string, deviceId?: string, search?: string, sortBy?: ApiV1ProfilesIdLogsGetSortByEnum, options: RawAxiosRequestConfig = {}): Promise => { + apiV1ProfilesIdLogsGet: async (id: string, page?: number, limit?: number, status?: ApiV1ProfilesIdLogsGetStatusEnum, timespan?: string, deviceId?: string, search?: string, sortBy?: ApiV1ProfilesIdLogsGetSortByEnum, options: RawAxiosRequestConfig = {}): Promise => { // verify required parameter 'id' is not null or undefined assertParamExists('apiV1ProfilesIdLogsGet', 'id', id) const localVarPath = `/api/v1/profiles/{id}/logs` @@ -6664,7 +6664,7 @@ export const QueryLogsApiFp = function(configuration?: Configuration) { * @param {string} id Profile ID * @param {number} [page] specify page number * @param {number} [limit] specify logs limit by page - * @param {string} [status] specify status for query + * @param {ApiV1ProfilesIdLogsGetStatusEnum} [status] specify status for query * @param {string} [timespan] specify timespan for query * @param {string} [deviceId] specify device ID for filtering * @param {string} [search] substring (case-insensitive) match against stored domain; free-form (short inputs may scan more) @@ -6672,7 +6672,7 @@ export const QueryLogsApiFp = function(configuration?: Configuration) { * @param {*} [options] Override http request option. * @throws {RequiredError} */ - async apiV1ProfilesIdLogsGet(id: string, page?: number, limit?: number, status?: string, timespan?: string, deviceId?: string, search?: string, sortBy?: ApiV1ProfilesIdLogsGetSortByEnum, options?: RawAxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise>> { + async apiV1ProfilesIdLogsGet(id: string, page?: number, limit?: number, status?: ApiV1ProfilesIdLogsGetStatusEnum, timespan?: string, deviceId?: string, search?: string, sortBy?: ApiV1ProfilesIdLogsGetSortByEnum, options?: RawAxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise>> { const localVarAxiosArgs = await localVarAxiosParamCreator.apiV1ProfilesIdLogsGet(id, page, limit, status, timespan, deviceId, search, sortBy, options); const localVarOperationServerIndex = configuration?.serverIndex ?? 0; const localVarOperationServerBasePath = operationServerMap['QueryLogsApi.apiV1ProfilesIdLogsGet']?.[localVarOperationServerIndex]?.url; @@ -6724,7 +6724,7 @@ export const QueryLogsApiFactory = function (configuration?: Configuration, base * @param {string} id Profile ID * @param {number} [page] specify page number * @param {number} [limit] specify logs limit by page - * @param {string} [status] specify status for query + * @param {ApiV1ProfilesIdLogsGetStatusEnum} [status] specify status for query * @param {string} [timespan] specify timespan for query * @param {string} [deviceId] specify device ID for filtering * @param {string} [search] substring (case-insensitive) match against stored domain; free-form (short inputs may scan more) @@ -6732,7 +6732,7 @@ export const QueryLogsApiFactory = function (configuration?: Configuration, base * @param {*} [options] Override http request option. * @throws {RequiredError} */ - apiV1ProfilesIdLogsGet(id: string, page?: number, limit?: number, status?: string, timespan?: string, deviceId?: string, search?: string, sortBy?: ApiV1ProfilesIdLogsGetSortByEnum, options?: RawAxiosRequestConfig): AxiosPromise> { + apiV1ProfilesIdLogsGet(id: string, page?: number, limit?: number, status?: ApiV1ProfilesIdLogsGetStatusEnum, timespan?: string, deviceId?: string, search?: string, sortBy?: ApiV1ProfilesIdLogsGetSortByEnum, options?: RawAxiosRequestConfig): AxiosPromise> { return localVarFp.apiV1ProfilesIdLogsGet(id, page, limit, status, timespan, deviceId, search, sortBy, options).then((request) => request(axios, basePath)); }, }; @@ -6787,7 +6787,7 @@ export class QueryLogsApi extends BaseAPI { * @param {string} id Profile ID * @param {number} [page] specify page number * @param {number} [limit] specify logs limit by page - * @param {string} [status] specify status for query + * @param {ApiV1ProfilesIdLogsGetStatusEnum} [status] specify status for query * @param {string} [timespan] specify timespan for query * @param {string} [deviceId] specify device ID for filtering * @param {string} [search] substring (case-insensitive) match against stored domain; free-form (short inputs may scan more) @@ -6796,11 +6796,21 @@ export class QueryLogsApi extends BaseAPI { * @throws {RequiredError} * @memberof QueryLogsApi */ - public apiV1ProfilesIdLogsGet(id: string, page?: number, limit?: number, status?: string, timespan?: string, deviceId?: string, search?: string, sortBy?: ApiV1ProfilesIdLogsGetSortByEnum, options?: RawAxiosRequestConfig) { + public apiV1ProfilesIdLogsGet(id: string, page?: number, limit?: number, status?: ApiV1ProfilesIdLogsGetStatusEnum, timespan?: string, deviceId?: string, search?: string, sortBy?: ApiV1ProfilesIdLogsGetSortByEnum, options?: RawAxiosRequestConfig) { return QueryLogsApiFp(this.configuration).apiV1ProfilesIdLogsGet(id, page, limit, status, timespan, deviceId, search, sortBy, options).then((request) => request(this.axios, this.basePath)); } } +/** + * @export + */ +export const ApiV1ProfilesIdLogsGetStatusEnum = { + All: 'all', + Blocked: 'blocked', + Processed: 'processed', + Unanswered: 'unanswered' +} as const; +export type ApiV1ProfilesIdLogsGetStatusEnum = typeof ApiV1ProfilesIdLogsGetStatusEnum[keyof typeof ApiV1ProfilesIdLogsGetStatusEnum]; /** * @export */ diff --git a/app/src/components/AccountSubscription.tsx b/app/src/components/AccountSubscription.tsx index abc3844b..8c168a1f 100644 --- a/app/src/components/AccountSubscription.tsx +++ b/app/src/components/AccountSubscription.tsx @@ -77,7 +77,7 @@ export default function AccountSubscription() { const isInactive = sub.status === "inactive"; const isPendingDelete = sub.status === "pending_delete"; // signup-reset retired const isCutOff = isInactive || isPendingDelete; - const hasAlerts = isLimited || isCutOff || sub.outage || !!error; + const hasAlerts = sub.outage || !!error; const statusBadge = syncing ? @@ -111,21 +111,6 @@ export default function AccountSubscription() { {/* Alerts — rendered first, meant to be placed above the cards by parent */} {hasAlerts && (
- {isLimited && ( -
- -
-

- Limited Access Mode -

-

- Your modDNS account is in limited access mode. To regain full access add time to your{" "} - IVPN account. -

-
-
- )} - {sub.outage && (
diff --git a/app/src/hooks/useDnsConnectionStatus.ts b/app/src/hooks/useDnsConnectionStatus.ts index b2b9dadd..5d34a27f 100644 --- a/app/src/hooks/useDnsConnectionStatus.ts +++ b/app/src/hooks/useDnsConnectionStatus.ts @@ -35,9 +35,10 @@ export function useDnsConnectionStatus(pollMs: number = 5000, options?: { enable setError(''); const alphabet = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'; const nanoid = customAlphabet(alphabet, 12); - const randID = nanoid(); - const profileId = activeProfile?.profile_id || ''; - const subdomain = `${randID}-${profileId}`; + // The probe name is pure randomness. The profile ID is reported back by + // the proxy (EDNS0), never sent in the hostname, where it would be visible + // to the resolver chain and in the TLS SNI. + const subdomain = nanoid(); const dnsCheckDomain = import.meta.env.VITE_DNS_CHECK_DOMAIN || 'test.moddns.net'; const url = `https://${subdomain}.${dnsCheckDomain}/`; const response = await axios.get(url); @@ -71,7 +72,6 @@ export function useDnsConnectionStatus(pollMs: number = 5000, options?: { enable executeDnsCheck(); intervalRef.current = setInterval(() => executeDnsCheck(), pollMs); return () => { if (intervalRef.current) clearInterval(intervalRef.current); }; - // eslint-disable-next-line react-hooks/exhaustive-deps }, [activeProfile?.profile_id, enabled, pollMs]); const getCurrentProfileName = () => { diff --git a/app/src/lib/formatOutcome.ts b/app/src/lib/formatOutcome.ts index 285b1289..49c81a75 100644 --- a/app/src/lib/formatOutcome.ts +++ b/app/src/lib/formatOutcome.ts @@ -1,7 +1,7 @@ // formatOutcome — map the proxy-computed resolution-outcome token to a // human-readable label for the query-log UI. // -// Source of truth: docs/specs/query-log-outcomes-behaviour.md (rows O1-O10, +// Source of truth: docs/specs/query-log-outcomes-behaviour.md (rows O1-O11, // Queries-display row C1). If the mapping changes, update that spec and // formatOutcome.test.ts with matching // `tableRef: query-log-outcomes-behaviour ` annotations. @@ -23,38 +23,30 @@ const OUTCOME_LABELS: Record = { timeout: 'Upstream timeout', // O7 network_error: 'Upstream unreachable', // O8 refused: 'Refused', // O9 + filter_unavailable: 'Filtering unavailable', // O11 }; // Failure-class tokens get the red tint on pair chips. const FAILURE_OUTCOMES = new Set([ 'blocked', 'servfail_dnssec', 'servfail_upstream', 'timeout', 'network_error', 'refused', + 'filter_unavailable', ]); -// O10 legacy fallback: entries written before the outcome field existed only -// carry a response code. -const LEGACY_RCODE_LABELS: Record = { - NOERROR: 'Resolved', - NXDOMAIN: 'Domain not found', - SERVFAIL: 'Upstream failure', - REFUSED: 'Refused', -}; - /** - * @param outcome Raw token from `ModelQueryLog.outcome` (may be absent). - * @param responseCode Raw rcode string, used only as the legacy fallback. + * @param outcome Raw token from `ModelQueryLog.outcome`. Every stored row + * carries one (the field predates the longest retention); + * an empty value is the proxy's defensive O10 case. + * @param responseCode Raw rcode string, shown verbatim when there is no outcome + * (OE4: rcodes outside the outcome table, e.g. FORMERR). */ export function formatOutcome(outcome?: string, responseCode?: string): string { if (outcome) { // Unknown tokens (a newer proxy than this app) render verbatim rather - // than disappearing — mirrors the O10 forward-compat rule. + // than disappearing — the O10 forward-compat rule. return OUTCOME_LABELS[outcome] ?? outcome; } - if (responseCode) { - // Rare rcodes outside the map (FORMERR, NOTIMP, ...) surface verbatim — - // "Unknown" is reserved for entries with neither outcome nor rcode. - return LEGACY_RCODE_LABELS[responseCode] ?? responseCode; - } - return 'Unknown'; + // "Unknown" is reserved for entries with neither outcome nor rcode. + return responseCode || 'Unknown'; } export interface OutcomePair { @@ -66,52 +58,39 @@ export interface OutcomePair { /** * Distinct (query type, outcome label) pairs for the always-rendered "Queries" * chip block (C1). Works for a single entry (pass `[log]`) and consolidated - * groups alike; exact duplicates collapse, member order is preserved, legacy - * members fall back per member via formatOutcome (O10). + * groups alike; exact duplicates collapse, member order is preserved. */ export function outcomePairs(members: ModelQueryLog[]): OutcomePair[] { const pairs: OutcomePair[] = []; const seen = new Set(); for (const m of members) { const queryType = m.dns_request?.query_type ?? ''; - // O10: legacy blocked entries have no outcome but a synthesized NOERROR - // rcode — the status is the truthful signal, never "Resolved". - const effectiveOutcome = !m.outcome && m.status === 'blocked' ? 'blocked' : m.outcome; - const label = formatOutcome(effectiveOutcome, m.dns_request?.response_code); + const label = formatOutcome(m.outcome, m.dns_request?.response_code); const key = `${queryType} ${label}`; if (seen.has(key)) continue; seen.add(key); - pairs.push({ queryType, label, failure: FAILURE_OUTCOMES.has(effectiveOutcome ?? '') }); + pairs.push({ queryType, label, failure: FAILURE_OUTCOMES.has(m.outcome ?? '') }); } return pairs; } -// Collapsed-card "Not answered" chip trigger set (C3). Deliberately narrower -// than FAILURE_OUTCOMES: `blocked` is owned by the red Blocked pill and -// `servfail_dnssec` by the red DNSSEC text label already on the collapsed row. +// Collapsed-card "No answer" label trigger set (C3), shared with the API's +// `status=unanswered` filter (C5). Deliberately narrower than FAILURE_OUTCOMES: +// `blocked` is owned by the red Blocked pill and `servfail_dnssec` by the red +// DNSSEC text label already on the collapsed row — both are verdicts, not +// failures to answer. const UNANSWERED_OUTCOMES = new Set([ - 'servfail_upstream', 'timeout', 'network_error', 'refused', + 'servfail_upstream', 'timeout', 'network_error', 'refused', 'filter_unavailable', ]); -// O10 legacy entries carry only an rcode; these two mean the query went -// unanswered. NOERROR/NXDOMAIN (and unmapped rcodes) do not trigger the chip. -const UNANSWERED_LEGACY_RCODES = new Set(['SERVFAIL', 'REFUSED']); - /** - * Should the collapsed row show the amber "Not answered" chip? True when ANY + * Should the collapsed row show the amber "No answer" label? True when ANY * member went unanswered (C3) — `outcome` is not part of the consolidation * signature, so a group can mix e.g. a resolved query with a timed-out retry * and the representative alone would hide the failure. */ export function hasUnansweredMember(members: ModelQueryLog[]): boolean { - return members.some((m) => { - if (m.status === 'blocked') return false; // O4/O10: Blocked pill owns it - if (m.outcome) return UNANSWERED_OUTCOMES.has(m.outcome); - // Legacy DNSSEC failures are SERVFAIL + dnssec_failed reason (the O5 - // signal) — the red DNSSEC label covers them, like modern servfail_dnssec. - if (m.reasons?.includes('dnssec_failed')) return false; - return UNANSWERED_LEGACY_RCODES.has(m.dns_request?.response_code ?? ''); - }); + return members.some((m) => UNANSWERED_OUTCOMES.has(m.outcome ?? '')); } export default formatOutcome; diff --git a/app/src/pages/custom_rules/CustomRulesCard.tsx b/app/src/pages/custom_rules/CustomRulesCard.tsx index a0ec4bfb..ab1eb7dd 100644 --- a/app/src/pages/custom_rules/CustomRulesCard.tsx +++ b/app/src/pages/custom_rules/CustomRulesCard.tsx @@ -838,19 +838,21 @@ export default function CustomRulesCard({ ? namedSections.find(s => s.name === activeGroupName) : undefined; - if (rules.length === 0) { - if (searchQuery.trim().length > 0) { - return ( - - - - ); - } + if (rules.length === 0 && searchQuery.trim().length > 0) { + return ( + + + + ); + } + // Groups are registry entries, not derived from rules, so an empty rule list still + // has folders to show; the empty state is only for a list with nothing at all. + if (rules.length === 0 && !hasNamedGroups) { return (
diff --git a/app/src/pages/custom_rules/RuleComposer.tsx b/app/src/pages/custom_rules/RuleComposer.tsx index 0543863d..7f30e450 100644 --- a/app/src/pages/custom_rules/RuleComposer.tsx +++ b/app/src/pages/custom_rules/RuleComposer.tsx @@ -146,6 +146,11 @@ const selectStyles: StylesConfig = { color: "var(--tailwind-colors-slate-100)", caretColor: "var(--tailwind-colors-rdns-400)", minWidth: "6rem", + // react-select sizes the inner to the mirrored typed text (0 min-content), + // so an empty field leaves a 2px-wide input under the placeholder. Browsers only + // offer Copy/Paste in the context or long-press menu when the pointer lands on + // the itself, so let it fill the wrapper instead. + gridTemplateColumns: "0 1fr", }), }; diff --git a/app/src/pages/legal/FAQ.tsx b/app/src/pages/legal/FAQ.tsx index 4f214b1e..cbb70163 100644 --- a/app/src/pages/legal/FAQ.tsx +++ b/app/src/pages/legal/FAQ.tsx @@ -9,6 +9,7 @@ import modDNSLogoLightTheme from '@/assets/logos/modDNS-light-theme.svg'; import { useTheme } from "@/components/theme-provider"; import AuthFooter from "@/components/auth/AuthFooter"; import { parseDnsServerLocations, firstAddress } from "@/lib/dnsServerLocations"; +import { LINKS } from "@/pages/landing/links"; interface FAQItemProps { question: string; @@ -68,7 +69,7 @@ function FAQItem({ question, answer, globalToggleSignal, globalToggleState }: FA
{typeof answer === 'string' ? ( @@ -110,7 +111,7 @@ function FAQSection({ title, children, globalToggleSignal, globalToggleState }: ); } -const FAQ_LAST_UPDATED = 'September 3, 2026'; +const FAQ_LAST_UPDATED = 'September 9, 2026'; const CODE_CLASS = "text-[var(--shadcn-ui-app-foreground)] px-2 py-0.5 rounded text-sm font-mono border border-[var(--shadcn-ui-app-border)]"; const TABLE_CELL_CLASS = "border border-[var(--shadcn-ui-app-border)] px-3 py-2 text-left align-top"; @@ -397,6 +398,27 @@ export default function FAQ(): JSX.Element {
); + const howToGetModDNS = ( +
+

modDNS is included in the IVPN Plus and IVPN Pro Suite plans. There is no standalone modDNS subscription, and it is not part of the IVPN Standard plan. See ivpn.net/pricing for current plans.

+

Once you have an eligible IVPN plan, start modDNS from your IVPN account area. IVPN sends you to a one-time signup link where you create your modDNS login with an email and password or a passkey. Your modDNS access then follows your IVPN subscription automatically.

+
+ ); + + const qnameMinimisation = ( +
+

Yes. When a resolver looks up a name, it walks the DNS hierarchy from the root servers down. Without QNAME minimisation it repeats the full name (for example mail.example.com) to every server on that path, so the root and top-level-domain servers learn which hosts you visit. With QNAME minimisation (RFC 9156) each server is asked only for the part it is responsible for.

+

modDNS applies this on every resolver location and for every profile. It is always on and there is no setting that disables it.

+
+ ); + + const ednsClientSubnet = ( +
+

No. EDNS Client Subnet (ECS, RFC 7871) is a mechanism by which a resolver attaches part of your IP address to the queries it sends to authoritative DNS servers, mainly so that content delivery networks can pick a server near you. It also reveals your approximate network location to every authoritative server involved in a lookup.

+

modDNS does not use ECS. Your address is never attached to upstream queries, and if your device adds an ECS option to its own queries it is discarded before the lookup leaves our resolver. Authoritative servers only see the address of the modDNS server location that handled your query. There is no setting to turn ECS on.

+
+ ); + const whatIsDNSSEC = (

DNSSEC stands for Domain Name System Security Extensions. It's a security protocol that adds digital signatures to DNS records to ensure their authenticity and integrity. This helps prevent DNS spoofing attacks, where malicious actors could redirect users to fake websites.

@@ -507,6 +529,17 @@ export default function FAQ(): JSX.Element { question="What is modDNS?" answer="modDNS is a privacy-focused DNS service that helps protect privacy and improve security by blocking ads, trackers, and malicious domains. It supports modern DNS protocols including DNS-over-HTTPS (DoH), DNS-over-TLS (DoT), and DNS-over-QUIC (DoQ)." /> + + + + + + @@ -800,7 +837,7 @@ export default function FAQ(): JSX.Element { ); return ( -
+
{hasHistory && ( diff --git a/app/src/pages/legal/PrivacyPolicy.tsx b/app/src/pages/legal/PrivacyPolicy.tsx index 018c70c5..02819ff6 100644 --- a/app/src/pages/legal/PrivacyPolicy.tsx +++ b/app/src/pages/legal/PrivacyPolicy.tsx @@ -46,7 +46,7 @@ export default function PrivacyPolicy() {

- Last updated: Mar 23, 2026 + Last updated: Sep 17, 2026

@@ -108,9 +108,9 @@ export default function PrivacyPolicy() {

  • DNS queries (e.g., which websites you visit)
    • -
  • Timestamps of DNS resolutions
  • Your IP addresses
  • Device information or identifiers
    • +
  • Timestamps of DNS resolutions with user/profile attribution

For more information on what is logged when you optionally enable "Query Logs", see the next section. @@ -125,13 +125,14 @@ export default function PrivacyPolicy() {

a) Default setting (Query Logs Disabled)

- When query logging is turned off, all queries are processed entirely in memory and are never written to disk. We log no information about your usage of the DNS resolver, with one exception: + When query logging is turned off, all queries are processed entirely in memory and are never written to disk.

- We store a total count of DNS requests processed by your profile. This is a simple counter and contains no specific details about your activity. Example of data stored: + The only DNS activity data we keep is anonymous service-wide statistics: the number of queries, blocked queries and DNSSEC-validated queries handled by each server location, added up across all users with hourly timestamps. These counters contain no profile, device or client reference. Example of data stored:

- 
{`"profile_id": "ju8eamnqfn"
-"queries": { "total": 244 }`}
+ 
{`"timestamp": "2026-09-16T10:00:00Z"
+"pop": "ams1"
+"queries": { "total": 18342, "blocked": 2917, "dnssec": 520 }`}

b) With Query Logs Enabled

diff --git a/app/src/pages/logs/Filters.tsx b/app/src/pages/logs/Filters.tsx index 69c51ad2..c6f368d2 100644 --- a/app/src/pages/logs/Filters.tsx +++ b/app/src/pages/logs/Filters.tsx @@ -299,6 +299,7 @@ const Filters = ({ All queries Blocked Processed + No answer diff --git a/app/src/pages/logs/QueryLogCard.tsx b/app/src/pages/logs/QueryLogCard.tsx index 97fc5b02..b880d626 100644 --- a/app/src/pages/logs/QueryLogCard.tsx +++ b/app/src/pages/logs/QueryLogCard.tsx @@ -50,6 +50,10 @@ const QueryLogCard = ({ log, group, isLast, lastLogRef, onQuickRule, quickRuleRe const quickRuleAvailable = Boolean(normalizedDomain); const isBlocked = log.status === "blocked"; const isProcessed = log.status === "processed"; + // Answered SERVFAIL by the proxy because its settings store was unreachable + // (spec: query-log-outcomes-behaviour.md O11/C4). Nothing was blocked, so the + // row keeps the neutral processed affordances. + const isUnavailable = log.status === "unavailable"; // Collapsed status indicator (spec: query-log-outcomes-behaviour.md C3): the // slot shows the red "Blocked" pill OR the amber "No answer" text micro-label // (any member unanswered — outcome is not in the consolidation signature) OR @@ -69,7 +73,7 @@ const QueryLogCard = ({ log, group, isLast, lastLogRef, onQuickRule, quickRuleRe }; const quickRuleButtonClasses = isBlocked ? "bg-[var(--tailwind-colors-rdns-600)] text-[var(--tailwind-colors-slate-900)] hover:!bg-[var(--tailwind-colors-slate-900)] hover:!text-[var(--tailwind-colors-rdns-600)]" - : isProcessed + : isProcessed || isUnavailable ? "bg-[var(--tailwind-colors-slate-800)] text-[var(--tailwind-colors-slate-100)] hover:!bg-[var(--tailwind-colors-red-600)] hover:!text-[var(--tailwind-colors-slate-50)]" : "bg-[var(--tailwind-colors-rdns-600)] text-[var(--tailwind-colors-slate-900)] hover:!bg-[var(--tailwind-colors-slate-900)] hover:!text-[var(--tailwind-colors-rdns-600)]"; // Quick-rule is the ONLY control excluded from the whole-card expand overlay; its wrapper diff --git a/app/src/pages/settings/ProfileManagementSection.tsx b/app/src/pages/settings/ProfileManagementSection.tsx index 541015b8..484d4a41 100644 --- a/app/src/pages/settings/ProfileManagementSection.tsx +++ b/app/src/pages/settings/ProfileManagementSection.tsx @@ -4,7 +4,6 @@ import type { ModelProfile } from "@/api/client/api"; import { ModelProfileUpdateOperationEnum, ModelProfileUpdatePathEnum } from "@/api/client/api"; import { useAppStore } from "@/store/general"; import { useSubscriptionGuard } from "@/hooks/useSubscriptionGuard"; -import LimitedAccessBanner from "@/components/LimitedAccessBanner"; import { toast } from "sonner"; import DeleteProfileDialog from "@/pages/settings/DeleteProfileDialog"; import QueryLogsSection from "./QueryLogsSection"; @@ -385,7 +384,6 @@ export default function ProfileManagementSection({ profiles }: ProfileManagement return ( <> -

{/* BLOCKLISTS + CUSTOM RULES — mutations blocked in LA */}
diff --git a/app/src/pages/setup/guides/Routers.tsx b/app/src/pages/setup/guides/Routers.tsx index 23eba321..9c787316 100644 --- a/app/src/pages/setup/guides/Routers.tsx +++ b/app/src/pages/setup/guides/Routers.tsx @@ -278,7 +278,7 @@ const StampsTab = ({ deps }: { deps: RoutersGuideDeps }) => { const buildRouterTabs = (deps: RoutersGuideDeps): RouterTabDef[] => [ { key: 'mikrotik', - label: 'Mikrotik Router OS', + label: 'Mikrotik RouterOS', content: (
Access the device’s command-line interface, and enter the following commands:} /> diff --git a/blocklists/cache/cache.go b/blocklists/cache/cache.go index 6e3ff01b..a4bdd814 100644 --- a/blocklists/cache/cache.go +++ b/blocklists/cache/cache.go @@ -13,11 +13,18 @@ const CacheTypeRedis = "redis" // Cache is an interface for caching functionalities type Cache interface { CreateOrUpdateBlocklist(ctx context.Context, blocklistId string, data []byte) error + // CreateOrUpdateBlocklistExceptions publishes the list's companion + // exception set with the same stage-and-swap flow as the main set; empty + // data removes the key (the list has no exceptions). + CreateOrUpdateBlocklistExceptions(ctx context.Context, blocklistId string, data []byte) error DeleteBlocklist(ctx context.Context, blocklistId string) error // BlocklistExists reports whether the live set for blocklistId is present // in the cache (used by the freshness check: metadata alone cannot prove // the published data survived, e.g. a cache flush). BlocklistExists(ctx context.Context, blocklistId string) (bool, error) + // BlocklistExceptionsExist reports whether the live exception set for + // blocklistId is present (freshness backstop, alongside BlocklistExists). + BlocklistExceptionsExist(ctx context.Context, blocklistId string) (bool, error) // Ping reports whether the cache backend is reachable (used for readiness). Ping(ctx context.Context) error // Locker returns a distributed locker sharing the cache's backend, so diff --git a/blocklists/cache/redis.go b/blocklists/cache/redis.go index 777f3eae..328f0dc9 100644 --- a/blocklists/cache/redis.go +++ b/blocklists/cache/redis.go @@ -1,6 +1,7 @@ package cache import ( + "bytes" "context" "fmt" "strings" @@ -69,8 +70,29 @@ func (c *RedisCache) Locker(prefix string) *dislock.Locker { // writers (peer instances racing on the same source) can therefore never // interleave on one staging set — and promoted with an atomic swap. func (c *RedisCache) CreateOrUpdateBlocklist(ctx context.Context, blocklistId string, data []byte) error { - blocklistName := fmt.Sprintf("blocklist:%s", blocklistId) - stagingName := fmt.Sprintf("%s:tmp:%s", blocklistName, uuid.NewString()) + return c.createOrUpdateSet(ctx, fmt.Sprintf("blocklist:%s", blocklistId), data) +} + +// CreateOrUpdateBlocklistExceptions publishes the list's companion exception +// set under blocklist:{id}:exceptions with the same stage-and-swap flow. +// Empty data removes the key: a missing exception set means "no exceptions" +// to the proxy, so absence is the correct representation, not an empty set +// (SADD of nothing cannot create one anyway). +func (c *RedisCache) CreateOrUpdateBlocklistExceptions(ctx context.Context, blocklistId string, data []byte) error { + key := exceptionsKey(blocklistId) + if len(bytes.TrimSpace(data)) == 0 { + if err := c.client.Unlink(ctx, key).Err(); err != nil { + return err + } + return nil + } + return c.createOrUpdateSet(ctx, key, data) +} + +// createOrUpdateSet stages data into a per-run key and atomically promotes it +// to liveName, replacing any existing set. +func (c *RedisCache) createOrUpdateSet(ctx context.Context, liveName string, data []byte) error { + stagingName := fmt.Sprintf("%s:tmp:%s", liveName, uuid.NewString()) // Step 1: Populate the staging set with new data. if err := c.populateStaging(ctx, stagingName, data); err != nil { @@ -79,18 +101,22 @@ func (c *RedisCache) CreateOrUpdateBlocklist(ctx context.Context, blocklistId st } // Step 2: Atomically swap the populated staging set into place. - if err := c.swapBlocklist(ctx, stagingName, blocklistName); err != nil { + if err := c.swapBlocklist(ctx, stagingName, liveName); err != nil { c.discardStaging(ctx, stagingName) return err } log.Debug(). Str("component", "cache"). - Str("blocklist_key", blocklistName). + Str("blocklist_key", liveName). Msg("Created/updated blocklist with atomic swap") return nil } +func exceptionsKey(blocklistId string) string { + return fmt.Sprintf("blocklist:%s:exceptions", blocklistId) +} + // populateStaging fills the staging set, flushing in bounded batches, and // bounds the key's lifetime with stagingTTL so an interrupted run self-cleans. func (c *RedisCache) populateStaging(ctx context.Context, stagingName string, data []byte) error { @@ -168,7 +194,16 @@ func (c *RedisCache) discardStaging(ctx context.Context, stagingName string) { // BlocklistExists reports whether the live blocklist set is present in Redis. func (c *RedisCache) BlocklistExists(ctx context.Context, blocklistId string) (bool, error) { - n, err := c.client.Exists(ctx, fmt.Sprintf("blocklist:%s", blocklistId)).Result() + return c.keyExists(ctx, fmt.Sprintf("blocklist:%s", blocklistId)) +} + +// BlocklistExceptionsExist reports whether the live exception set is present. +func (c *RedisCache) BlocklistExceptionsExist(ctx context.Context, blocklistId string) (bool, error) { + return c.keyExists(ctx, exceptionsKey(blocklistId)) +} + +func (c *RedisCache) keyExists(ctx context.Context, key string) (bool, error) { + n, err := c.client.Exists(ctx, key).Result() if err != nil { return false, err } @@ -180,10 +215,11 @@ func (c *RedisCache) Ping(ctx context.Context) error { return c.client.Ping(ctx).Err() } -// DeleteBlocklist removes a blocklist set from the cache +// DeleteBlocklist removes a blocklist set and its companion exception set +// from the cache func (c *RedisCache) DeleteBlocklist(ctx context.Context, blocklistId string) error { key := fmt.Sprintf("blocklist:%s", blocklistId) - if err := c.client.Del(ctx, key).Err(); err != nil { + if err := c.client.Del(ctx, key, exceptionsKey(blocklistId)).Err(); err != nil { return err } log.Debug().Str("component", "cache").Str("blocklist_key", key).Msg("Deleted blocklist from cache") diff --git a/blocklists/cache/redis_test.go b/blocklists/cache/redis_test.go index 751dfc80..5d378fb2 100644 --- a/blocklists/cache/redis_test.go +++ b/blocklists/cache/redis_test.go @@ -261,3 +261,60 @@ func TestCreateOrUpdateBlocklist_LiveKeyHasNoTTL(t *testing.T) { require.NoError(t, err) assert.Equal(t, time.Duration(-1), ttl, "live key must be persistent (TTL -1)") } + +// specRef: #F4 — the exception set is published to the companion key with the +// same stage-and-swap flow. +func TestCreateOrUpdateBlocklistExceptions_Publishes(t *testing.T) { + c, mr := newTestCache(t) + ctx := context.Background() + + err := c.CreateOrUpdateBlocklistExceptions(ctx, "adguard_dns_filter", []byte("cdn.example.com\nsbs.demdex.net")) + require.NoError(t, err) + + members, err := mr.SMembers("blocklist:adguard_dns_filter:exceptions") + require.NoError(t, err) + assert.ElementsMatch(t, []string{"cdn.example.com", "sbs.demdex.net"}, members) + // The live key must not expire (staging TTL cleared by the swap). + assert.Equal(t, time.Duration(0), mr.TTL("blocklist:adguard_dns_filter:exceptions")) +} + +// specRef: #F4 — empty data removes the key: a missing exception set means +// "no exceptions" to the proxy. +func TestCreateOrUpdateBlocklistExceptions_EmptyDeletesKey(t *testing.T) { + c, mr := newTestCache(t) + ctx := context.Background() + + require.NoError(t, c.CreateOrUpdateBlocklistExceptions(ctx, "bl1", []byte("old.example.com"))) + require.NoError(t, c.CreateOrUpdateBlocklistExceptions(ctx, "bl1", []byte(""))) + + assert.False(t, mr.Exists("blocklist:bl1:exceptions")) +} + +// specRef: #G11 — the freshness backstop can check the exception key's +// existence separately from the main set's. +func TestBlocklistExceptionsExist(t *testing.T) { + c, _ := newTestCache(t) + ctx := context.Background() + + exists, err := c.BlocklistExceptionsExist(ctx, "bl1") + require.NoError(t, err) + assert.False(t, exists) + + require.NoError(t, c.CreateOrUpdateBlocklistExceptions(ctx, "bl1", []byte("a.example.com"))) + exists, err = c.BlocklistExceptionsExist(ctx, "bl1") + require.NoError(t, err) + assert.True(t, exists) +} + +// specRef: #F4 — deleting a blocklist removes its exception set too. +func TestDeleteBlocklist_RemovesExceptions(t *testing.T) { + c, mr := newTestCache(t) + ctx := context.Background() + + require.NoError(t, c.CreateOrUpdateBlocklist(ctx, "bl1", []byte("blocked.example.com"))) + require.NoError(t, c.CreateOrUpdateBlocklistExceptions(ctx, "bl1", []byte("a.example.com"))) + require.NoError(t, c.DeleteBlocklist(ctx, "bl1")) + + assert.False(t, mr.Exists("blocklist:bl1")) + assert.False(t, mr.Exists("blocklist:bl1:exceptions")) +} diff --git a/blocklists/internal/extractor/adguard.go b/blocklists/internal/extractor/adguard.go index af155fca..6cfae5ca 100644 --- a/blocklists/internal/extractor/adguard.go +++ b/blocklists/internal/extractor/adguard.go @@ -20,6 +20,10 @@ const ( // Rule prefixes and special characters exceptionPrefix = "@@" modifierSeparator = "$" + badfilterModifier = "badfilter" + importantModifier = "important" + regexDelimiter = "/" + wildcard = "*" ) var ( @@ -35,9 +39,15 @@ func NewAdguardExtractor() *AdguardExtractor { return &AdguardExtractor{} } -// Convert transforms AdGuard format rules into a simple domain list -func (e *AdguardExtractor) Convert(blocklistBytes []byte) ([]byte, error) { +// Convert transforms AdGuard format rules into a simple domain list. Rules +// whose syntax the extractor does not understand are dropped rather than +// widened into unconditional blocks (fail open), counted per reason in the +// returned stats. +func (e *AdguardExtractor) Convert(blocklistBytes []byte) ([]byte, ConversionResult, error) { domains := make([]string, 0) + exceptions := make([]string, 0) + disabled := make(map[string]struct{}) + var stats ConversionStats scanner := bufio.NewScanner(bytes.NewReader(blocklistBytes)) for scanner.Scan() { @@ -48,17 +58,111 @@ func (e *AdguardExtractor) Convert(blocklistBytes []byte) ([]byte, error) { continue } + // Exception rules are the list's built-in allowlist + // (https://adguard-dns.io/kb/general/dns-filtering-syntax/): extracted + // into the companion exception set the proxy consults before blocking + // a same-list match. Unpublishable ones are counted, not widened. + if strings.HasPrefix(line, exceptionPrefix) { + if domain := processException(strings.TrimPrefix(line, exceptionPrefix)); domain != "" { + exceptions = append(exceptions, domain) + } else { + stats.SkippedExceptions++ + } + continue + } + + // Regex rules can contain '$' in the expression, so they must be + // recognized before modifier parsing. + if strings.HasPrefix(line, regexDelimiter) { + stats.SkippedInvalid++ + continue + } + + pattern, modifiers := splitModifiers(line) + + // A $badfilter rule disables the rule matching its remaining text + // instead of adding one + // (https://adguard-dns.io/kb/general/dns-filtering-syntax/#badfilter). + if hasModifier(modifiers, badfilterModifier) { + stats.SkippedBadfilter++ + if domain := processRule(line); domain != "" { + disabled[domain] = struct{}{} + } + continue + } + + // $important only strengthens a block at DNS level, so the block is + // kept. Every other modifier conditions, scopes or rewrites the rule + // ($dnstype, $client, $dnsrewrite, …) — keeping the bare pattern + // would over-block, so the rule is dropped instead. + if hasUnsupportedModifier(modifiers) { + stats.SkippedModifiers++ + continue + } + + if strings.Contains(pattern, wildcard) { + stats.SkippedWildcards++ + continue + } + + // A single-pipe pattern ending with a dot (`|load.gtm.`) matches + // hostnames *starting with* the token; extracting it would emit the + // literal token as a bogus domain. + if isPrefixRule(pattern) { + stats.SkippedPrefixes++ + continue + } + // Process the line to extract the domain if domain := processRule(line); domain != "" { domains = append(domains, domain) + } else { + stats.SkippedInvalid++ } } if err := scanner.Err(); err != nil { - return nil, fmt.Errorf("error scanning blocklist: %w", err) + return nil, ConversionResult{}, fmt.Errorf("error scanning blocklist: %w", err) + } + + if len(disabled) > 0 { + kept := domains[:0] + for _, d := range domains { + if _, ok := disabled[d]; ok { + stats.SkippedBadfilter++ + continue + } + kept = append(kept, d) + } + domains = kept } - return []byte(strings.Join(domains, "\n")), nil + return []byte(strings.Join(domains, "\n")), ConversionResult{Exceptions: exceptions, Stats: stats}, nil +} + +// processException extracts the domain from an @@ rule body (the rule with +// its "@@" prefix removed), applying the same syntax policy as block rules. +// A $badfilter modifier disables the exception itself, and $important on an +// exception is tolerated as a plain exception (it only matters against +// $important blocks, which the compiled format cannot distinguish). Returns +// "" when the exception cannot be published. +func processException(body string) string { + if strings.HasPrefix(body, regexDelimiter) { + return "" + } + pattern, modifiers := splitModifiers(body) + if hasModifier(modifiers, badfilterModifier) || hasUnsupportedModifier(modifiers) { + return "" + } + if strings.Contains(pattern, wildcard) || isPrefixRule(pattern) { + return "" + } + pattern = strings.ReplaceAll(pattern, "^", "") + pattern = strings.ReplaceAll(pattern, "|", "") + if d := NormalizeDomain(pattern); ValidDomain(d) { + return d + } + return "" } // ExtractMetadata extracts metadata from the blocklist including last modified time, @@ -121,6 +225,45 @@ func processRule(rule string) string { return "" } +// splitModifiers splits a rule at the first '$' into its pattern and its +// comma-separated modifier list (nil when the rule has none). +func splitModifiers(rule string) (string, []string) { + pattern, modifiers, found := strings.Cut(rule, modifierSeparator) + if !found { + return pattern, nil + } + return pattern, strings.Split(modifiers, ",") +} + +func hasModifier(modifiers []string, name string) bool { + for _, m := range modifiers { + if m == name { + return true + } + } + return false +} + +// hasUnsupportedModifier reports whether the rule carries any modifier other +// than the bare $important (the only one that leaves a DNS-level block a +// block). +func hasUnsupportedModifier(modifiers []string) bool { + for _, m := range modifiers { + if m != importantModifier { + return true + } + } + return false +} + +// isPrefixRule reports whether the pattern is a single-pipe hostname-prefix +// match: anchored to the name start and ending with a dot, e.g. `|load.gtm.`. +func isPrefixRule(pattern string) bool { + return strings.HasPrefix(pattern, "|") && + !strings.HasPrefix(pattern, "||") && + strings.HasSuffix(pattern, ".") +} + // isCommentOrEmpty checks if a line is either empty or a comment func isCommentOrEmpty(line string) bool { return line == "" || diff --git a/blocklists/internal/extractor/adguard_test.go b/blocklists/internal/extractor/adguard_test.go index 60ea1bf0..7d6f33f9 100644 --- a/blocklists/internal/extractor/adguard_test.go +++ b/blocklists/internal/extractor/adguard_test.go @@ -48,11 +48,48 @@ example.org`, want: "example.com\nexample.org", }, { + // specRef: #D2 — $important keeps the block; #D2d — any other + // modifier drops the rule instead of widening it into an + // unconditional block. name: "with modifiers", input: `example.com$important example.org^$third-party ||example.net^`, - want: "example.com\nexample.org\nexample.net", + want: "example.com\nexample.net", + }, + { + // specRef: #D2d — a conditioned rule ($dnstype scopes it to one + // query type) must not compile to an unconditional block. + name: "dnstype modifier drops the rule", + input: `||example.com^$dnstype=AAAA`, + want: "", + }, + { + // specRef: #D2d — fail open on modifiers that do not exist yet. + name: "unknown future modifier drops the rule", + input: `||example.com^$frobnicate=1`, + want: "", + }, + { + // specRef: #D2e — wildcard patterns are unsupported. + name: "wildcard pattern skipped", + input: `||ads*.example.com^ +||example.net^`, + want: "example.net", + }, + { + // specRef: #D2f — a hostname-prefix rule must not leak the literal + // token as a domain. + name: "hostname prefix rule skipped", + input: `|load.gtm. +||example.net^`, + want: "example.net", + }, + { + // specRef: #D5 — regex rules fail validation and are dropped. + name: "regex rule skipped", + input: `/^ad[0-9]+\.example\.com$/`, + want: "", }, { name: "invalid domains", @@ -61,6 +98,60 @@ example.com also-not-a-domain`, want: "example.com", }, + { + // specRef: #D2a — a $badfilter rule disables a rule; it is never + // emitted as a block itself. + name: "badfilter rule alone", + input: `||example.com^$badfilter`, + want: "", + }, + { + // specRef: #D2b — a $badfilter rule removes its target from the + // output, regardless of line order. + name: "badfilter removes matching block rule", + input: `||example.com^ +||tracker.org^ +||example.com^$badfilter`, + want: "tracker.org", + }, + { + // specRef: #D2b — order-independent: badfilter before the block. + name: "badfilter before matching block rule", + input: `||example.com^$badfilter +||example.com^ +||tracker.org^`, + want: "tracker.org", + }, + { + // specRef: #D2b — bare-domain badfilter form. + name: "bare domain badfilter", + input: `wykop.pl$badfilter +||tracker.org^`, + want: "tracker.org", + }, + { + // specRef: #D2b — badfilter among comma-separated modifiers. + name: "badfilter combined with another modifier", + input: `||example.com^$important,badfilter +||example.com^`, + want: "", + }, + { + // specRef: #D2c — badfilter on an exception disables the + // exception, not the block rule for the same domain. + name: "badfilter on exception does not disable block", + input: `@@||example.com^$badfilter +||example.com^`, + want: "example.com", + }, + { + // specRef: #D2d — a modifier value containing "badfilter" is not + // $badfilter; the rule is dropped as an unsupported modifier, not + // treated as a disable directive. + name: "modifier value containing badfilter text", + input: `||example.com^$dnstype=badfilter`, + want: "", + }, { name: "empty input", input: "", @@ -73,7 +164,7 @@ also-not-a-domain`, for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := extractor.Convert([]byte(tt.input)) + got, _, err := extractor.Convert([]byte(tt.input)) if tt.wantErr { assert.Error(t, err) return @@ -84,6 +175,131 @@ also-not-a-domain`, } } +// specRef: #D2a #D2d #D2e #D2f — per-reason skip counts reported alongside the +// converted output, published as blocklists_rules_skipped{source,reason}. +func TestAdguardExtractor_ConvertStats(t *testing.T) { + input := `! comment +||blocked.com^ +@@||allowed.com^ +||disabled.com^ +||disabled.com^$badfilter +||typed.com^$dnstype=AAAA +||ads*.example.com^ +|load.gtm. +/^regex$/ +not a domain line +||important.com^$important` + + got, res, err := NewAdguardExtractor().Convert([]byte(input)) + assert.NoError(t, err) + assert.Equal(t, "blocked.com\nimportant.com", string(got)) + // specRef: #D3 — the parseable exception is extracted, not skipped. + assert.Equal(t, []string{"allowed.com"}, res.Exceptions) + assert.Equal(t, ConversionStats{ + SkippedBadfilter: 2, // the $badfilter rule and the target it disabled + SkippedModifiers: 1, + SkippedWildcards: 1, + SkippedPrefixes: 1, + SkippedInvalid: 2, // regex rule + non-domain line + }, res.Stats) +} + +// specRef: #D3 #D3a — @@ rules become the source's companion exception set; +// unpublishable ones are counted as `exception`. +func TestAdguardExtractor_ConvertExceptions(t *testing.T) { + tests := []struct { + name string + input string + wantDomains string + wantExceptions []string + wantSkipped int + }{ + { + // specRef: #D3 — the canonical pair: block plus same-list unblock. + name: "double-pipe exception extracted", + input: `||blocked.com^ +@@||cdn.blocked.com^`, + wantDomains: "blocked.com", + wantExceptions: []string{"cdn.blocked.com"}, + }, + { + // specRef: #D3 — single-pipe anchored form. + name: "single-pipe exception extracted", + input: `@@|cdn.example.com^|`, + wantDomains: "", + wantExceptions: []string{"cdn.example.com"}, + }, + { + // specRef: #D3 — bare-domain form. + name: "bare exception extracted", + input: `@@exception.com`, + wantDomains: "", + wantExceptions: []string{"exception.com"}, + }, + { + // specRef: #D3 — $important on an exception is tolerated and + // treated as a plain exception. + name: "important exception treated as plain", + input: `@@||cdn.example.com^$important`, + wantDomains: "", + wantExceptions: []string{"cdn.example.com"}, + }, + { + // specRef: #D3a — wildcard exceptions cannot be published. + name: "wildcard exception skipped", + input: `@@||cdn-*.example.com^`, + wantDomains: "", + wantSkipped: 1, + }, + { + // specRef: #D3a — exceptions with unsupported modifiers dropped. + name: "modified exception skipped", + input: `@@||cdn.example.com^$dnstype=AAAA`, + wantDomains: "", + wantSkipped: 1, + }, + { + // specRef: #D3a #D2c — $badfilter disables the exception itself; + // the block for the same domain is unaffected. + name: "badfilter exception dropped and block kept", + input: `@@||example.com^$badfilter +||example.com^`, + wantDomains: "example.com", + wantSkipped: 1, + }, + { + // specRef: #D3a — regex exceptions cannot be published. + name: "regex exception skipped", + input: `@@/^ads\./`, + wantDomains: "", + wantSkipped: 1, + }, + { + // specRef: #D3a — invalid exception domains are dropped. + name: "invalid exception skipped", + input: `@@not a domain`, + wantDomains: "", + wantSkipped: 1, + }, + } + + extractor := NewAdguardExtractor() + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, res, err := extractor.Convert([]byte(tt.input)) + assert.NoError(t, err) + assert.Equal(t, tt.wantDomains, string(got)) + if tt.wantExceptions == nil { + assert.Empty(t, res.Exceptions) + } else { + assert.Equal(t, tt.wantExceptions, res.Exceptions) + } + assert.Equal(t, tt.wantSkipped, res.Stats.SkippedExceptions) + }) + } +} + func TestAdguardExtractor_ExtractMetadata(t *testing.T) { tests := []struct { name string diff --git a/blocklists/internal/extractor/domains.go b/blocklists/internal/extractor/domains.go index 3a04e751..d5f082e9 100644 --- a/blocklists/internal/extractor/domains.go +++ b/blocklists/internal/extractor/domains.go @@ -34,9 +34,9 @@ func NewDomainsExtractor() *DomainsExtractor { // actually in hosts format (`0.0.0.0 domain`), so a leading IP field is // stripped. Comments and blank lines are dropped; the shared NormalizeDomain + // ValidDomain gate downstream does the final cleaning/validation. -func (e *DomainsExtractor) Convert(blocklistBytes []byte) ([]byte, error) { +func (e *DomainsExtractor) Convert(blocklistBytes []byte) ([]byte, ConversionResult, error) { if len(blocklistBytes) == 0 { - return []byte{}, nil + return []byte{}, ConversionResult{}, nil } out := make([]string, 0) @@ -47,9 +47,9 @@ func (e *DomainsExtractor) Convert(blocklistBytes []byte) ([]byte, error) { } } if err := scanner.Err(); err != nil { - return nil, err + return nil, ConversionResult{}, err } - return []byte(strings.Join(out, "\n")), nil + return []byte(strings.Join(out, "\n")), ConversionResult{}, nil } // stripHostsIP returns the domain candidate from a plain-list or hosts-format diff --git a/blocklists/internal/extractor/domains_test.go b/blocklists/internal/extractor/domains_test.go index 3a4ff4fa..8621b02a 100644 --- a/blocklists/internal/extractor/domains_test.go +++ b/blocklists/internal/extractor/domains_test.go @@ -43,7 +43,7 @@ func TestDomainsExtractor_Convert(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := extractor.Convert([]byte(tt.input)) + got, _, err := extractor.Convert([]byte(tt.input)) if tt.wantErr { assert.Error(t, err) return diff --git a/blocklists/internal/extractor/extractor.go b/blocklists/internal/extractor/extractor.go index 62f4ab7c..90373495 100644 --- a/blocklists/internal/extractor/extractor.go +++ b/blocklists/internal/extractor/extractor.go @@ -16,10 +16,32 @@ const ( TypeStevenBlack = "steven_black" ) +// ConversionStats counts the input rules Convert dropped, by reason. Formats +// without unsupported syntax (plain domain lists, hosts files) report the zero +// value; only the AdGuard extractor currently distinguishes reasons. +type ConversionStats struct { + SkippedExceptions int // @@ exception rules + SkippedBadfilter int // $badfilter rules plus the block targets they disabled + SkippedModifiers int // rules carrying a modifier outside the supported set + SkippedWildcards int // patterns containing '*' + SkippedPrefixes int // '|host.' hostname-prefix patterns + SkippedInvalid int // remaining rules failing domain validation +} + +// ConversionResult carries the non-domain outputs of Convert. +type ConversionResult struct { + // Exceptions are the domains the list's own @@ rules unblock (its + // built-in allowlist); published as a companion set the proxy consults + // before blocking a match from the same list. Empty for formats without + // exception syntax. + Exceptions []string + Stats ConversionStats +} + type Extractor interface { ExtractMetadata(blocklistBytes []byte) (time.Time, string, int, error) ProcessLine(line string) (string, error) - Convert(blocklistBytes []byte) ([]byte, error) + Convert(blocklistBytes []byte) ([]byte, ConversionResult, error) } // NewExtractor creates a new Extractor instance based on the blocklist ID diff --git a/blocklists/internal/extractor/hagezi.go b/blocklists/internal/extractor/hagezi.go index 6488644d..123657c6 100644 --- a/blocklists/internal/extractor/hagezi.go +++ b/blocklists/internal/extractor/hagezi.go @@ -32,11 +32,11 @@ func NewHageziExtractor() *HageziExtractor { // Convert processes the blocklist bytes and returns them unchanged // as Hagezi format is already in the desired format -func (e *HageziExtractor) Convert(blocklistBytes []byte) ([]byte, error) { +func (e *HageziExtractor) Convert(blocklistBytes []byte) ([]byte, ConversionResult, error) { if len(blocklistBytes) == 0 { - return []byte{}, nil + return []byte{}, ConversionResult{}, nil } - return blocklistBytes, nil + return blocklistBytes, ConversionResult{}, nil } // ExtractMetadata extracts metadata from the blocklist including: diff --git a/blocklists/internal/extractor/hagezi_test.go b/blocklists/internal/extractor/hagezi_test.go index c52770a2..f61adcc8 100644 --- a/blocklists/internal/extractor/hagezi_test.go +++ b/blocklists/internal/extractor/hagezi_test.go @@ -35,7 +35,7 @@ example.org`, for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := extractor.Convert([]byte(tt.input)) + got, _, err := extractor.Convert([]byte(tt.input)) if tt.wantErr { assert.Error(t, err) return diff --git a/blocklists/internal/extractor/oisd.go b/blocklists/internal/extractor/oisd.go index 9d882324..9db464aa 100644 --- a/blocklists/internal/extractor/oisd.go +++ b/blocklists/internal/extractor/oisd.go @@ -32,11 +32,11 @@ func NewOISDExtractor() *OISDExtractor { // Convert processes the blocklist bytes and returns them unchanged // as OISD format is already in the desired format -func (e *OISDExtractor) Convert(blocklistBytes []byte) ([]byte, error) { +func (e *OISDExtractor) Convert(blocklistBytes []byte) ([]byte, ConversionResult, error) { if len(blocklistBytes) == 0 { - return []byte{}, nil + return []byte{}, ConversionResult{}, nil } - return blocklistBytes, nil + return blocklistBytes, ConversionResult{}, nil } // ExtractMetadata extracts metadata from the blocklist including: diff --git a/blocklists/internal/extractor/oisd_test.go b/blocklists/internal/extractor/oisd_test.go index 2444da65..cb0e6f60 100644 --- a/blocklists/internal/extractor/oisd_test.go +++ b/blocklists/internal/extractor/oisd_test.go @@ -45,7 +45,7 @@ example.com`, for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := extractor.Convert([]byte(tt.input)) + got, _, err := extractor.Convert([]byte(tt.input)) if tt.wantErr { assert.Error(t, err) return diff --git a/blocklists/internal/extractor/steven_black.go b/blocklists/internal/extractor/steven_black.go index 276cab9b..5021089d 100644 --- a/blocklists/internal/extractor/steven_black.go +++ b/blocklists/internal/extractor/steven_black.go @@ -37,9 +37,9 @@ func NewStevenBlackExtractor() *StevenBlackExtractor { } // Convert transforms Steven Black hosts file format into a simple domain list -func (e *StevenBlackExtractor) Convert(blocklistBytes []byte) ([]byte, error) { +func (e *StevenBlackExtractor) Convert(blocklistBytes []byte) ([]byte, ConversionResult, error) { if len(blocklistBytes) == 0 { - return []byte{}, nil + return []byte{}, ConversionResult{}, nil } domains := make([]string, 0) @@ -75,10 +75,10 @@ func (e *StevenBlackExtractor) Convert(blocklistBytes []byte) ([]byte, error) { } if err := scanner.Err(); err != nil { - return nil, fmt.Errorf("error scanning hosts file: %w", err) + return nil, ConversionResult{}, fmt.Errorf("error scanning hosts file: %w", err) } - return []byte(strings.Join(domains, "\n")), nil + return []byte(strings.Join(domains, "\n")), ConversionResult{}, nil } // ExtractMetadata extracts metadata from the Steven Black hosts file including: diff --git a/blocklists/internal/extractor/steven_black_test.go b/blocklists/internal/extractor/steven_black_test.go index 571908bb..d11f2a5a 100644 --- a/blocklists/internal/extractor/steven_black_test.go +++ b/blocklists/internal/extractor/steven_black_test.go @@ -95,7 +95,7 @@ another-valid.example.org`, for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := extractor.Convert([]byte(tt.input)) + got, _, err := extractor.Convert([]byte(tt.input)) if tt.wantErr { assert.Error(t, err) return diff --git a/blocklists/internal/metrics/metrics.go b/blocklists/internal/metrics/metrics.go index 8e832905..0b666f0e 100644 --- a/blocklists/internal/metrics/metrics.go +++ b/blocklists/internal/metrics/metrics.go @@ -28,6 +28,16 @@ const ( ReasonTruncated = "truncated" ) +// Reason label values for blocklists_rules_skipped. +const ( + SkipRuleException = "exception" + SkipRuleBadfilter = "badfilter" + SkipRuleModifier = "modifier" + SkipRuleWildcard = "wildcard" + SkipRulePrefix = "prefix" + SkipRuleInvalid = "invalid" +) + // Reason label values for blocklists_refresh_skipped_total. const ( // SkipReasonFresh: the source was refreshed recently (typically by a peer @@ -54,6 +64,18 @@ type Updates interface { // Compared against SetDomainsExtracted it is a divergence signal (a large // drop hints at a partial download or many malformed/duplicate lines). SetDeclaredEntries(source string, n int) + // SetExceptionsExtracted records the number of exception domains published + // to the source's companion exception set in the last update (0 = the + // source has none and no exception key exists). + SetExceptionsExtracted(source string, n int) + // SetRulesSkipped records how many input rules the last published + // conversion dropped for a source, by reason + // (exception|badfilter|modifier|wildcard|prefix|invalid). The extractor + // fails open — an unrecognized rule is dropped, never widened into a + // block — so a jump in modifier/wildcard means the upstream list started + // shipping syntax the extractor does not understand: an under-block worth + // review, not a false-positive risk. + SetRulesSkipped(source, reason string, n int) // SetLastSuccess records the wall-clock time of the last successful swap for a source. SetLastSuccess(source string, ts time.Time) // RecordDownloadBytes records the number of bytes downloaded for a source. @@ -81,6 +103,8 @@ func (NoopUpdates) RecordUpdate(string, string) {} func (NoopUpdates) RecordDuration(string, time.Duration) {} func (NoopUpdates) SetDomainsExtracted(string, int) {} func (NoopUpdates) SetDeclaredEntries(string, int) {} +func (NoopUpdates) SetExceptionsExtracted(string, int) {} +func (NoopUpdates) SetRulesSkipped(string, string, int) {} func (NoopUpdates) SetLastSuccess(string, time.Time) {} func (NoopUpdates) RecordDownloadBytes(string, int64) {} func (NoopUpdates) RecordValidationRejected(string, string) {} @@ -94,6 +118,8 @@ type PromUpdates struct { updateDuration *prometheus.HistogramVec domainsExtracted *prometheus.GaugeVec declaredEntries *prometheus.GaugeVec + rulesSkipped *prometheus.GaugeVec + exceptionsCount *prometheus.GaugeVec lastSuccess *prometheus.GaugeVec downloadBytes *prometheus.GaugeVec validationRejects *prometheus.CounterVec @@ -122,6 +148,14 @@ func NewPromUpdates(reg prometheus.Registerer) *PromUpdates { Name: "blocklists_source_declared_entries", Help: "Entry count reported by the source (header value, or non-comment line count when no header is present) in the last update by source.", }, []string{"source"}), + exceptionsCount: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "blocklists_exceptions_extracted", + Help: "Number of exception domains published to the source's companion exception set in the last update.", + }, []string{"source"}), + rulesSkipped: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "blocklists_rules_skipped", + Help: "Number of input rules dropped during the last published conversion, by source and reason (exception|badfilter|modifier|wildcard|prefix|invalid). Rising modifier/wildcard counts signal upstream syntax the extractor does not understand (under-block, fail open).", + }, []string{"source", "reason"}), lastSuccess: prometheus.NewGaugeVec(prometheus.GaugeOpts{ Name: "blocklists_last_success_timestamp_seconds", Help: "Unix timestamp of the last successful blocklist update by source.", @@ -152,6 +186,8 @@ func NewPromUpdates(reg prometheus.Registerer) *PromUpdates { m.updateDuration, m.domainsExtracted, m.declaredEntries, + m.rulesSkipped, + m.exceptionsCount, m.lastSuccess, m.downloadBytes, m.validationRejects, @@ -178,6 +214,14 @@ func (m *PromUpdates) SetDeclaredEntries(source string, n int) { m.declaredEntries.WithLabelValues(source).Set(float64(n)) } +func (m *PromUpdates) SetExceptionsExtracted(source string, n int) { + m.exceptionsCount.WithLabelValues(source).Set(float64(n)) +} + +func (m *PromUpdates) SetRulesSkipped(source, reason string, n int) { + m.rulesSkipped.WithLabelValues(source, reason).Set(float64(n)) +} + func (m *PromUpdates) SetLastSuccess(source string, ts time.Time) { m.lastSuccess.WithLabelValues(source).Set(float64(ts.Unix())) } diff --git a/blocklists/internal/metrics/metrics_test.go b/blocklists/internal/metrics/metrics_test.go index 366615db..63702dbb 100644 --- a/blocklists/internal/metrics/metrics_test.go +++ b/blocklists/internal/metrics/metrics_test.go @@ -22,6 +22,8 @@ func TestPromUpdates_RecordsAllSeries(t *testing.T) { m.SetLastSuccess(source, time.Unix(1700000000, 0)) m.RecordDownloadBytes(source, 4096) m.RecordValidationRejected(source, ReasonShrink) + m.SetRulesSkipped(source, SkipRuleModifier, 7) + m.SetExceptionsExtracted(source, 42) if got := testutil.ToFloat64(m.updates.WithLabelValues(source, StatusSuccess)); got != 1 { t.Errorf("update_total{success} = %v, want 1", got) @@ -44,6 +46,12 @@ func TestPromUpdates_RecordsAllSeries(t *testing.T) { if got := testutil.ToFloat64(m.validationRejects.WithLabelValues(source, ReasonShrink)); got != 1 { t.Errorf("validation_rejected{shrink} = %v, want 1", got) } + if got := testutil.ToFloat64(m.rulesSkipped.WithLabelValues(source, SkipRuleModifier)); got != 7 { + t.Errorf("rules_skipped{modifier} = %v, want 7", got) + } + if got := testutil.ToFloat64(m.exceptionsCount.WithLabelValues(source)); got != 42 { + t.Errorf("exceptions_extracted = %v, want 42", got) + } } func TestPromUpdates_DurationObserved(t *testing.T) { diff --git a/blocklists/main.go b/blocklists/main.go index d23cec45..50e233a2 100644 --- a/blocklists/main.go +++ b/blocklists/main.go @@ -23,6 +23,11 @@ import ( ) func main() { + os.Exit(run()) +} + +// run holds main's body so its defers execute before os.Exit. +func run() int { defer func() { if r := recover(); r != nil { sentry.CurrentHub().Recover(r) @@ -118,8 +123,8 @@ func main() { service.CatchUp(sources) service.PurgeStaleCoordinated(sources) + // Stop() runs on the signal paths before the exit code is sent. updater.Start() - defer updater.Stop() signalChan := make(chan os.Signal, 1) signal.Notify(signalChan, @@ -168,7 +173,7 @@ func main() { } cancel() } - os.Exit(code) + return code } // safelyRun wraps each goroutine with panic recovery to ensure the application continues even if a panic occurs diff --git a/blocklists/model/blocklist.go b/blocklists/model/blocklist.go index ad269f41..d8807b24 100644 --- a/blocklists/model/blocklist.go +++ b/blocklists/model/blocklist.go @@ -10,18 +10,26 @@ import ( const ( BlocklistTypePublic = "public" + + // ContentKindExceptions marks a BlocklistContent document holding the + // list's own exception domains (@@ rules) rather than block domains. + ContentKindExceptions = "exceptions" ) // BlocklistMetadata is a blocklist model type BlocklistMetadata struct { - ID primitive.ObjectID `json:"id" bson:"_id"` - BlocklistID string `json:"blocklist_id" bson:"blocklist_id" binding:"required"` - Name string `json:"name" binding:"required"` // conventional blocklist name, displayed to the user - Description string `json:"description" binding:"required"` // displayed to the user - Entries int `json:"entries"` - Homepage string `json:"homepage"` - SourceUrl string `json:"source_url" bson:"source_url"` - LastModified time.Time `json:"last_modified" bson:"last_modified"` + ID primitive.ObjectID `json:"id" bson:"_id"` + BlocklistID string `json:"blocklist_id" bson:"blocklist_id" binding:"required"` + Name string `json:"name" binding:"required"` // conventional blocklist name, displayed to the user + Description string `json:"description" binding:"required"` // displayed to the user + Entries int `json:"entries"` + // ExceptionEntries is the number of exception domains published alongside + // the block set; 0 means no companion exception set is expected to exist + // (drives the freshness backstop's EXISTS check). + ExceptionEntries int `json:"exception_entries" bson:"exception_entries"` + Homepage string `json:"homepage"` + SourceUrl string `json:"source_url" bson:"source_url"` + LastModified time.Time `json:"last_modified" bson:"last_modified"` // UpdatedAt is when this service last published the list (Redis + Mongo). // Unlike LastModified (the upstream's own header date) it is our publish // timestamp, used by peer instances to skip refreshing a fresh source. @@ -42,6 +50,7 @@ type BlocklistContent struct { ID primitive.ObjectID `json:"id" bson:"_id"` BlocklistID string `json:"blocklist_id" bson:"blocklist_id"` Part int `json:"part" bson:"part"` + Kind string `json:"kind,omitempty" bson:"kind,omitempty"` // "" = block domains, "exceptions" = exception domains Data []byte `json:"-" bson:"data"` } diff --git a/blocklists/service/blocklists.go b/blocklists/service/blocklists.go index edab0077..9ae467e9 100644 --- a/blocklists/service/blocklists.go +++ b/blocklists/service/blocklists.go @@ -163,7 +163,7 @@ func (s *Service) processBlocklist(metadata model.BlocklistMetadata) (*model.Blo return nil, err } - domainsBytes, err := extr.Convert(blocklistBytes) + domainsBytes, convRes, err := extr.Convert(blocklistBytes) if err != nil { log.Err(err).Str("blocklist_id", metadata.BlocklistID).Msg("Failed to convert blocklist") return nil, err @@ -214,7 +214,32 @@ func (s *Service) processBlocklist(metadata model.BlocklistMetadata) (*model.Blo return nil, err } + // Exceptions pass the same shared validation gate as domains. They do not + // feed the shrink gate: the set is small and volatile, and losing it only + // under-suppresses, never over-blocks. + exceptions := make([]string, 0, len(convRes.Exceptions)) + for _, d := range convRes.Exceptions { + if nd := extractor.NormalizeDomain(d); extractor.ValidDomain(nd) { + exceptions = append(exceptions, nd) + } + } + s.Metrics.SetDomainsExtracted(metadata.BlocklistID, totalDomains) + // Published every refresh, zeros included, so each series reflects the + // last successful conversion rather than holding a stale spike. + for _, rs := range []struct { + reason string + n int + }{ + {metrics.SkipRuleException, convRes.Stats.SkippedExceptions}, + {metrics.SkipRuleBadfilter, convRes.Stats.SkippedBadfilter}, + {metrics.SkipRuleModifier, convRes.Stats.SkippedModifiers}, + {metrics.SkipRuleWildcard, convRes.Stats.SkippedWildcards}, + {metrics.SkipRulePrefix, convRes.Stats.SkippedPrefixes}, + {metrics.SkipRuleInvalid, convRes.Stats.SkippedInvalid}, + } { + s.Metrics.SetRulesSkipped(metadata.BlocklistID, rs.reason, rs.n) + } if numEntries > 0 { // Source's own count (header or self-counted) — a divergence signal // against the published count above. @@ -237,6 +262,27 @@ func (s *Service) processBlocklist(metadata model.BlocklistMetadata) (*model.Blo } } + // Persist the exception domains as a single content document; the + // per-blocklist cleanup below covers it like any chunk. + if len(exceptions) > 0 { + exceptionsContent, err := model.NewBlocklistContent(metadata.BlocklistID, 1, exceptions) + if err != nil { + return nil, fmt.Errorf("failed to create exceptions content: %w", err) + } + exceptionsContent.Kind = model.ContentKindExceptions + if err := s.Store.UpsertContent(ctx, *exceptionsContent); err != nil { + return nil, fmt.Errorf("failed to upsert exceptions content: %w", err) + } + } + + // Publish the exception set BEFORE the main set so a reader never sees + // new blocks paired with the previous run's exception set; when there are + // no exceptions the companion key is removed. + if err := s.Cache.CreateOrUpdateBlocklistExceptions(ctx, metadata.BlocklistID, []byte(strings.Join(exceptions, "\n"))); err != nil { + return nil, err + } + s.Metrics.SetExceptionsExtracted(metadata.BlocklistID, len(exceptions)) + // Publish the SAME validated domains to the Redis set the proxy reads. data := []byte(strings.Join(validated, "\n")) if err := s.Cache.CreateOrUpdateBlocklist(ctx, metadata.BlocklistID, data); err != nil { @@ -246,6 +292,7 @@ func (s *Service) processBlocklist(metadata model.BlocklistMetadata) (*model.Blo metadata.LastModified = lastModified metadata.Version = version metadata.Entries = totalDomains + metadata.ExceptionEntries = len(exceptions) metadata.Type = model.BlocklistTypePublic metadata.UpdatedAt = time.Now().UTC() diff --git a/blocklists/service/blocklists_test.go b/blocklists/service/blocklists_test.go index 9fca619d..8dc4a185 100644 --- a/blocklists/service/blocklists_test.go +++ b/blocklists/service/blocklists_test.go @@ -129,7 +129,7 @@ func TestStevenBlackEndToEnd(t *testing.T) { t.Fatalf("NewExtractor: %v", err) } hosts := "# Title: test\n0.0.0.0 Ads.Example.COM\n0.0.0.0 0.0.0.0\n127.0.0.1 skip.example.org\n" - converted, err := extr.Convert([]byte(hosts)) + converted, _, err := extr.Convert([]byte(hosts)) if err != nil { t.Fatalf("Convert: %v", err) } @@ -179,16 +179,29 @@ func (f *fakeStore) DeleteMetadata(_ context.Context, filter map[string]any) err return nil } -// fakeCache implements cache.Cache, recording blocklist deletions. Live sets -// are treated as present unless listed in missing. +// fakeCache implements cache.Cache, recording blocklist deletions and +// exception publishes. Live sets are treated as present unless listed in +// missing; exception sets unless listed in missingExceptions. type fakeCache struct { - deleted []string - missing map[string]bool + deleted []string + missing map[string]bool + missingExceptions map[string]bool + exceptionsPut map[string]string // blocklistID -> last published data } func (f *fakeCache) CreateOrUpdateBlocklist(_ context.Context, _ string, _ []byte) error { return nil } +func (f *fakeCache) CreateOrUpdateBlocklistExceptions(_ context.Context, blocklistId string, data []byte) error { + if f.exceptionsPut == nil { + f.exceptionsPut = make(map[string]string) + } + f.exceptionsPut[blocklistId] = string(data) + return nil +} +func (f *fakeCache) BlocklistExceptionsExist(_ context.Context, blocklistId string) (bool, error) { + return !f.missingExceptions[blocklistId], nil +} func (f *fakeCache) DeleteBlocklist(_ context.Context, blocklistId string) error { f.deleted = append(f.deleted, blocklistId) return nil diff --git a/blocklists/service/coordinate.go b/blocklists/service/coordinate.go index 1b517de5..163a5044 100644 --- a/blocklists/service/coordinate.go +++ b/blocklists/service/coordinate.go @@ -68,7 +68,16 @@ func (s *Service) isFresh(ctx context.Context, src model.BlocklistMetadata) bool return false } exists, err := s.Cache.BlocklistExists(ctx, src.BlocklistID) - return err == nil && exists + if err != nil || !exists { + return false + } + // A live main set without its expected exception set would reintroduce + // the false positives the exceptions prevent, so it counts as lost too. + if existing[0].ExceptionEntries > 0 { + exists, err = s.Cache.BlocklistExceptionsExist(ctx, src.BlocklistID) + return err == nil && exists + } + return true } // RefreshDue processes the source unless it was published recently. It is the diff --git a/blocklists/service/coordinate_test.go b/blocklists/service/coordinate_test.go index a221e6f2..1b852814 100644 --- a/blocklists/service/coordinate_test.go +++ b/blocklists/service/coordinate_test.go @@ -355,3 +355,38 @@ func TestPurgeStale_AllowsPurgeAtMax(t *testing.T) { t.Fatalf("deleted cache = %v, want 2 entries", cache.deleted) } } + +// specRef: #G11 — when the stored metadata records exception entries, a lost +// exception key makes the source stale even with the main set present: a live +// main set without its exception set would reintroduce the false positives +// the exceptions prevent. +func TestIsFresh_MissingExceptionSetIsStale(t *testing.T) { + stored := []model.BlocklistMetadata{{BlocklistID: "adg", UpdatedAt: time.Now().UTC().Add(-5 * time.Minute), ExceptionEntries: 10}} + s, _ := newCoordService(metrics.NoopUpdates{}, nil, stored) + s.Cache = &fakeCache{missingExceptions: map[string]bool{"adg": true}} + + if s.isFresh(context.Background(), coordSource("adg", "0 * * * *", "")) { + t.Fatal("missing exception set must be stale when exception_entries > 0") + } +} + +// specRef: #G11 — exception key present alongside the main set stays fresh. +func TestIsFresh_PresentExceptionSetIsFresh(t *testing.T) { + stored := []model.BlocklistMetadata{{BlocklistID: "adg", UpdatedAt: time.Now().UTC().Add(-5 * time.Minute), ExceptionEntries: 10}} + s, _ := newCoordService(metrics.NoopUpdates{}, nil, stored) + + if !s.isFresh(context.Background(), coordSource("adg", "0 * * * *", "")) { + t.Fatal("fresh metadata with both sets present must be fresh") + } +} + +// specRef: #G11 — sources without exceptions never require the companion key. +func TestIsFresh_NoExceptionsExpectedIgnoresKey(t *testing.T) { + stored := []model.BlocklistMetadata{{BlocklistID: "blp_x", UpdatedAt: time.Now().UTC().Add(-5 * time.Minute)}} + s, _ := newCoordService(metrics.NoopUpdates{}, nil, stored) + s.Cache = &fakeCache{missingExceptions: map[string]bool{"blp_x": true}} + + if !s.isFresh(context.Background(), coordSource("blp_x", "0 * * * *", "")) { + t.Fatal("source without exception entries must not require the exception key") + } +} diff --git a/blocklists/service/real_blocklists_test.go b/blocklists/service/real_blocklists_test.go index 479c39ee..b52eadf8 100644 --- a/blocklists/service/real_blocklists_test.go +++ b/blocklists/service/real_blocklists_test.go @@ -119,7 +119,7 @@ func TestRealBlocklists(t *testing.T) { t.Errorf("%s: expected a non-zero Last-Modified from the header block", fx.extractor) } - converted, err := extr.Convert(raw) + converted, _, err := extr.Convert(raw) if err != nil { t.Fatalf("Convert: %v", err) } diff --git a/dnscheck/.env.sample b/dnscheck/.env.sample index c940a3c9..870d578f 100644 --- a/dnscheck/.env.sample +++ b/dnscheck/.env.sample @@ -2,7 +2,7 @@ DNS_AUTH_SERVER_DOMAIN="test.moddns.net" DNS_AUTH_SERVER_IP_ADDRESS="127.0.0.1" DNS_AUTH_SERVER_ASN="" -DNS_AUTH_SERVER_IP_RANGE="10.5." +DNS_AUTH_SERVER_IP_RANGE="10.5.0.0/16" ### API CONFIG API_PORT=":3000" diff --git a/dnscheck/api/check.go b/dnscheck/api/check.go index f7f489d9..17ed772c 100644 --- a/dnscheck/api/check.go +++ b/dnscheck/api/check.go @@ -17,7 +17,6 @@ var subdomainRegex = regexp.MustCompile(dns.SubdomainRegexPattern) func (s *APIServer) DnsCheck() fiber.Handler { handler := func(c *fiber.Ctx) error { host := c.Hostname() - log.Debug().Str("host", host).Msg("Host") hostParts := strings.Split(host, ".") if len(hostParts) < 2 { log.Error().Msg(ErrInvalidHostHeader) @@ -27,13 +26,12 @@ func (s *APIServer) DnsCheck() fiber.Handler { subdomain := strings.ToLower(hostParts[0]) if !subdomainRegex.MatchString(subdomain) { - log.Warn().Str("subdomain", subdomain).Msg("Invalid subdomain format") + log.Warn().Msg("Invalid subdomain format") return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{"error": "invalid request"}) } // get data from cache cacheKey := cache.HMACKey(s.Config.Cache.HMACKey, subdomain) - log.Debug().Str("ID", subdomain).Msg("Getting query data") data, err := s.Cache.GetQueryData(cacheKey) if err != nil { return HandleError(c, err, ErrFailedToGetQueryData) @@ -42,7 +40,7 @@ func (s *APIServer) DnsCheck() fiber.Handler { // Delete-on-read: each subdomain is single-use (frontend generates a fresh // nanoid per poll), so delete immediately to minimize the replay window. if delErr := s.Cache.DeleteQueryData(cacheKey); delErr != nil { - log.Warn().Err(delErr).Str("ID", subdomain).Msg("Failed to delete cache entry after read") + log.Warn().Err(delErr).Msg("Failed to delete cache entry after read") } var dnsRecord dns.DNSLogRecord diff --git a/dnscheck/api/check_test.go b/dnscheck/api/check_test.go new file mode 100644 index 00000000..6cbf7afa --- /dev/null +++ b/dnscheck/api/check_test.go @@ -0,0 +1,191 @@ +package api + +import ( + "bytes" + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/dnscheck/cache" + "github.com/dnscheck/config" + "github.com/dnscheck/dns" + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" +) + +type memCache struct { + saved map[string][]byte +} + +func (c *memCache) SaveQueryData(key string, value []byte) error { + if c.saved == nil { + c.saved = map[string][]byte{} + } + c.saved[key] = value + return nil +} + +func (c *memCache) GetQueryData(key string) ([]byte, error) { + v, ok := c.saved[key] + if !ok { + return nil, errors.New(ErrEntryNotFound) + } + return v, nil +} + +func (c *memCache) DeleteQueryData(key string) error { delete(c.saved, key); return nil } + +const ( + testHMACKey = "test-key" + testSubdomain = "abcdefghijkl" + testHost = testSubdomain + ".check.example.test" +) + +func newTestServer(c *memCache) *APIServer { + return newTestServerWithAccessLog(c, io.Discard) +} + +func newTestServerWithAccessLog(c *memCache, accessLog io.Writer) *APIServer { + s := NewServer(&config.Config{ + API: &config.APIConfig{ApiAllowOrigin: "*"}, + Cache: &config.CacheConfig{HMACKey: testHMACKey}, + }, c) + s.AccessLog = accessLog + s.RegisterRoutes() + return s +} + +func get(t *testing.T, s *APIServer, host string) (*http.Response, []byte) { + t.Helper() + req := httptest.NewRequest(http.MethodGet, "/", nil) + req.Host = host + resp, err := s.App.Test(req) + if err != nil { + t.Fatalf("request failed: %v", err) + } + body, _ := io.ReadAll(resp.Body) + resp.Body.Close() + return resp, body +} + +// specRef: dnscheck-behaviour.md #A1 +func TestDnsCheckRejectsHostWithoutSubdomain(t *testing.T) { + resp, _ := get(t, newTestServer(&memCache{}), "localhost") + if resp.StatusCode != http.StatusInternalServerError { + t.Errorf("status = %d, want 500", resp.StatusCode) + } +} + +// specRef: dnscheck-behaviour.md #A1 +func TestDnsCheckRejectsMalformedSubdomain(t *testing.T) { + for _, label := range []string{"short", "abcdefghijklm", "abcdefghijkl-", "abcdefghij_l"} { + resp, _ := get(t, newTestServer(&memCache{}), label+".check.example.test") + if resp.StatusCode != http.StatusBadRequest { + t.Errorf("label %q: status = %d, want 400", label, resp.StatusCode) + } + } +} + +// The previous frontend bundle appended "-"; it keeps working until +// every client has picked up the new bundle. +// +// specRef: dnscheck-behaviour.md #A1 +func TestDnsCheckToleratesLegacySuffix(t *testing.T) { + c := &memCache{} + rec, _ := json.Marshal(dns.DNSLogRecord{Status: dns.StatusConfigured, ProfileId: "profile1"}) + legacy := testSubdomain + "-profile1" + _ = c.SaveQueryData(cache.HMACKey(testHMACKey, legacy), rec) + + resp, _ := get(t, newTestServer(c), legacy+".check.example.test") + if resp.StatusCode != http.StatusOK { + t.Errorf("status = %d, want 200 for a legacy-format label", resp.StatusCode) + } +} + +// specRef: dnscheck-behaviour.md #A2 +func TestDnsCheckReturnsDisconnectedWhenNoRecord(t *testing.T) { + resp, body := get(t, newTestServer(&memCache{}), testHost) + if resp.StatusCode != http.StatusNotFound { + t.Errorf("status = %d, want 404", resp.StatusCode) + } + var er ErrResponse + if err := json.Unmarshal(body, &er); err != nil || er.Error != StatusDisconnected { + t.Errorf("body = %s, want error=%q", body, StatusDisconnected) + } +} + +// The record is keyed by an HMAC of the subdomain, only status and profile ID +// are returned, and the entry is deleted on first read. +// +// specRef: dnscheck-behaviour.md #A3, #A4 +func TestDnsCheckReturnsNarrowRecordOnceOnly(t *testing.T) { + c := &memCache{} + rec, _ := json.Marshal(dns.DNSLogRecord{Status: dns.StatusConfigured, ProfileId: "profile1"}) + if err := c.SaveQueryData(cache.HMACKey(testHMACKey, testSubdomain), rec); err != nil { + t.Fatal(err) + } + s := newTestServer(c) + + resp, body := get(t, s, testHost) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200 (body %s)", resp.StatusCode, body) + } + var got map[string]any + if err := json.Unmarshal(body, &got); err != nil { + t.Fatalf("body is not JSON: %s", body) + } + if got["status"] != dns.StatusConfigured || got["profile_id"] != "profile1" { + t.Errorf("body = %s", body) + } + if len(got) != 2 { + t.Errorf("response must carry exactly status and profile_id: %s", body) + } + + if resp, _ := get(t, s, testHost); resp.StatusCode != http.StatusNotFound { + t.Errorf("second read status = %d, want 404 (delete-on-read)", resp.StatusCode) + } +} + +// The Host header (probe ID + profile ID) and the client address must not be +// logged by the handler or the access-log middleware. +// +// specRef: dnscheck-behaviour.md #A5 +func TestDnsCheckLogsCarryNoClientIdentifiers(t *testing.T) { + var buf bytes.Buffer + prev := log.Logger + prevLevel := zerolog.GlobalLevel() + log.Logger = zerolog.New(&buf) + zerolog.SetGlobalLevel(zerolog.TraceLevel) + t.Cleanup(func() { log.Logger = prev; zerolog.SetGlobalLevel(prevLevel) }) + + c := &memCache{} + rec, _ := json.Marshal(dns.DNSLogRecord{Status: dns.StatusConfigured, ProfileId: "profile1"}) + _ = c.SaveQueryData(cache.HMACKey(testHMACKey, testSubdomain), rec) + // The access log shares the buffer so the middleware format is covered too. + s := newTestServerWithAccessLog(c, &buf) + + for _, host := range []string{testHost, testHost, "short.check.example.test", "localhost"} { + req := httptest.NewRequest(http.MethodGet, "/", nil) + req.Host = host + req.RemoteAddr = "203.0.113.5:40000" + resp, err := s.App.Test(req) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + } + + out := buf.String() + if !strings.Contains(out, "GET") { + t.Fatalf("expected access-log lines in output, got:\n%s", out) + } + for _, secret := range []string{"203.0.113.5", testSubdomain, "profile1", "check.example.test"} { + if strings.Contains(out, secret) { + t.Errorf("log output contains %q:\n%s", secret, out) + } + } +} diff --git a/dnscheck/api/server.go b/dnscheck/api/server.go index 49f34551..cba8fd94 100644 --- a/dnscheck/api/server.go +++ b/dnscheck/api/server.go @@ -1,6 +1,9 @@ package api import ( + "io" + "os" + "github.com/dnscheck/cache" "github.com/dnscheck/config" @@ -19,6 +22,8 @@ type APIServer struct { Config *config.Config Validator *APIValidator Cache cache.Cache + // AccessLog receives the per-request access log; defaults to stdout. + AccessLog io.Writer } // NewServer inititiates database connection and sets up API endpoints @@ -38,13 +43,18 @@ func NewServer(config *config.Config, cache cache.Cache) *APIServer { Config: config, Validator: apiValidator, Cache: cache, + AccessLog: os.Stdout, } } // RegisterRoutes registers API endpoints func (s *APIServer) RegisterRoutes() { s.App.Use(requestid.New()) - s.App.Use(logger.New()) + // Default format includes ${ip}; the client address is not logged. + s.App.Use(logger.New(logger.Config{ + Format: "${time} | ${status} | ${latency} | ${method} | ${path} | ${error}\n", + Output: s.AccessLog, + })) s.App.Use(limiter.New( limiter.Config{ Max: 100, diff --git a/dnscheck/cache/bigcache.go b/dnscheck/cache/bigcache.go index 7f98b5da..41976791 100644 --- a/dnscheck/cache/bigcache.go +++ b/dnscheck/cache/bigcache.go @@ -7,16 +7,14 @@ import ( "github.com/allegro/bigcache/v3" ) -const expirationTime = 1 * time.Minute - type BigCache struct { cache *bigcache.BigCache } -// NewBigcache creates a new BigCache instance -func NewBigcache() (*BigCache, error) { +// NewBigcache creates a new BigCache instance with the given entry lifetime. +func NewBigcache(ttl time.Duration) (*BigCache, error) { queriesCache := &BigCache{} - cache, err := bigcache.New(context.Background(), bigcache.DefaultConfig(expirationTime)) + cache, err := bigcache.New(context.Background(), bigcache.DefaultConfig(ttl)) if err != nil { return nil, err } diff --git a/dnscheck/cache/cache.go b/dnscheck/cache/cache.go index 10d1e673..3e577c1c 100644 --- a/dnscheck/cache/cache.go +++ b/dnscheck/cache/cache.go @@ -2,6 +2,7 @@ package cache import ( "errors" + "time" ) const CacheTypeBigCache = "bigcache" @@ -13,11 +14,11 @@ type Cache interface { DeleteQueryData(key string) error } -// New creates a new Cache instance -func New(cacheType string) (Cache, error) { +// New creates a new Cache instance whose entries expire after ttl. +func New(cacheType string, ttl time.Duration) (Cache, error) { switch cacheType { case CacheTypeBigCache: - return NewBigcache() + return NewBigcache(ttl) } return nil, errors.New("unknown cache type") } diff --git a/dnscheck/config/config.go b/dnscheck/config/config.go index 55be33a4..04f7cb85 100644 --- a/dnscheck/config/config.go +++ b/dnscheck/config/config.go @@ -2,11 +2,19 @@ package config import ( "errors" + "fmt" + "net" "os" "strconv" + "strings" "time" ) +// DefaultCacheTTL bounds how long a check record stays in memory. The frontend +// reads it within milliseconds of the DNS query, so this only needs to cover +// resolver retries. +const DefaultCacheTTL = 15 * time.Second + // Config represents the application configuration type Config struct { Server *AuthoritativeDNSServerConfig @@ -20,7 +28,43 @@ type AuthoritativeDNSServerConfig struct { Domain string IPAddress string ASN uint - IPRange string + // IPRanges are the CIDR blocks our resolvers query from; at least one is + // required. PoPs sit in unrelated address blocks, so this is a list. + IPRanges []IPRange +} + +// IPRange is one trusted source block. Label is operator-facing only (the PoP +// name) and plays no part in matching. +type IPRange struct { + Label string + Net *net.IPNet +} + +// String renders "label net" or just "net" when unlabelled. +func (r IPRange) String() string { + if r.Label == "" { + return r.Net.String() + } + return r.Label + " " + r.Net.String() +} + +// ContainsIP reports whether ip falls inside any configured range. +func (c *AuthoritativeDNSServerConfig) ContainsIP(ip net.IP) bool { + for _, r := range c.IPRanges { + if r.Net.Contains(ip) { + return true + } + } + return false +} + +// IPRangesString lists the configured ranges for the startup log. +func (c *AuthoritativeDNSServerConfig) IPRangesString() string { + parts := make([]string, 0, len(c.IPRanges)) + for _, r := range c.IPRanges { + parts = append(parts, r.String()) + } + return strings.Join(parts, ", ") } // APIConfig represents the API configuration @@ -39,31 +83,33 @@ type CacheConfig struct { HMACKey string } -// GeoLookupConfig represents access to MaxMind GeoIP database +// GeoLookupConfig represents access to the MaxMind GeoIP ASN database type GeoLookupConfig struct { - DBFile string DBASNFile string } // IsValid check whether config section is valid func (cfg *GeoLookupConfig) IsValid() error { - if cfg.DBFile == "" { - return errors.New("[GeoIP] DBFile is required") - - } if cfg.DBASNFile == "" { - return errors.New("[GeoIP] DBISP is required") - + return errors.New("GEOIP_DB_ASN_FILE environment variable is required") } return nil } // New creates a new Config instance func New() (*Config, error) { - cacheTTL := os.Getenv("CACHE_TTL") - ttl, err := time.ParseDuration(cacheTTL) + ttl := DefaultCacheTTL + if raw := os.Getenv("CACHE_TTL"); raw != "" { + parsed, err := time.ParseDuration(raw) + if err != nil || parsed <= 0 { + return nil, fmt.Errorf("CACHE_TTL must be a positive duration, got %q", raw) + } + ttl = parsed + } + + ipRanges, err := parseIPRanges(os.Getenv("DNS_AUTH_SERVER_IP_RANGE")) if err != nil { - ttl = 1 * time.Minute + return nil, err } asn := os.Getenv("DNS_AUTH_SERVER_ASN") @@ -77,12 +123,19 @@ func New() (*Config, error) { return nil, errors.New("CACHE_HMAC_KEY environment variable is required") } + geoLookup := &GeoLookupConfig{ + DBASNFile: os.Getenv("GEOIP_DB_ASN_FILE"), + } + if err := geoLookup.IsValid(); err != nil { + return nil, err + } + return &Config{ Server: &AuthoritativeDNSServerConfig{ Domain: os.Getenv("DNS_AUTH_SERVER_DOMAIN"), IPAddress: os.Getenv("DNS_AUTH_SERVER_IP_ADDRESS"), ASN: uint(asnUint), - IPRange: os.Getenv("DNS_AUTH_SERVER_IP_RANGE"), + IPRanges: ipRanges, }, API: &APIConfig{ Port: os.Getenv("API_PORT"), @@ -92,9 +145,36 @@ func New() (*Config, error) { TTL: ttl, HMACKey: cacheHMACKey, }, - GeoLookupConfig: &GeoLookupConfig{ - DBFile: os.Getenv("GEOIP_DB_FILE"), - DBASNFile: os.Getenv("GEOIP_DB_ASN_FILE"), - }, + GeoLookupConfig: geoLookup, }, nil } + +// parseIPRanges parses a comma-separated list of CIDR blocks, each optionally +// prefixed with a label ("tor1=198.51.100.7/32"), the key=value,key=value +// convention used by e.g. docker --label. Every entry must parse and at least +// one is required. +func parseIPRanges(raw string) ([]IPRange, error) { + var ranges []IPRange + for _, part := range strings.Split(raw, ",") { + part = strings.TrimSpace(part) + if part == "" { + continue + } + label, cidr := "", part + if i := strings.Index(part, "="); i >= 0 { + label, cidr = strings.TrimSpace(part[:i]), strings.TrimSpace(part[i+1:]) + if label == "" { + return nil, fmt.Errorf("DNS_AUTH_SERVER_IP_RANGE entry %q has an empty label before '='", part) + } + } + _, n, err := net.ParseCIDR(cidr) + if err != nil { + return nil, fmt.Errorf("DNS_AUTH_SERVER_IP_RANGE entries must be [label=]CIDR (e.g. 10.5.0.0/16 or tor1=198.51.100.7/32), got %q", part) + } + ranges = append(ranges, IPRange{Label: label, Net: n}) + } + if len(ranges) == 0 { + return nil, errors.New("DNS_AUTH_SERVER_IP_RANGE environment variable is required (comma-separated [label=]CIDR list)") + } + return ranges, nil +} diff --git a/dnscheck/config/config_test.go b/dnscheck/config/config_test.go new file mode 100644 index 00000000..57497b48 --- /dev/null +++ b/dnscheck/config/config_test.go @@ -0,0 +1,131 @@ +package config + +import ( + "net" + "testing" + "time" +) + +// specRef: dnscheck-behaviour.md #S2 +func TestNewRequiresASNDatabasePath(t *testing.T) { + t.Setenv("CACHE_HMAC_KEY", "test-key") + t.Setenv("DNS_AUTH_SERVER_IP_RANGE", "10.5.0.0/16") + t.Setenv("GEOIP_DB_ASN_FILE", "") + + if _, err := New(); err == nil { + t.Fatal("expected an error when GEOIP_DB_ASN_FILE is unset") + } + + t.Setenv("GEOIP_DB_ASN_FILE", "/opt/dnscheck/GeoLite2-ASN.mmdb") + cfg, err := New() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if cfg.GeoLookupConfig.DBASNFile != "/opt/dnscheck/GeoLite2-ASN.mmdb" { + t.Errorf("DBASNFile = %q", cfg.GeoLookupConfig.DBASNFile) + } +} + +// specRef: dnscheck-behaviour.md #S4 +func TestNewParsesIPRangeAsCIDR(t *testing.T) { + t.Setenv("CACHE_HMAC_KEY", "test-key") + t.Setenv("GEOIP_DB_ASN_FILE", "/opt/dnscheck/GeoLite2-ASN.mmdb") + + t.Setenv("DNS_AUTH_SERVER_IP_RANGE", "10.5.0.0/16") + cfg, err := New() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(cfg.Server.IPRanges) != 1 || cfg.Server.IPRanges[0].Net.String() != "10.5.0.0/16" || cfg.Server.IPRanges[0].Label != "" { + t.Errorf("IPRanges = %v, want one unlabelled 10.5.0.0/16", cfg.Server.IPRanges) + } + + // PoPs live in unrelated blocks, so a comma-separated list is accepted; + // whitespace and a trailing comma are tolerated. + t.Setenv("DNS_AUTH_SERVER_IP_RANGE", "198.51.100.7/32, 203.0.113.0/24,") + cfg, err = New() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(cfg.Server.IPRanges) != 2 { + t.Fatalf("IPRanges = %v, want two entries", cfg.Server.IPRanges) + } + if !cfg.Server.ContainsIP(net.ParseIP("198.51.100.7")) || !cfg.Server.ContainsIP(net.ParseIP("203.0.113.9")) { + t.Errorf("ContainsIP does not cover both configured ranges: %v", cfg.Server.IPRanges) + } + if cfg.Server.ContainsIP(net.ParseIP("198.51.100.8")) { + t.Errorf("/32 entry must match a single address only") + } + + // One bad entry fails the whole list. + t.Setenv("DNS_AUTH_SERVER_IP_RANGE", "198.51.100.7/32,10.5.") + if _, err := New(); err == nil { + t.Fatal("expected an error when one list entry is not CIDR") + } + + // Entries may carry an operator-facing label; it never affects matching. + t.Setenv("DNS_AUTH_SERVER_IP_RANGE", "tor1=198.51.100.7/32, lab = 203.0.113.0/24,192.0.2.0/24") + cfg, err = New() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got := cfg.Server.IPRangesString(); got != "tor1 198.51.100.7/32, lab 203.0.113.0/24, 192.0.2.0/24" { + t.Errorf("IPRangesString() = %q", got) + } + if !cfg.Server.ContainsIP(net.ParseIP("198.51.100.7")) || !cfg.Server.ContainsIP(net.ParseIP("192.0.2.9")) { + t.Errorf("labelled and unlabelled entries must both match: %v", cfg.Server.IPRanges) + } + + for _, bad := range []string{"=198.51.100.7/32", "tor1=", "tor1=10.5."} { + t.Setenv("DNS_AUTH_SERVER_IP_RANGE", bad) + if _, err := New(); err == nil { + t.Errorf("expected an error for DNS_AUTH_SERVER_IP_RANGE=%q", bad) + } + } + + // The range is what makes a query "ours"; without it every check would + // depend on the ASN alone, so an unset value is a boot error. + t.Setenv("DNS_AUTH_SERVER_IP_RANGE", "") + if _, err := New(); err == nil { + t.Fatal("expected an error when DNS_AUTH_SERVER_IP_RANGE is unset") + } + + // The legacy string-prefix form is rejected so a misconfiguration fails at + // boot instead of silently matching the wrong addresses. + t.Setenv("DNS_AUTH_SERVER_IP_RANGE", "10.5.") + if _, err := New(); err == nil { + t.Fatal("expected an error for a non-CIDR IP range") + } +} + +// specRef: dnscheck-behaviour.md #S5 +func TestNewCacheTTLDefaultsAndValidates(t *testing.T) { + t.Setenv("CACHE_HMAC_KEY", "test-key") + t.Setenv("GEOIP_DB_ASN_FILE", "/opt/dnscheck/GeoLite2-ASN.mmdb") + t.Setenv("DNS_AUTH_SERVER_IP_RANGE", "10.5.0.0/16") + + t.Setenv("CACHE_TTL", "") + cfg, err := New() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if cfg.Cache.TTL != DefaultCacheTTL { + t.Errorf("TTL = %v, want default %v", cfg.Cache.TTL, DefaultCacheTTL) + } + + t.Setenv("CACHE_TTL", "30s") + cfg, err = New() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if cfg.Cache.TTL != 30*time.Second { + t.Errorf("TTL = %v, want 30s", cfg.Cache.TTL) + } + + for _, bad := range []string{"soon", "-5s", "0"} { + t.Setenv("CACHE_TTL", bad) + if _, err := New(); err == nil { + t.Errorf("expected an error for CACHE_TTL=%q", bad) + } + } +} diff --git a/dnscheck/dns/handler.go b/dnscheck/dns/handler.go index f8ab113e..e6921fb3 100644 --- a/dnscheck/dns/handler.go +++ b/dnscheck/dns/handler.go @@ -2,25 +2,29 @@ package dns import ( "encoding/json" - "errors" + "fmt" "net" "regexp" "strings" "time" "github.com/dnscheck/cache" + "github.com/dnscheck/internal/maxmind" "github.com/miekg/dns" "github.com/rs/zerolog/log" ) const ( - // SubdomainRegexPattern validates the expected dnscheck subdomain format: - // 12 alphanumeric chars (nanoid), a dash, then the profile ID. - SubdomainRegexPattern = `^[a-zA-Z0-9]{12}-[a-zA-Z0-9-]+$` + // SubdomainRegexPattern validates the dnscheck probe label: 12 alphanumeric + // chars (nanoid). A "-suffix" is tolerated for clients still running the + // previous frontend bundle, which appended the profile ID. + SubdomainRegexPattern = `^[a-zA-Z0-9]{12}(-[a-zA-Z0-9-]+)?$` ProfileIdAdditionalSectionCode = 0xfeed TTL = 300 ) +var subdomainRegex = regexp.MustCompile(SubdomainRegexPattern) + type Handler struct { srv *DNSServer } @@ -31,7 +35,6 @@ func (h *Handler) ServeDNS(w dns.ResponseWriter, r *dns.Msg) { defer func() { if rec := recover(); rec != nil { log.Error().Interface("panic", rec). - Str("remote", w.RemoteAddr().String()). Msg("Recovered from panic while serving DNS request") } }() @@ -40,8 +43,7 @@ func (h *Handler) ServeDNS(w dns.ResponseWriter, r *dns.Msg) { // body, so a message can declare a question yet carry none. That unpacks // without error, leaving Question empty here. if len(r.Question) == 0 { - log.Debug().Str("remote", w.RemoteAddr().String()). - Msg("Rejecting DNS request with no question section") + log.Debug().Msg("Rejecting DNS request with no question section") m := new(dns.Msg) m.SetRcode(r, dns.RcodeFormatError) if err := w.WriteMsg(m); err != nil { @@ -50,7 +52,7 @@ func (h *Handler) ServeDNS(w dns.ResponseWriter, r *dns.Msg) { return } - log.Debug().Str("protocol", w.RemoteAddr().Network()).Str("qtype", dns.Type(r.Question[0].Qtype).String()).Msgf("Received DNS request: %s", r.Question[0].Name) + log.Debug().Str("protocol", w.RemoteAddr().Network()).Str("qtype", dns.Type(r.Question[0].Qtype).String()).Msg("Received DNS request") msg := dns.Msg{} msg.SetReply(r) @@ -63,47 +65,33 @@ func (h *Handler) ServeDNS(w dns.ResponseWriter, r *dns.Msg) { if strings.Contains(domain, h.srv.Config.Server.Domain) { subdomain := strings.Split(domain, ".")[0] - // Regex to identify the subdomain with the first part being exactly 12 characters - matched, err := regexp.MatchString(SubdomainRegexPattern, subdomain) - if err != nil { - log.Error().Err(err).Msg("Failed to compile regex") - return - } - - if !matched { - log.Warn().Str("subdomain", subdomain).Msg("Unidentified subdomain") + if !subdomainRegex.MatchString(subdomain) { + log.Warn().Msg("Unidentified subdomain") return } record := DNSLogRecord{} - udp := strings.HasPrefix(w.RemoteAddr().Network(), "udp") - var extractionMode string - if udp { - extractionMode = "udp" - } else { - extractionMode = "tcp" - } - IPAddress, _, err := h.extractIPAddressAndHostname(w, extractionMode) + clientAddr, err := clientIP(w.RemoteAddr()) if err != nil { - log.Warn().Err(err).Msgf("Error resolving address %s %s, defaulting to hostname None", w.RemoteAddr().Network(), w.RemoteAddr().String()) + log.Warn().Err(err).Msg("Cannot determine client IP address") return } - lookupData, err := h.srv.GeoLookup.GetGeoLookup(IPAddress) - if err != nil { - log.Error().Err(err).Msgf("Error getting GeoLookup for %s", IPAddress) + // A failed lookup degrades to "no ASN information"; the IP-range check + // below still decides the status and the answer is still written. + lookupData, err := h.srv.GeoLookup.GetGeoLookup(clientAddr.String()) + if err != nil || lookupData == nil { + log.Error().Err(err).Msg("GeoIP lookup failed, continuing without ASN") + lookupData = &maxmind.GeoLookup{} } - record.IPAddress = IPAddress - record.ASN = lookupData.ASN - record.ASNOrganization = lookupData.ASNOrganization - // decide whether IP address or ASN is from modDNS - log.Trace().Bool("isOurIPRange", strings.HasPrefix(IPAddress, h.srv.Config.Server.IPRange)). - Bool("isOurASN", lookupData.ASN == h.srv.Config.Server.ASN). + isOurIPRange := h.srv.Config.Server.ContainsIP(clientAddr) + isOurASN := lookupData.ASN != 0 && lookupData.ASN == h.srv.Config.Server.ASN + log.Trace().Bool("isOurIPRange", isOurIPRange).Bool("isOurASN", isOurASN). Msg("Checking if IP address or ASN is from our range") - if strings.HasPrefix(IPAddress, h.srv.Config.Server.IPRange) || lookupData.ASN == h.srv.Config.Server.ASN { + if isOurIPRange || isOurASN { profileId := h.extractConfiguredProfileId(r) record.Status = StatusConfigured record.ProfileId = profileId @@ -117,9 +105,9 @@ func (h *Handler) ServeDNS(w dns.ResponseWriter, r *dns.Msg) { } cacheKey := cache.HMACKey(h.srv.Config.Cache.HMACKey, subdomain) if err = h.srv.Cache.SaveQueryData(cacheKey, recordBytes); err != nil { - log.Error().Err(err).Str("ID", subdomain).Msg("Failed to save record") + log.Error().Err(err).Msg("Failed to save record") } - log.Debug().Str("ID", subdomain).Msg("Record saved") + log.Debug().Msg("Record saved") } msg.Answer = append(msg.Answer, &dns.A{ @@ -224,33 +212,17 @@ func (h *Handler) createSOA() []dns.RR { } } -func (h *Handler) extractIPAddressAndHostname(w dns.ResponseWriter, extractionMode string) (IPAddress string, hostname string, err error) { - switch extractionMode { - case "udp": - addr, err := net.ResolveUDPAddr(w.RemoteAddr().Network(), w.RemoteAddr().String()) - if err != nil { - return "", "", err - } - IPAddress = addr.IP.String() - hostnames, err := net.LookupAddr(IPAddress) - if err == nil && len(hostnames) > 0 { - hostname = hostnames[0] - } - case "tcp": - addr, err := net.ResolveTCPAddr(w.RemoteAddr().Network(), w.RemoteAddr().String()) - if err != nil { - return "", "", err - } - IPAddress = addr.IP.String() - hostnames, err := net.LookupAddr(IPAddress) - if err == nil && len(hostnames) > 0 { - hostname = hostnames[0] - } +// clientIP returns the transport-level source address of the query. It is read +// straight from the socket address and never resolved. +func clientIP(addr net.Addr) (net.IP, error) { + switch a := addr.(type) { + case *net.UDPAddr: + return a.IP, nil + case *net.TCPAddr: + return a.IP, nil default: - return "", "", errors.New("invalid extraction mode") + return nil, fmt.Errorf("unsupported remote address type %T", addr) } - - return IPAddress, hostname, nil } func FindStringSubmatchMap(rs string, s string) map[string]string { diff --git a/dnscheck/dns/handler_geolookup_test.go b/dnscheck/dns/handler_geolookup_test.go new file mode 100644 index 00000000..293249bc --- /dev/null +++ b/dnscheck/dns/handler_geolookup_test.go @@ -0,0 +1,370 @@ +package dns + +import ( + "bytes" + "encoding/json" + "errors" + "net" + "os" + "strings" + "testing" + + "github.com/dnscheck/config" + "github.com/dnscheck/internal/maxmind" + "github.com/miekg/dns" + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" +) + +const ( + testDomain = "check.example.test" + testSubdomain = "abcdefghijkl" + testOurASN = 64512 +) + +type fakeGeoLookup struct { + result *maxmind.GeoLookup + err error + askedIP string +} + +func (f *fakeGeoLookup) GetGeoLookup(ip string) (*maxmind.GeoLookup, error) { + f.askedIP = ip + return f.result, f.err +} + +type memCache struct { + saved map[string][]byte +} + +func (c *memCache) SaveQueryData(key string, value []byte) error { + if c.saved == nil { + c.saved = map[string][]byte{} + } + c.saved[key] = value + return nil +} +func (c *memCache) GetQueryData(key string) ([]byte, error) { return c.saved[key], nil } +func (c *memCache) DeleteQueryData(key string) error { delete(c.saved, key); return nil } + +// tcpCaptureWriter reports a TCP remote address so both transports are covered. +type tcpCaptureWriter struct{ captureWriter } + +func (w *tcpCaptureWriter) RemoteAddr() net.Addr { + return &net.TCPAddr{IP: net.IPv4(203, 0, 113, 5), Port: 40000} +} +func (w *tcpCaptureWriter) Network() string { return "tcp" } + +// Handler logs are noise in test output; the log-hygiene test re-enables them +// on its own buffer. +func TestMain(m *testing.M) { + zerolog.SetGlobalLevel(zerolog.Disabled) + os.Exit(m.Run()) +} + +func mustCIDR(t *testing.T, s string) *net.IPNet { + t.Helper() + _, n, err := net.ParseCIDR(s) + if err != nil { + t.Fatal(err) + } + return n +} + +func newTestHandler(t *testing.T, geo GeoLookuper, cache *memCache) *Handler { + return &Handler{srv: &DNSServer{ + Config: &config.Config{ + Server: &config.AuthoritativeDNSServerConfig{ + Domain: testDomain, + IPAddress: "192.0.2.1", + ASN: testOurASN, + IPRanges: []config.IPRange{ + {Label: "lab", Net: mustCIDR(t, "198.51.100.0/24")}, + {Net: mustCIDR(t, "192.0.2.77/32")}, + }, + }, + Cache: &config.CacheConfig{HMACKey: "test-key"}, + }, + Cache: cache, + GeoLookup: geo, + }} +} + +func checkQuery() *dns.Msg { + req := new(dns.Msg) + req.SetQuestion(testSubdomain+"."+testDomain+".", dns.TypeA) + return req +} + +func savedRecord(t *testing.T, c *memCache) DNSLogRecord { + t.Helper() + if len(c.saved) != 1 { + t.Fatalf("expected exactly one saved record, got %d", len(c.saved)) + } + var rec DNSLogRecord + for _, raw := range c.saved { + if err := json.Unmarshal(raw, &rec); err != nil { + t.Fatalf("saved record is not JSON: %v", err) + } + // Data minimisation: the stored record must hold nothing beyond what the + // HTTP side returns. + var keys map[string]json.RawMessage + if err := json.Unmarshal(raw, &keys); err != nil { + t.Fatal(err) + } + for k := range keys { + if k != "status" && k != "profile_id" { + t.Errorf("stored record carries unexpected field %q: %s", k, raw) + } + } + } + return rec +} + +// A failing GeoIP lookup must degrade to "no ASN information", not crash or go +// silent: the A answer is still written and the record is saved with status +// derived from the IP range alone. +// +// specRef: dnscheck-behaviour.md #D6 +func TestServeDNSDegradesWhenGeoLookupFails(t *testing.T) { + cache := &memCache{} + h := newTestHandler(t, &fakeGeoLookup{err: errors.New("lookup failed")}, cache) + w := &captureWriter{} + + defer func() { + if r := recover(); r != nil { + t.Fatalf("ServeDNS panicked on a failed GeoIP lookup: %v", r) + } + }() + + h.ServeDNS(w, checkQuery()) + + if w.msg == nil || len(w.msg.Answer) != 1 { + t.Fatalf("expected an A answer despite the lookup failure, got %+v", w.msg) + } + rec := savedRecord(t, cache) + if rec.Status != StatusUnconfigured { + t.Errorf("status = %q, want %q", rec.Status, StatusUnconfigured) + } +} + +// With no ASN information the IP-range check alone can still mark the query +// as ours. +// +// specRef: dnscheck-behaviour.md #D6, #D7 +func TestServeDNSFallsBackToIPRangeWhenGeoLookupFails(t *testing.T) { + cache := &memCache{} + h := newTestHandler(t, &fakeGeoLookup{err: errors.New("lookup failed")}, cache) + w := &rangeCaptureWriter{} + + h.ServeDNS(w, checkQuery()) + + if rec := savedRecord(t, cache); rec.Status != StatusConfigured { + t.Errorf("status = %q, want %q for an in-range client", rec.Status, StatusConfigured) + } +} + +// rangeCaptureWriter reports a client inside DNS_AUTH_SERVER_IP_RANGE. +type rangeCaptureWriter struct{ captureWriter } + +func (w *rangeCaptureWriter) RemoteAddr() net.Addr { + return &net.UDPAddr{IP: net.IPv4(198, 51, 100, 7), Port: 40000} +} + +// Any configured range qualifies, including a single-address /32. +// +// specRef: dnscheck-behaviour.md #D7 +func TestServeDNSMatchesAnyConfiguredRange(t *testing.T) { + cache := &memCache{} + h := newTestHandler(t, &fakeGeoLookup{result: &maxmind.GeoLookup{}}, cache) + + h.ServeDNS(&singleAddrCaptureWriter{}, checkQuery()) + + if rec := savedRecord(t, cache); rec.Status != StatusConfigured { + t.Errorf("status = %q, want %q for a client matching the second range", rec.Status, StatusConfigured) + } +} + +type singleAddrCaptureWriter struct{ captureWriter } + +func (w *singleAddrCaptureWriter) RemoteAddr() net.Addr { + return &net.UDPAddr{IP: net.IPv4(192, 0, 2, 77), Port: 40000} +} + +// A CIDR range must not match by string prefix: 198.51.100.0/24 is not +// 198.51.10.x. +// +// specRef: dnscheck-behaviour.md #D7 +func TestServeDNSIPRangeIsCIDRNotStringPrefix(t *testing.T) { + cache := &memCache{} + h := newTestHandler(t, &fakeGeoLookup{result: &maxmind.GeoLookup{}}, cache) + w := &nearMissCaptureWriter{} + + h.ServeDNS(w, checkQuery()) + + if rec := savedRecord(t, cache); rec.Status != StatusUnconfigured { + t.Errorf("status = %q, want %q for an out-of-range client", rec.Status, StatusUnconfigured) + } +} + +type nearMissCaptureWriter struct{ captureWriter } + +func (w *nearMissCaptureWriter) RemoteAddr() net.Addr { + return &net.UDPAddr{IP: net.IPv4(198, 51, 10, 7), Port: 40000} +} + +// specRef: dnscheck-behaviour.md #D7 +func TestServeDNSMarksConfiguredWhenASNMatches(t *testing.T) { + cache := &memCache{} + h := newTestHandler(t, &fakeGeoLookup{result: &maxmind.GeoLookup{ + IPAddress: "203.0.113.5", ASN: testOurASN, ASNOrganization: "OURS", + }}, cache) + + req := checkQuery() + opt := &dns.OPT{Hdr: dns.RR_Header{Name: ".", Rrtype: dns.TypeOPT}} + opt.Option = append(opt.Option, &dns.EDNS0_LOCAL{Code: ProfileIdAdditionalSectionCode, Data: []byte("profile1")}) + req.Extra = append(req.Extra, opt) + + h.ServeDNS(&captureWriter{}, req) + + rec := savedRecord(t, cache) + if rec.Status != StatusConfigured { + t.Errorf("status = %q, want %q", rec.Status, StatusConfigured) + } + if rec.ProfileId != "profile1" { + t.Errorf("profile_id = %q, want profile1", rec.ProfileId) + } +} + +// specRef: dnscheck-behaviour.md #D8 +func TestServeDNSMarksUnconfiguredWhenNeitherASNNorRangeMatch(t *testing.T) { + cache := &memCache{} + h := newTestHandler(t, &fakeGeoLookup{result: &maxmind.GeoLookup{ + IPAddress: "203.0.113.5", ASN: 15169, ASNOrganization: "GOOGLE", + }}, cache) + + h.ServeDNS(&captureWriter{}, checkQuery()) + + if rec := savedRecord(t, cache); rec.Status != StatusUnconfigured { + t.Errorf("status = %q, want %q", rec.Status, StatusUnconfigured) + } +} + +// The remote address is taken from the transport as-is; it is never resolved. +// +// specRef: dnscheck-behaviour.md #D4 +func TestServeDNSExtractsClientIPOverTCP(t *testing.T) { + cache := &memCache{} + geo := &fakeGeoLookup{result: &maxmind.GeoLookup{}} + h := newTestHandler(t, geo, cache) + + h.ServeDNS(&tcpCaptureWriter{}, checkQuery()) + + savedRecord(t, cache) + if geo.askedIP != "203.0.113.5" { + t.Errorf("looked up %q, want the TCP socket address 203.0.113.5", geo.askedIP) + } +} + +// Nothing that identifies the client or the probe may reach the logs, at any +// level: not the source address, the query name, the subdomain, or the profile +// ID. +// +// specRef: dnscheck-behaviour.md #D9 +func TestServeDNSLogsCarryNoClientIdentifiers(t *testing.T) { + var buf bytes.Buffer + prev := log.Logger + prevLevel := zerolog.GlobalLevel() + log.Logger = zerolog.New(&buf) + zerolog.SetGlobalLevel(zerolog.TraceLevel) + t.Cleanup(func() { log.Logger = prev; zerolog.SetGlobalLevel(prevLevel) }) + + cache := &memCache{} + h := newTestHandler(t, &fakeGeoLookup{err: errors.New("lookup failed")}, cache) + req := checkQuery() + opt := &dns.OPT{Hdr: dns.RR_Header{Name: ".", Rrtype: dns.TypeOPT}} + opt.Option = append(opt.Option, &dns.EDNS0_LOCAL{Code: ProfileIdAdditionalSectionCode, Data: []byte("profile1")}) + req.Extra = append(req.Extra, opt) + h.ServeDNS(&captureWriter{}, req) + + // Also drive the malformed-subdomain and no-question paths. + bad := new(dns.Msg) + bad.SetQuestion("short-profile1."+testDomain+".", dns.TypeA) + h.ServeDNS(&captureWriter{}, bad) + h.ServeDNS(&captureWriter{}, new(dns.Msg)) + + out := buf.String() + if out == "" { + t.Fatal("expected some log output at trace level") + } + for _, secret := range []string{"203.0.113.5", testSubdomain, "profile1", "short-profile1"} { + if strings.Contains(out, secret) { + t.Errorf("log output contains %q:\n%s", secret, out) + } + } +} + +// specRef: dnscheck-behaviour.md #D3 +func TestServeDNSProbeLabelFormat(t *testing.T) { + cases := []struct { + label string + saved bool + }{ + {"abcdefghijkl", true}, // current frontend: bare nanoid + {"ABCdef123456", true}, // mixed alphabet + {"abcdefghijkl-profile1", true}, // previous frontend bundle: tolerated + {"abcdefghijk", false}, // 11 chars + {"abcdefghijklm", false}, // 13 chars + {"abcdefghijkl-", false}, // dangling separator + {"abcdefghij_l", false}, // non-alphanumeric + } + for _, tc := range cases { + cache := &memCache{} + h := newTestHandler(t, &fakeGeoLookup{result: &maxmind.GeoLookup{}}, cache) + req := new(dns.Msg) + req.SetQuestion(tc.label+"."+testDomain+".", dns.TypeA) + h.ServeDNS(&captureWriter{}, req) + if got := len(cache.saved) == 1; got != tc.saved { + t.Errorf("label %q: record saved = %v, want %v", tc.label, got, tc.saved) + } + } +} + +// A queries outside the check domain get the authoritative A answer and leave no +// trace: no lookup, no cache entry. +// +// specRef: dnscheck-behaviour.md #D2 +func TestServeDNSAnswersForeignDomainWithoutRecord(t *testing.T) { + cache := &memCache{} + h := newTestHandler(t, &fakeGeoLookup{err: errors.New("must not be called")}, cache) + w := &captureWriter{} + + req := new(dns.Msg) + req.SetQuestion("www.other.example.", dns.TypeA) + h.ServeDNS(w, req) + + if w.msg == nil || len(w.msg.Answer) != 1 { + t.Fatalf("expected an A answer, got %+v", w.msg) + } + if len(cache.saved) != 0 { + t.Errorf("expected no saved record, got %d", len(cache.saved)) + } +} + +// specRef: dnscheck-behaviour.md #D3 +func TestServeDNSIgnoresMalformedSubdomain(t *testing.T) { + cache := &memCache{} + h := newTestHandler(t, &fakeGeoLookup{err: errors.New("must not be called")}, cache) + w := &captureWriter{} + + req := new(dns.Msg) + req.SetQuestion("short-profile1."+testDomain+".", dns.TypeA) + h.ServeDNS(w, req) + + if len(cache.saved) != 0 { + t.Errorf("expected no saved record for a malformed subdomain, got %d", len(cache.saved)) + } + if w.msg != nil { + t.Errorf("expected no response for a malformed subdomain, got %+v", w.msg) + } +} diff --git a/dnscheck/dns/handler_malformed_test.go b/dnscheck/dns/handler_malformed_test.go index edb392e0..ce09bd9d 100644 --- a/dnscheck/dns/handler_malformed_test.go +++ b/dnscheck/dns/handler_malformed_test.go @@ -35,6 +35,8 @@ func (w *captureWriter) Network() string { return "udp" } // nil Question slice. Indexing Question[0] then panics, and the handler runs in a // goroutine per packet, so the panic would terminate the process -- taking the DNS // listener and the HTTP API sharing it down together. +// +// specRef: dnscheck-behaviour.md #D1 func TestServeDNSHandlesMissingQuestionSection(t *testing.T) { // 12-byte header, QDCOUNT=1, everything else zero. Passes the library's // length and accept checks. @@ -79,6 +81,8 @@ func TestServeDNSHandlesMissingQuestionSection(t *testing.T) { } // QDCOUNT=0 with no question: same code path, pinned for completeness. +// +// specRef: dnscheck-behaviour.md #D1 func TestServeDNSHandlesZeroQuestionCount(t *testing.T) { req := new(dns.Msg) req.Id = 1234 diff --git a/dnscheck/dns/models.go b/dnscheck/dns/models.go index 979d0188..60755332 100644 --- a/dnscheck/dns/models.go +++ b/dnscheck/dns/models.go @@ -5,12 +5,11 @@ const ( StatusUnconfigured = "unconfigured" ) +// DNSLogRecord is what the DNS side stores for the HTTP side to read. The +// client IP and ASN only ever feed the status decision and are not kept. type DNSLogRecord struct { - Status string `json:"status"` - ProfileId string `json:"profile_id"` - IPAddress string `json:"ip_address"` - ASN uint `json:"asn"` - ASNOrganization string `json:"asn_organization"` + Status string `json:"status"` + ProfileId string `json:"profile_id"` } // DNSCheckResponse is the minimal HTTP response payload. diff --git a/dnscheck/dns/server.go b/dnscheck/dns/server.go index 230693c4..f361c1b8 100644 --- a/dnscheck/dns/server.go +++ b/dnscheck/dns/server.go @@ -1,12 +1,19 @@ package dns import ( + "fmt" + "github.com/dnscheck/cache" "github.com/dnscheck/config" "github.com/dnscheck/internal/maxmind" "github.com/miekg/dns" ) +// GeoLookuper resolves a client IP to its ASN record. +type GeoLookuper interface { + GetGeoLookup(ip string) (*maxmind.GeoLookup, error) +} + // DNSServer represents a DNS server type DNSServer struct { Config *config.Config @@ -15,7 +22,7 @@ type DNSServer struct { DNSTCP *dns.Server Cache cache.Cache - GeoLookup *maxmind.GeoLookupManager + GeoLookup GeoLookuper } // New creates a new DNS server @@ -25,7 +32,11 @@ func New(config *config.Config, cache cache.Cache) (*DNSServer, error) { Cache: cache, } - srv.GeoLookup = maxmind.NewGeoLookupManager(config.GeoLookupConfig.DBFile, config.GeoLookupConfig.DBASNFile) + geoLookup, err := maxmind.NewGeoLookupManager(config.GeoLookupConfig.DBASNFile) + if err != nil { + return nil, fmt.Errorf("geoip: %w", err) + } + srv.GeoLookup = geoLookup // DNS srv.DNSTCP = &dns.Server{Addr: ":53", Net: "tcp"} diff --git a/dnscheck/internal/maxmind/country.go b/dnscheck/internal/maxmind/country.go deleted file mode 100644 index b96b1ef3..00000000 --- a/dnscheck/internal/maxmind/country.go +++ /dev/null @@ -1,277 +0,0 @@ -package maxmind - -type Country struct { - Code string - Name string -} - -func GetCountries() []Country { - return []Country{ - {Code: "A1", Name: "Anonymous Proxy"}, - {Code: "A2", Name: "Satellite Provider"}, - {Code: "O1", Name: "Other Country"}, - {Code: "AD", Name: "Andorra"}, - {Code: "AE", Name: "United Arab Emirates"}, - {Code: "AF", Name: "Afghanistan"}, - {Code: "AG", Name: "Antigua and Barbuda"}, - {Code: "AI", Name: "Anguilla"}, - {Code: "AL", Name: "Albania"}, - {Code: "AM", Name: "Armenia"}, - {Code: "AO", Name: "Angola"}, - {Code: "AP", Name: "Asia/Pacific Region"}, - {Code: "AQ", Name: "Antarctica"}, - {Code: "AR", Name: "Argentina"}, - {Code: "AS", Name: "American Samoa"}, - {Code: "AT", Name: "Austria"}, - {Code: "AU", Name: "Australia"}, - {Code: "AW", Name: "Aruba"}, - {Code: "AX", Name: "Aland Islands"}, - {Code: "AZ", Name: "Azerbaijan"}, - {Code: "BA", Name: "Bosnia and Herzegovina"}, - {Code: "BB", Name: "Barbados"}, - {Code: "BD", Name: "Bangladesh"}, - {Code: "BE", Name: "Belgium"}, - {Code: "BF", Name: "Burkina Faso"}, - {Code: "BG", Name: "Bulgaria"}, - {Code: "BH", Name: "Bahrain"}, - {Code: "BI", Name: "Burundi"}, - {Code: "BJ", Name: "Benin"}, - {Code: "BL", Name: "Saint Barthelemy"}, - {Code: "BM", Name: "Bermuda"}, - {Code: "BN", Name: "Brunei Darussalam"}, - {Code: "BO", Name: "Bolivia"}, - {Code: "BQ", Name: "Bonaire, Saint Eustatius and Saba"}, - {Code: "BR", Name: "Brazil"}, - {Code: "BS", Name: "Bahamas"}, - {Code: "BT", Name: "Bhutan"}, - {Code: "BV", Name: "Bouvet Island"}, - {Code: "BW", Name: "Botswana"}, - {Code: "BY", Name: "Belarus"}, - {Code: "BZ", Name: "Belize"}, - {Code: "CA", Name: "Canada"}, - {Code: "CC", Name: "Cocos (Keeling) Islands"}, - {Code: "CD", Name: "Congo, The Democratic Republic of the"}, - {Code: "CF", Name: "Central African Republic"}, - {Code: "CG", Name: "Congo"}, - {Code: "CH", Name: "Switzerland"}, - {Code: "CI", Name: "Cote d'Ivoire"}, - {Code: "CK", Name: "Cook Islands"}, - {Code: "CL", Name: "Chile"}, - {Code: "CM", Name: "Cameroon"}, - {Code: "CN", Name: "China"}, - {Code: "CO", Name: "Colombia"}, - {Code: "CR", Name: "Costa Rica"}, - {Code: "CU", Name: "Cuba"}, - {Code: "CV", Name: "Cape Verde"}, - {Code: "CW", Name: "Curacao"}, - {Code: "CX", Name: "Christmas Island"}, - {Code: "CY", Name: "Cyprus"}, - {Code: "CZ", Name: "Czech Republic"}, - {Code: "DE", Name: "Germany"}, - {Code: "DJ", Name: "Djibouti"}, - {Code: "DK", Name: "Denmark"}, - {Code: "DM", Name: "Dominica"}, - {Code: "DO", Name: "Dominican Republic"}, - {Code: "DZ", Name: "Algeria"}, - {Code: "EC", Name: "Ecuador"}, - {Code: "EE", Name: "Estonia"}, - {Code: "EG", Name: "Egypt"}, - {Code: "EH", Name: "Western Sahara"}, - {Code: "ER", Name: "Eritrea"}, - {Code: "ES", Name: "Spain"}, - {Code: "ET", Name: "Ethiopia"}, - {Code: "EU", Name: "Europe"}, - {Code: "FI", Name: "Finland"}, - {Code: "FJ", Name: "Fiji"}, - {Code: "FK", Name: "Falkland Islands (Malvinas)"}, - {Code: "FM", Name: "Micronesia, Federated States of"}, - {Code: "FO", Name: "Faroe Islands"}, - {Code: "FR", Name: "France"}, - {Code: "GA", Name: "Gabon"}, - {Code: "GB", Name: "United Kingdom"}, - {Code: "GD", Name: "Grenada"}, - {Code: "GE", Name: "Georgia"}, - {Code: "GF", Name: "French Guiana"}, - {Code: "GG", Name: "Guernsey"}, - {Code: "GH", Name: "Ghana"}, - {Code: "GI", Name: "Gibraltar"}, - {Code: "GL", Name: "Greenland"}, - {Code: "GM", Name: "Gambia"}, - {Code: "GN", Name: "Guinea"}, - {Code: "GP", Name: "Guadeloupe"}, - {Code: "GQ", Name: "Equatorial Guinea"}, - {Code: "GR", Name: "Greece"}, - {Code: "GS", Name: "South Georgia and the South Sandwich Islands"}, - {Code: "GT", Name: "Guatemala"}, - {Code: "GU", Name: "Guam"}, - {Code: "GW", Name: "Guinea-Bissau"}, - {Code: "GY", Name: "Guyana"}, - {Code: "HK", Name: "Hong Kong"}, - {Code: "HM", Name: "Heard Island and McDonald Islands"}, - {Code: "HN", Name: "Honduras"}, - {Code: "HR", Name: "Croatia"}, - {Code: "HT", Name: "Haiti"}, - {Code: "HU", Name: "Hungary"}, - {Code: "ID", Name: "Indonesia"}, - {Code: "IE", Name: "Ireland"}, - {Code: "IL", Name: "Israel"}, - {Code: "IM", Name: "Isle of Man"}, - {Code: "IN", Name: "India"}, - {Code: "IO", Name: "British Indian Ocean Territory"}, - {Code: "IQ", Name: "Iraq"}, - {Code: "IR", Name: "Iran, Islamic Republic of"}, - {Code: "IS", Name: "Iceland"}, - {Code: "IT", Name: "Italy"}, - {Code: "JE", Name: "Jersey"}, - {Code: "JM", Name: "Jamaica"}, - {Code: "JO", Name: "Jordan"}, - {Code: "JP", Name: "Japan"}, - {Code: "KE", Name: "Kenya"}, - {Code: "KG", Name: "Kyrgyzstan"}, - {Code: "KH", Name: "Cambodia"}, - {Code: "KI", Name: "Kiribati"}, - {Code: "KM", Name: "Comoros"}, - {Code: "KN", Name: "Saint Kitts and Nevis"}, - {Code: "KP", Name: "Korea, Democratic People's Republic of"}, - {Code: "KR", Name: "Korea, Republic of"}, - {Code: "KW", Name: "Kuwait"}, - {Code: "KY", Name: "Cayman Islands"}, - {Code: "KZ", Name: "Kazakhstan"}, - {Code: "LA", Name: "Lao People's Democratic Republic"}, - {Code: "LB", Name: "Lebanon"}, - {Code: "LC", Name: "Saint Lucia"}, - {Code: "LI", Name: "Liechtenstein"}, - {Code: "LK", Name: "Sri Lanka"}, - {Code: "LR", Name: "Liberia"}, - {Code: "LS", Name: "Lesotho"}, - {Code: "LT", Name: "Lithuania"}, - {Code: "LU", Name: "Luxembourg"}, - {Code: "LV", Name: "Latvia"}, - {Code: "LY", Name: "Libyan Arab Jamahiriya"}, - {Code: "MA", Name: "Morocco"}, - {Code: "MC", Name: "Monaco"}, - {Code: "MD", Name: "Moldova, Republic of"}, - {Code: "ME", Name: "Montenegro"}, - {Code: "MF", Name: "Saint Martin"}, - {Code: "MG", Name: "Madagascar"}, - {Code: "MH", Name: "Marshall Islands"}, - {Code: "MK", Name: "Macedonia"}, - {Code: "ML", Name: "Mali"}, - {Code: "MM", Name: "Myanmar"}, - {Code: "MN", Name: "Mongolia"}, - {Code: "MO", Name: "Macao"}, - {Code: "MP", Name: "Northern Mariana Islands"}, - {Code: "MQ", Name: "Martinique"}, - {Code: "MR", Name: "Mauritania"}, - {Code: "MS", Name: "Montserrat"}, - {Code: "MT", Name: "Malta"}, - {Code: "MU", Name: "Mauritius"}, - {Code: "MV", Name: "Maldives"}, - {Code: "MW", Name: "Malawi"}, - {Code: "MX", Name: "Mexico"}, - {Code: "MY", Name: "Malaysia"}, - {Code: "MZ", Name: "Mozambique"}, - {Code: "NA", Name: "Namibia"}, - {Code: "NC", Name: "New Caledonia"}, - {Code: "NE", Name: "Niger"}, - {Code: "NF", Name: "Norfolk Island"}, - {Code: "NG", Name: "Nigeria"}, - {Code: "NI", Name: "Nicaragua"}, - {Code: "NL", Name: "Netherlands"}, - {Code: "NO", Name: "Norway"}, - {Code: "NP", Name: "Nepal"}, - {Code: "NR", Name: "Nauru"}, - {Code: "NU", Name: "Niue"}, - {Code: "NZ", Name: "New Zealand"}, - {Code: "OM", Name: "Oman"}, - {Code: "PA", Name: "Panama"}, - {Code: "PE", Name: "Peru"}, - {Code: "PF", Name: "French Polynesia"}, - {Code: "PG", Name: "Papua New Guinea"}, - {Code: "PH", Name: "Philippines"}, - {Code: "PK", Name: "Pakistan"}, - {Code: "PL", Name: "Poland"}, - {Code: "PM", Name: "Saint Pierre and Miquelon"}, - {Code: "PN", Name: "Pitcairn"}, - {Code: "PR", Name: "Puerto Rico"}, - {Code: "PS", Name: "Palestinian Territory"}, - {Code: "PT", Name: "Portugal"}, - {Code: "PW", Name: "Palau"}, - {Code: "PY", Name: "Paraguay"}, - {Code: "QA", Name: "Qatar"}, - {Code: "RE", Name: "Reunion"}, - {Code: "RO", Name: "Romania"}, - {Code: "RS", Name: "Serbia"}, - {Code: "RU", Name: "Russian Federation"}, - {Code: "RW", Name: "Rwanda"}, - {Code: "SA", Name: "Saudi Arabia"}, - {Code: "SB", Name: "Solomon Islands"}, - {Code: "SC", Name: "Seychelles"}, - {Code: "SD", Name: "Sudan"}, - {Code: "SE", Name: "Sweden"}, - {Code: "SG", Name: "Singapore"}, - {Code: "SH", Name: "Saint Helena"}, - {Code: "SI", Name: "Slovenia"}, - {Code: "SJ", Name: "Svalbard and Jan Mayen"}, - {Code: "SK", Name: "Slovakia"}, - {Code: "SL", Name: "Sierra Leone"}, - {Code: "SM", Name: "San Marino"}, - {Code: "SN", Name: "Senegal"}, - {Code: "SO", Name: "Somalia"}, - {Code: "SR", Name: "Suriname"}, - {Code: "SS", Name: "South Sudan"}, - {Code: "ST", Name: "Sao Tome and Principe"}, - {Code: "SV", Name: "El Salvador"}, - {Code: "SX", Name: "Sint Maarten"}, - {Code: "SY", Name: "Syrian Arab Republic"}, - {Code: "SZ", Name: "Swaziland"}, - {Code: "TC", Name: "Turks and Caicos Islands"}, - {Code: "TD", Name: "Chad"}, - {Code: "TF", Name: "French Southern Territories"}, - {Code: "TG", Name: "Togo"}, - {Code: "TH", Name: "Thailand"}, - {Code: "TJ", Name: "Tajikistan"}, - {Code: "TK", Name: "Tokelau"}, - {Code: "TL", Name: "Timor-Leste"}, - {Code: "TM", Name: "Turkmenistan"}, - {Code: "TN", Name: "Tunisia"}, - {Code: "TO", Name: "Tonga"}, - {Code: "TR", Name: "Turkey"}, - {Code: "TT", Name: "Trinidad and Tobago"}, - {Code: "TV", Name: "Tuvalu"}, - {Code: "TW", Name: "Taiwan"}, - {Code: "TZ", Name: "Tanzania, United Republic of"}, - {Code: "UA", Name: "Ukraine"}, - {Code: "UG", Name: "Uganda"}, - {Code: "UM", Name: "United States Minor Outlying Islands"}, - {Code: "US", Name: "United States"}, - {Code: "UY", Name: "Uruguay"}, - {Code: "UZ", Name: "Uzbekistan"}, - {Code: "VA", Name: "Holy See (Vatican City State)"}, - {Code: "VC", Name: "Saint Vincent and the Grenadines"}, - {Code: "VE", Name: "Venezuela"}, - {Code: "VG", Name: "Virgin Islands, British"}, - {Code: "VI", Name: "Virgin Islands, U.S."}, - {Code: "VN", Name: "Vietnam"}, - {Code: "VU", Name: "Vanuatu"}, - {Code: "WF", Name: "Wallis and Futuna"}, - {Code: "WS", Name: "Samoa"}, - {Code: "YE", Name: "Yemen"}, - {Code: "YT", Name: "Mayotte"}, - {Code: "ZA", Name: "South Africa"}, - {Code: "ZM", Name: "Zambia"}, - {Code: "ZW", Name: "Zimbabwe"}, - } -} - -func GetCountryByCode(countryCode string) string { - for _, country := range GetCountries() { - if country.Code == countryCode { - return country.Name - } - } - - // @TODO Notify Sentry - - return "Unknown" -} diff --git a/dnscheck/internal/maxmind/maxmind.go b/dnscheck/internal/maxmind/maxmind.go index ce6fdd31..ad2b0fb1 100644 --- a/dnscheck/internal/maxmind/maxmind.go +++ b/dnscheck/internal/maxmind/maxmind.go @@ -4,40 +4,52 @@ import ( "fmt" "net" - "github.com/rs/zerolog/log" - "github.com/oschwald/geoip2-golang" ) +// GeoLookupManager answers ASN lookups from a MaxMind database that is opened +// once and shared by every request; geoip2.Reader is safe for concurrent use. type GeoLookupManager struct { - DBFile string - DBASNFile string + asnDB *geoip2.Reader } -func NewGeoLookupManager(dbFile, dbASNFile string) *GeoLookupManager { - return &GeoLookupManager{ - DBFile: dbFile, - DBASNFile: dbASNFile, +// NewGeoLookupManager opens the ASN database and fails if the file is missing, +// unreadable or not an ASN-capable database type. +func NewGeoLookupManager(dbASNFile string) (*GeoLookupManager, error) { + asnDB, err := geoip2.Open(dbASNFile) + if err != nil { + return nil, fmt.Errorf("cannot open geoip ASN database %q: %w", dbASNFile, err) } + + // geoip2 only reports a database/method mismatch at lookup time, so probe + // once here rather than on every request. + if _, err := asnDB.ASN(net.IPv4(192, 0, 2, 1)); err != nil { + asnDB.Close() + return nil, fmt.Errorf("geoip database %q does not support ASN lookups: %w", dbASNFile, err) + } + + return &GeoLookupManager{asnDB: asnDB}, nil } +// Close releases the underlying database. +func (g *GeoLookupManager) Close() error { + return g.asnDB.Close() +} + +// GetGeoLookup returns the ASN record for ip. An address that is not in the +// database yields an empty record and no error. func (g *GeoLookupManager) GetGeoLookup(ip string) (*GeoLookup, error) { ipnet := net.ParseIP(ip) - ipDB, err := geoip2.Open(g.DBFile) - if err != nil { - return nil, fmt.Errorf("cannot open geoip database: %v", err) + if ipnet == nil { + return nil, fmt.Errorf("invalid IP address %q", ip) } - defer ipDB.Close() - ispDB, err := geoip2.Open(g.DBASNFile) + asn, err := g.asnDB.ASN(ipnet) if err != nil { - return nil, fmt.Errorf("cannot open geoip ISP database: %v", err) + return nil, fmt.Errorf("cannot get ASN: %w", err) } - defer ispDB.Close() - - asn, err := ispDB.ASN(ipnet) - if err != nil { - log.Error().Err(err).Msg("cannot get ASN") + if asn == nil { + asn = &geoip2.ASN{} } return &GeoLookup{ diff --git a/dnscheck/internal/maxmind/maxmind_test.go b/dnscheck/internal/maxmind/maxmind_test.go new file mode 100644 index 00000000..ac53c442 --- /dev/null +++ b/dnscheck/internal/maxmind/maxmind_test.go @@ -0,0 +1,124 @@ +package maxmind + +import ( + "os" + "path/filepath" + "testing" +) + +const ( + asnFixture = "testdata/GeoLite2-ASN.mmdb" + cityFixture = "testdata/GeoLite2-City.mmdb" +) + +// specRef: dnscheck-behaviour.md #S2 +func TestNewGeoLookupManagerRejectsMissingFile(t *testing.T) { + _, err := NewGeoLookupManager(filepath.Join(t.TempDir(), "missing.mmdb")) + if err == nil { + t.Fatal("expected an error for a missing database file") + } +} + +// specRef: dnscheck-behaviour.md #S2 +func TestNewGeoLookupManagerRejectsCorruptFile(t *testing.T) { + path := filepath.Join(t.TempDir(), "garbage.mmdb") + if err := os.WriteFile(path, []byte("this is not an mmdb file"), 0o600); err != nil { + t.Fatal(err) + } + _, err := NewGeoLookupManager(path) + if err == nil { + t.Fatal("expected an error for a corrupt database file") + } +} + +// specRef: dnscheck-behaviour.md #S3 +func TestNewGeoLookupManagerRejectsNonASNDatabase(t *testing.T) { + _, err := NewGeoLookupManager(cityFixture) + if err == nil { + t.Fatal("expected an error when the ASN path points at a City database") + } +} + +// specRef: dnscheck-behaviour.md #D5 +func TestGetGeoLookupReturnsASNForKnownIP(t *testing.T) { + g, err := NewGeoLookupManager(asnFixture) + if err != nil { + t.Fatalf("open fixture: %v", err) + } + defer g.Close() + + got, err := g.GetGeoLookup("8.8.8.8") + if err != nil { + t.Fatalf("lookup: %v", err) + } + if got.ASN != 15169 || got.ASNOrganization != "GOOGLE" { + t.Errorf("got ASN=%d org=%q, want 15169 GOOGLE", got.ASN, got.ASNOrganization) + } + if got.IPAddress != "8.8.8.8" { + t.Errorf("got IPAddress=%q, want 8.8.8.8", got.IPAddress) + } +} + +// An address absent from the database is not an error: the record is empty and +// the caller falls back to its IP-range check. +// +// specRef: dnscheck-behaviour.md #D6 +func TestGetGeoLookupUnknownIPYieldsEmptyRecord(t *testing.T) { + g, err := NewGeoLookupManager(asnFixture) + if err != nil { + t.Fatalf("open fixture: %v", err) + } + defer g.Close() + + got, err := g.GetGeoLookup("203.0.113.5") + if err != nil { + t.Fatalf("lookup: %v", err) + } + if got == nil { + t.Fatal("got nil record for an unknown IP, want an empty record") + } + if got.ASN != 0 || got.ASNOrganization != "" { + t.Errorf("got ASN=%d org=%q, want empty record", got.ASN, got.ASNOrganization) + } +} + +// specRef: dnscheck-behaviour.md #D6 +func TestGetGeoLookupRejectsUnparseableIP(t *testing.T) { + g, err := NewGeoLookupManager(asnFixture) + if err != nil { + t.Fatalf("open fixture: %v", err) + } + defer g.Close() + + got, err := g.GetGeoLookup("not-an-ip") + if err == nil { + t.Fatal("expected an error for an unparseable IP") + } + if got != nil { + t.Errorf("expected a nil record alongside the error, got %+v", got) + } +} + +// Readers are opened once at startup and shared by every request goroutine. +// +// specRef: dnscheck-behaviour.md #S1 +func TestGetGeoLookupIsSafeForConcurrentUse(t *testing.T) { + g, err := NewGeoLookupManager(asnFixture) + if err != nil { + t.Fatalf("open fixture: %v", err) + } + defer g.Close() + + done := make(chan error, 32) + for i := 0; i < 32; i++ { + go func() { + _, err := g.GetGeoLookup("8.8.8.8") + done <- err + }() + } + for i := 0; i < 32; i++ { + if err := <-done; err != nil { + t.Errorf("concurrent lookup: %v", err) + } + } +} diff --git a/dnscheck/internal/maxmind/testdata/GeoLite2-ASN.mmdb b/dnscheck/internal/maxmind/testdata/GeoLite2-ASN.mmdb new file mode 100644 index 00000000..9e2b2d5f Binary files /dev/null and b/dnscheck/internal/maxmind/testdata/GeoLite2-ASN.mmdb differ diff --git a/dnscheck/internal/maxmind/testdata/GeoLite2-City.mmdb b/dnscheck/internal/maxmind/testdata/GeoLite2-City.mmdb new file mode 100644 index 00000000..67c4e793 Binary files /dev/null and b/dnscheck/internal/maxmind/testdata/GeoLite2-City.mmdb differ diff --git a/dnscheck/internal/maxmind/testdata/README.md b/dnscheck/internal/maxmind/testdata/README.md new file mode 100644 index 00000000..8d6150df --- /dev/null +++ b/dnscheck/internal/maxmind/testdata/README.md @@ -0,0 +1,12 @@ +Stub MaxMind databases for dnscheck unit tests, NOT full GeoLite2 files. + +- `GeoLite2-ASN.mmdb` — database type `GeoLite2-ASN`, a handful of entries + (8.8.8.8 → AS15169 GOOGLE, Cloudflare / Apple / Microsoft ranges). Any other + IP yields an empty record with no error. +- `GeoLite2-City.mmdb` — database type `GeoLite2-City`. Used as the + wrong-database-type case for the startup guard; never queried for content. + +Regenerate from the repo's `tests/` directory (needs its venv): + + cd tests && source venv/bin/activate && \ + python scripts/generate_stub_mmdb.py --out-dir ../dnscheck/internal/maxmind/testdata --city-typed diff --git a/dnscheck/internal/maxmind/types.go b/dnscheck/internal/maxmind/types.go index 16b91f25..6652c387 100644 --- a/dnscheck/internal/maxmind/types.go +++ b/dnscheck/internal/maxmind/types.go @@ -4,5 +4,4 @@ type GeoLookup struct { IPAddress string `json:"ip_address"` ASN uint `json:"asn"` ASNOrganization string `json:"asn_organization"` - IsIvpnServer bool `json:"is_ivpn_server"` } diff --git a/dnscheck/main.go b/dnscheck/main.go index 3c51c8ab..a5676d63 100644 --- a/dnscheck/main.go +++ b/dnscheck/main.go @@ -23,8 +23,12 @@ func main() { if err != nil { log.Fatal().Err(err).Msg("Failed to read app configuration") } + // Infrastructure addresses, not client data: safe to log so operators can + // confirm which PoPs a running instance trusts. + log.Info().Str("ip_ranges", cfg.Server.IPRangesString()).Uint("asn", cfg.Server.ASN). + Msg("Trusted resolver sources") - cache, err := cache.New(cache.CacheTypeBigCache) + cache, err := cache.New(cache.CacheTypeBigCache, cfg.Cache.TTL) if err != nil { log.Fatal().Err(err).Msg("Failed to create cache") } diff --git a/libs/cache/config.go b/libs/cache/config.go index 483467f0..8490ad16 100644 --- a/libs/cache/config.go +++ b/libs/cache/config.go @@ -1,5 +1,7 @@ package cache +import "time" + // Config represents the cache configuration type Config struct { Address string @@ -14,4 +16,9 @@ type Config struct { KeyFile string CACertFile string TLSInsecureSkipVerify bool // Only for testing & development, use false in production + // CommandTimeout, when > 0, bounds every dial, read and write and limits + // go-redis to a single retry. Zero keeps the go-redis defaults + // (5s dial, 3s read/write, 3 retries), which suit bulk writers but not a + // per-query read path. + CommandTimeout time.Duration } diff --git a/libs/cache/redis.go b/libs/cache/redis.go index 1c0dafce..c7e49d33 100644 --- a/libs/cache/redis.go +++ b/libs/cache/redis.go @@ -50,6 +50,7 @@ func NewDirectClient(cfg *Config) (*redis.Client, error) { Username: cfg.Username, Password: cfg.Password, } + cfg.applyCommandTimeout(&options.DialTimeout, &options.ReadTimeout, &options.WriteTimeout, &options.MaxRetries, &options.ContextTimeoutEnabled) return redis.NewClient(options), nil } @@ -66,6 +67,7 @@ func NewFailoverClient(cfg *Config) (*redis.Client, error) { SentinelPassword: cfg.FailoverPassword, DB: 0, } + cfg.applyCommandTimeout(&options.DialTimeout, &options.ReadTimeout, &options.WriteTimeout, &options.MaxRetries, &options.ContextTimeoutEnabled) if cfg.TLSEnabled { log.Debug().Msg("Using TLS to connect to Redis") @@ -87,8 +89,24 @@ func NewFailoverClient(cfg *Config) (*redis.Client, error) { options.TLSConfig = &tls.Config{ Certificates: []tls.Certificate{cert}, RootCAs: caCertPool, - InsecureSkipVerify: cfg.TLSInsecureSkipVerify, // Only for testing, use false in production + InsecureSkipVerify: cfg.TLSInsecureSkipVerify, //nolint:gosec // operator opt-in for dev/test only; false in production } } return redis.NewFailoverClient(options), nil } + +// applyCommandTimeout maps the single configured budget onto go-redis, which +// has no command-level timeout of its own: connecting, each read and each +// write are separate knobs, and retries multiply them. One value for all of +// them keeps the worst case per operation at CommandTimeout × 2 attempts. +// Context deadlines are only applied to socket I/O when ContextTimeoutEnabled +// is set, so a caller's per-query deadline can cut a hung read as well. +// Zero leaves the go-redis defaults untouched. +func (c *Config) applyCommandTimeout(dial, read, write *time.Duration, maxRetries *int, contextTimeoutEnabled *bool) { + if c.CommandTimeout <= 0 { + return + } + *dial, *read, *write = c.CommandTimeout, c.CommandTimeout, c.CommandTimeout + *maxRetries = 1 + *contextTimeoutEnabled = true +} diff --git a/libs/cache/redis_test.go b/libs/cache/redis_test.go new file mode 100644 index 00000000..e6363b55 --- /dev/null +++ b/libs/cache/redis_test.go @@ -0,0 +1,49 @@ +package cache + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNewDirectClient_CommandTimeoutApplied(t *testing.T) { + c, err := NewDirectClient(&Config{Address: "127.0.0.1:1", CommandTimeout: 750 * time.Millisecond}) + require.NoError(t, err) + defer c.Close() + + opts := c.Options() + assert.Equal(t, 750*time.Millisecond, opts.DialTimeout) + assert.Equal(t, 750*time.Millisecond, opts.ReadTimeout) + assert.Equal(t, 750*time.Millisecond, opts.WriteTimeout) + assert.Equal(t, 1, opts.MaxRetries) + assert.True(t, opts.ContextTimeoutEnabled) +} + +func TestNewDirectClient_ZeroTimeoutKeepsClientDefaults(t *testing.T) { + c, err := NewDirectClient(&Config{Address: "127.0.0.1:1"}) + require.NoError(t, err) + defer c.Close() + + // go-redis fills its documented defaults in when nothing is set. + opts := c.Options() + assert.Equal(t, 5*time.Second, opts.DialTimeout) + assert.Equal(t, 3*time.Second, opts.ReadTimeout) + assert.Equal(t, 3*time.Second, opts.WriteTimeout) + assert.Equal(t, 3, opts.MaxRetries) + assert.False(t, opts.ContextTimeoutEnabled) +} + +func TestNewFailoverClient_CommandTimeoutApplied(t *testing.T) { + c, err := NewFailoverClient(&Config{MasterName: "m", FailoverAddresses: []string{"127.0.0.1:1"}, CommandTimeout: 400 * time.Millisecond}) + require.NoError(t, err) + defer c.Close() + + opts := c.Options() + assert.Equal(t, 400*time.Millisecond, opts.DialTimeout) + assert.Equal(t, 400*time.Millisecond, opts.ReadTimeout) + assert.Equal(t, 400*time.Millisecond, opts.WriteTimeout) + assert.Equal(t, 1, opts.MaxRetries) + assert.True(t, opts.ContextTimeoutEnabled) +} diff --git a/proxy/.env.sample b/proxy/.env.sample index 717811ac..b1ba22dc 100644 --- a/proxy/.env.sample +++ b/proxy/.env.sample @@ -3,6 +3,7 @@ SERVER_NAME="moddns.dev" DNS_CHECK_DOMAIN="test.moddns.dev" DNS_CHECK_PORT="53" MAX_GOROUTINES=10000 +POP_NAME=dev1 ### UPSTREAM CONFIG # Format: name=address,name=address,... @@ -56,9 +57,8 @@ EMITTER_SINK_DB_AUTH_SOURCE="admin" COLLECTOR_QUERY_LOGS_BATCH_SIZE=100 COLLECTOR_QUERY_LOGS_BATCH_INTERVAL=10s -COLLECTOR_STATISTICS_BATCH_SIZE=1000 -COLLECTOR_STATISTICS_BATCH_INTERVAL=30s - +COLLECTOR_SERVICE_STATISTICS_BATCH_SIZE=1000 +COLLECTOR_SERVICE_STATISTICS_BATCH_INTERVAL=30s ### SENTRY CONFIG SENTRY_DSN="" diff --git a/proxy/cache/cache.go b/proxy/cache/cache.go index abc3dc1b..cc78f257 100644 --- a/proxy/cache/cache.go +++ b/proxy/cache/cache.go @@ -10,23 +10,25 @@ import ( const CacheTypeRedis = "redis" -// Cache is an interface for caching functionalities -type Cache interface { - GetProfileBlocklists(ctx context.Context, profileId string) ([]string, error) - GetProfileServicesBlocked(ctx context.Context, profileId string) ([]string, error) - GetProfileLogsSettings(ctx context.Context, profileId string) (map[string]string, error) - GetProfileDNSSECSettings(ctx context.Context, profileId string) (map[string]string, error) - GetProfileAdvancedSettings(ctx context.Context, profileId string) (map[string]string, error) - GetProfileStatisticsSettings(ctx context.Context, profileId string) (map[string]string, error) - GetProfilePrivacySettings(ctx context.Context, profileId string) (map[string]string, error) - GetBlocklistEntry(ctx context.Context, blocklistId string, domain string) (bool, error) - GetCustomRulesHashes(ctx context.Context, profileId string) ([]string, error) - GetCustomRulesHash(ctx context.Context, hashId string) (map[string]string, error) +// ErrSettingsNotFound is model.ErrSettingsNotFound, re-exported for callers +// that only know the cache. +var ErrSettingsNotFound = model.ErrSettingsNotFound - // GetProfileSettingsBatch fetches privacy, logs, DNSSEC, and advanced - // settings for a profile in a single Redis pipeline round-trip. - // The returned ProfileSettings contains per-key results and errors. +// Cache is the proxy's read-only view of the settings store. Per-profile +// inputs come from one GetProfileSettingsBatch call and travel on the request +// context; blocklist membership and its exception sets are the only per-query +// lookups. +type Cache interface { + // GetProfileSettingsBatch fetches every per-profile input in one batch. + // It returns an error only when the store is unreachable; per-key + // outcomes are reported on the returned ProfileSettings. GetProfileSettingsBatch(ctx context.Context, profileId string) (*model.ProfileSettings, error) + // GetBlocklistEntry reports whether fqdn is a member of the blocklist set. + GetBlocklistEntry(ctx context.Context, blocklistId string, domain string) (bool, error) + // GetBlocklistExceptionEntry checks if a domain is present in the + // blocklist's companion exception set — the domains the list's own + // authors unblock. A missing set means no exceptions. + GetBlocklistExceptionEntry(ctx context.Context, blocklistId string, domain string) (bool, error) // Close shuts down the cache and releases resources. Close() diff --git a/proxy/cache/redis.go b/proxy/cache/redis.go index 77e11ef2..0d567564 100644 --- a/proxy/cache/redis.go +++ b/proxy/cache/redis.go @@ -106,7 +106,7 @@ func (c *RedisCache) getProfileSettings(ctx context.Context, profileId string, s // Profile ID goes in a structured (Sentry-denylisted) field, never in // the message or error text. log.Warn().Str("profile_id", profileId).Msgf("No %s settings found for profile", settingsName) - return nil, fmt.Errorf("no %s settings found for profile", settingsName) + return nil, fmt.Errorf("%w: %s", ErrSettingsNotFound, settingsName) } return cmd.Val(), nil } @@ -121,6 +121,18 @@ func (c *RedisCache) GetBlocklistEntry(ctx context.Context, blocklistId string, return cmd.Val(), nil } +// GetBlocklistExceptionEntry checks if a domain is present in the blocklist's +// companion exception set. SISMEMBER on a missing key returns false, which is +// exactly the "no exceptions" semantics for lists without a published set. +func (c *RedisCache) GetBlocklistExceptionEntry(ctx context.Context, blocklistId string, fqdn string) (bool, error) { + exceptionsKey := "blocklist:" + blocklistId + ":exceptions" + cmd := c.client().SIsMember(ctx, exceptionsKey, fqdn) + if err := cmd.Err(); err != nil { + return false, err + } + return cmd.Val(), nil +} + // GetCustomRulesHashes gets list of custom rules set names func (c *RedisCache) GetCustomRulesHashes(ctx context.Context, profileId string) ([]string, error) { customRulesSetKey := fmt.Sprintf("settings:%s:custom_rules", profileId) @@ -140,93 +152,111 @@ func (c *RedisCache) GetCustomRulesHash(ctx context.Context, hashId string) (map return cmd.Val(), nil } -// GetProfileSettingsBatch fetches privacy, logs, DNSSEC, rebinding protection, and -// advanced settings for a profile in a single Redis pipeline round-trip. +// GetProfileSettingsBatch fetches every per-profile input the proxy needs in +// two pipeline round-trips: the settings hashes, lists and the custom-rule +// set first, then the custom-rule hashes named by that set. func (c *RedisCache) GetProfileSettingsBatch(ctx context.Context, profileId string) (*model.ProfileSettings, error) { if profileId == "" { return nil, fmt.Errorf("profile ID cannot be empty") } - privacyKey := "settings:" + profileId + ":privacy" - logsKey := "settings:" + profileId + ":logs" - dnssecKey := "settings:" + profileId + ":security:dnssec" - rebindingKey := "settings:" + profileId + ":security:rebinding_protection" - advancedKey := "settings:" + profileId + ":advanced" - + settingsKey := "settings:" + profileId pipe := c.client().Pipeline() - privacyCmd := pipe.HGetAll(ctx, privacyKey) - logsCmd := pipe.HGetAll(ctx, logsKey) - dnssecCmd := pipe.HGetAll(ctx, dnssecKey) - rebindingCmd := pipe.HGetAll(ctx, rebindingKey) - advancedCmd := pipe.HGetAll(ctx, advancedKey) + privacyCmd := pipe.HGetAll(ctx, settingsKey+":privacy") + logsCmd := pipe.HGetAll(ctx, settingsKey+":logs") + dnssecCmd := pipe.HGetAll(ctx, settingsKey+":security:dnssec") + rebindingCmd := pipe.HGetAll(ctx, settingsKey+":security:rebinding_protection") + advancedCmd := pipe.HGetAll(ctx, settingsKey+":advanced") + statisticsCmd := pipe.HGetAll(ctx, settingsKey+":statistics") + blocklistsCmd := pipe.LRange(ctx, settingsKey+":blocklists", 0, -1) + servicesCmd := pipe.LRange(ctx, settingsKey+":services", 0, -1) + customRulesCmd := pipe.SMembers(ctx, settingsKey+":custom_rules") - _, err := pipe.Exec(ctx) - // Pipeline Exec returns the error of the first failed command, but - // individual commands still hold their own results/errors. We only - // treat a total pipeline failure (e.g. connection lost) as fatal. - if err != nil && err != redis.Nil { - // If all commands failed with the same error, it's a connection-level - // failure (e.g. TCP reset, auth error) — return it so the caller can - // log the real cause instead of a misleading "profile not found". - if privacyCmd.Err() == err && logsCmd.Err() == err && - dnssecCmd.Err() == err && rebindingCmd.Err() == err && advancedCmd.Err() == err { - return nil, fmt.Errorf("redis pipeline failed: %w", err) - } - // Otherwise it's a partial failure — handle per-command below. - log.Warn().Err(err).Msg("Redis pipeline partial error, checking individual commands") + cmds := []redis.Cmder{privacyCmd, logsCmd, dnssecCmd, rebindingCmd, advancedCmd, statisticsCmd, blocklistsCmd, servicesCmd, customRulesCmd} + if err := execPipeline(ctx, pipe, cmds); err != nil { + return nil, err } result := &model.ProfileSettings{} + result.Privacy, result.PrivacyErr = hashResult(privacyCmd, "privacy") + result.Logs, result.LogsErr = hashResult(logsCmd, "logs") + result.DNSSEC, result.DNSSECErr = hashResult(dnssecCmd, "security dnssec") + // Missing hash = empty map = opt-in OFF. + result.RebindingProtection, result.RebindingProtectionErr = hashResult(rebindingCmd, "security rebinding_protection") + result.Advanced, result.AdvancedErr = hashResult(advancedCmd, "advanced") + result.Statistics, result.StatisticsErr = hashResult(statisticsCmd, "statistics") + result.Blocklists, result.BlocklistsErr = blocklistsCmd.Result() + result.Services, result.ServicesErr = servicesCmd.Result() - // Privacy - switch { - case privacyCmd.Err() != nil: - result.PrivacyErr = privacyCmd.Err() - case len(privacyCmd.Val()) == 0: - result.PrivacyErr = errors.New("no [privacy] settings found for profile") - default: - result.Privacy = privacyCmd.Val() - } - - // Logs - switch { - case logsCmd.Err() != nil: - result.LogsErr = logsCmd.Err() - case len(logsCmd.Val()) == 0: - result.LogsErr = errors.New("no [logs] settings found for profile") - default: - result.Logs = logsCmd.Val() - } - - // DNSSEC - switch { - case dnssecCmd.Err() != nil: - result.DNSSECErr = dnssecCmd.Err() - case len(dnssecCmd.Val()) == 0: - result.DNSSECErr = errors.New("no [security dnssec] settings found for profile") - default: - result.DNSSEC = dnssecCmd.Val() - } - - // Rebinding protection (security). Missing hash = empty map = opt-in OFF. - switch { - case rebindingCmd.Err() != nil: - result.RebindingProtectionErr = rebindingCmd.Err() - case len(rebindingCmd.Val()) == 0: - result.RebindingProtectionErr = errors.New("no [security rebinding_protection] settings found for profile") - default: - result.RebindingProtection = rebindingCmd.Val() - } - - // Advanced - switch { - case advancedCmd.Err() != nil: - result.AdvancedErr = advancedCmd.Err() - case len(advancedCmd.Val()) == 0: - result.AdvancedErr = errors.New("no [advanced] settings found for profile") - default: - result.Advanced = advancedCmd.Val() + ruleIDs, err := customRulesCmd.Result() + if err != nil { + result.CustomRulesErr = err + return result, nil } - + result.CustomRules, result.CustomRulesErr = c.getCustomRules(ctx, ruleIDs) return result, nil } + +// getCustomRules loads the named rule hashes in one pipeline. A rule whose +// hash is empty (a set member left behind by a deleted rule) is skipped. +func (c *RedisCache) getCustomRules(ctx context.Context, ruleIDs []string) ([]map[string]string, error) { + if len(ruleIDs) == 0 { + return nil, nil + } + pipe := c.client().Pipeline() + cmds := make([]*redis.MapStringStringCmd, len(ruleIDs)) + cmders := make([]redis.Cmder, len(ruleIDs)) + for i, id := range ruleIDs { + cmds[i] = pipe.HGetAll(ctx, id) + cmders[i] = cmds[i] + } + if err := execPipeline(ctx, pipe, cmders); err != nil { + return nil, err + } + rules := make([]map[string]string, 0, len(ruleIDs)) + for i, cmd := range cmds { + rule, err := cmd.Result() + if err != nil { + // The rule key embeds the profile ID; it stays out of the error text. + return nil, fmt.Errorf("custom rule hash %d of %d: %w", i+1, len(ruleIDs), err) + } + if len(rule) == 0 { + continue + } + rules = append(rules, rule) + } + return rules, nil +} + +// execPipeline runs the pipeline and returns an error for a connection-level +// failure. Only server replies (WRONGTYPE, MOVED, ...) implement redis.Error; +// a dial or transport failure surfaces as a plain error from Exec while the +// individual commands may carry no error at all, so classification goes by +// type, never by comparing per-command errors. +func execPipeline(ctx context.Context, pipe redis.Pipeliner, cmds []redis.Cmder) error { + _, err := pipe.Exec(ctx) + if err == nil { + return nil + } + // redis.Nil is itself a reply error and lands in the per-command branch. + var replyErr redis.Error + if !errors.As(err, &replyErr) { + return fmt.Errorf("redis pipeline failed: %w", err) + } + // A server reply error belongs to one command and is handled per command. + log.Warn().Err(err).Int("commands", len(cmds)).Msg("Redis pipeline partial error, checking individual commands") + return nil +} + +// hashResult maps an HGETALL outcome to (value, error): a read error is kept +// as is, an empty hash becomes ErrSettingsNotFound. +func hashResult(cmd *redis.MapStringStringCmd, name string) (map[string]string, error) { + val, err := cmd.Result() + if err != nil { + return nil, err + } + if len(val) == 0 { + return nil, fmt.Errorf("%w: [%s]", ErrSettingsNotFound, name) + } + return val, nil +} diff --git a/proxy/cache/redis_batch_test.go b/proxy/cache/redis_batch_test.go new file mode 100644 index 00000000..169e5533 --- /dev/null +++ b/proxy/cache/redis_batch_test.go @@ -0,0 +1,146 @@ +package cache + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + libscache "github.com/ivpn/dns/libs/cache" + goredis "github.com/redis/go-redis/v9" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/modules/redis" +) + +// startRedis runs a throwaway Redis and returns a RedisCache bound to it plus +// a raw client for seeding. Skips when no container runtime is available. +func startRedis(t *testing.T) (*RedisCache, *goredis.Client) { + t.Helper() + testcontainers.SkipIfProviderIsNotHealthy(t) + ctx := context.Background() + + redisC, err := redis.Run(ctx, "redis:7") + if err != nil { + t.Skipf("redis container unavailable: %v", err) + } + t.Cleanup(func() { _ = testcontainers.TerminateContainer(redisC) }) + + host, err := redisC.Host(ctx) + require.NoError(t, err) + port, err := redisC.MappedPort(ctx, "6379") + require.NoError(t, err) + addr := fmt.Sprintf("%s:%s", host, port.Port()) + + rdb := goredis.NewClient(&goredis.Options{Addr: addr}) + t.Cleanup(func() { _ = rdb.Close() }) + return &RedisCache{dual: libscache.NewSingleClient(rdb)}, rdb +} + +// specRef: proxy-request-admission-behaviour.md #S10 +func TestGetProfileSettingsBatch_PopulatesListsAndRules(t *testing.T) { + c, rdb := startRedis(t) + ctx := context.Background() + const profileID = "batchprofile1" + key := "settings:" + profileID + + pipe := rdb.Pipeline() + pipe.HSet(ctx, key+":privacy", map[string]interface{}{"default_rule": "allow", "blocklists_subdomains_rule": "block"}) + pipe.HSet(ctx, key+":logs", map[string]interface{}{"enabled": "true"}) + pipe.HSet(ctx, key+":statistics", map[string]interface{}{"enabled": "false"}) + pipe.RPush(ctx, key+":blocklists", "bl-ads", "bl-tracking") + pipe.RPush(ctx, key+":services", "google") + pipe.HSet(ctx, key+":custom_rule:r1", map[string]interface{}{"value": "ads.example", "action": "block", "syntax": "domain"}) + pipe.HSet(ctx, key+":custom_rule:r2", map[string]interface{}{"value": "cdn.example", "action": "allow", "syntax": "domain"}) + // A set member whose hash no longer exists must be skipped, not fail the batch. + pipe.SAdd(ctx, key+":custom_rules", key+":custom_rule:r1", key+":custom_rule:r2", key+":custom_rule:gone") + _, err := pipe.Exec(ctx) + require.NoError(t, err) + + got, err := c.GetProfileSettingsBatch(ctx, profileID) + require.NoError(t, err) + + require.NoError(t, got.PrivacyErr) + assert.Equal(t, "allow", got.Privacy["default_rule"]) + require.NoError(t, got.LogsErr) + require.NoError(t, got.StatisticsErr) + assert.Equal(t, "false", got.Statistics["enabled"]) + + // Absent hashes are reported as not-found, never as store failures. + assert.ErrorIs(t, got.DNSSECErr, ErrSettingsNotFound) + assert.ErrorIs(t, got.AdvancedErr, ErrSettingsNotFound) + assert.ErrorIs(t, got.RebindingProtectionErr, ErrSettingsNotFound) + assert.NoError(t, got.StoreError()) + + require.NoError(t, got.BlocklistsErr) + assert.Equal(t, []string{"bl-ads", "bl-tracking"}, got.Blocklists, "subscription order preserved") + require.NoError(t, got.ServicesErr) + assert.Equal(t, []string{"google"}, got.Services) + + require.NoError(t, got.CustomRulesErr) + require.Len(t, got.CustomRules, 2, "stale set member with an empty hash is skipped") + values := map[string]string{} + for _, rule := range got.CustomRules { + values[rule["value"]] = rule["action"] + } + assert.Equal(t, map[string]string{"ads.example": "block", "cdn.example": "allow"}, values) +} + +// specRef: proxy-request-admission-behaviour.md #S10 +func TestGetProfileSettingsBatch_UnknownProfile(t *testing.T) { + c, _ := startRedis(t) + ctx := context.Background() + + got, err := c.GetProfileSettingsBatch(ctx, "nosuchprofile") + require.NoError(t, err, "an empty profile is a per-key outcome, not a batch failure") + + assert.ErrorIs(t, got.PrivacyErr, ErrSettingsNotFound) + assert.NoError(t, got.StoreError()) + assert.Empty(t, got.Blocklists) + assert.Empty(t, got.Services) + assert.Empty(t, got.CustomRules) + assert.NoError(t, got.BlocklistsErr) + assert.NoError(t, got.ServicesErr) + assert.NoError(t, got.CustomRulesErr) +} + +// specRef: proxy-request-admission-behaviour.md #S10 +func TestGetProfileSettingsBatch_NoRules(t *testing.T) { + c, rdb := startRedis(t) + ctx := context.Background() + const profileID = "batchprofile2" + require.NoError(t, rdb.HSet(ctx, "settings:"+profileID+":privacy", "default_rule", "block").Err()) + + got, err := c.GetProfileSettingsBatch(ctx, profileID) + require.NoError(t, err) + require.NoError(t, got.PrivacyErr) + assert.NoError(t, got.CustomRulesErr) + assert.Empty(t, got.CustomRules, "no second pipeline is needed for a profile without rules") +} + +// specRef: proxy-request-admission-behaviour.md #S10 #S4 +func TestGetProfileSettingsBatch_StoreUnreachable(t *testing.T) { + // A closed port on loopback refuses immediately; no container needed. + rdb := goredis.NewClient(&goredis.Options{ + Addr: "127.0.0.1:1", + DialTimeout: 200 * time.Millisecond, + MaxRetries: 0, + }) + t.Cleanup(func() { _ = rdb.Close() }) + c := &RedisCache{dual: libscache.NewSingleClient(rdb)} + + got, err := c.GetProfileSettingsBatch(context.Background(), "anyprofile") + + require.Error(t, err, "a connection-level failure is the batch's error, not a per-key outcome") + assert.Nil(t, got) + assert.False(t, errors.Is(err, ErrSettingsNotFound)) +} + +func TestGetProfileSettingsBatch_EmptyProfileID(t *testing.T) { + c := &RedisCache{} + got, err := c.GetProfileSettingsBatch(context.Background(), "") + require.Error(t, err) + assert.Nil(t, got) +} diff --git a/proxy/cache/redis_benchmark_test.go b/proxy/cache/redis_benchmark_test.go index ef032434..fbd81dc7 100644 --- a/proxy/cache/redis_benchmark_test.go +++ b/proxy/cache/redis_benchmark_test.go @@ -8,8 +8,8 @@ import ( toxiclient "github.com/Shopify/toxiproxy/v2/client" libscache "github.com/ivpn/dns/libs/cache" + "github.com/ivpn/dns/proxy/internal/settingscache" "github.com/ivpn/dns/proxy/model" - gocache "github.com/patrickmn/go-cache" goredis "github.com/redis/go-redis/v9" "github.com/stretchr/testify/require" "github.com/testcontainers/testcontainers-go" @@ -48,18 +48,53 @@ func seedTestData(ctx context.Context, t testing.TB, redisAddr string) { pipe.HSet(ctx, "settings:"+benchProfileID+":advanced", map[string]interface{}{ "recursor": "default", }) + pipe.HSet(ctx, "settings:"+benchProfileID+":statistics", map[string]interface{}{ + "enabled": "false", + }) + pipe.RPush(ctx, "settings:"+benchProfileID+":blocklists", "bl-ads", "bl-tracking", "bl-malware") + pipe.RPush(ctx, "settings:"+benchProfileID+":services", "google", "meta") + for i, rule := range benchCustomRules { + key := fmt.Sprintf("settings:%s:custom_rule:%d", benchProfileID, i) + pipe.HSet(ctx, key, rule) + pipe.SAdd(ctx, "settings:"+benchProfileID+":custom_rules", key) + } _, err := pipe.Exec(ctx) require.NoError(t, err) } -// getSettingsSequential mimics the old sequential settings-fetch code path: -// 4 sequential Redis round-trips for profile settings. +// benchCustomRules is a small, realistic rule set for the seeded profile. +var benchCustomRules = []map[string]interface{}{ + {"value": "ads.example", "action": "block", "syntax": "domain"}, + {"value": "*.tracker.example", "action": "block", "syntax": "domain"}, + {"value": "cdn.example", "action": "allow", "syntax": "domain"}, + {"value": "203.0.113.7", "action": "block", "syntax": "ip"}, +} + +// getSettingsSequential mimics per-stage store reads: one round-trip per +// settings group, list and custom rule, so the comparison against the batch +// covers the same inputs. func getSettingsSequential(ctx context.Context, c *RedisCache, profileId string) *model.ProfileSettings { result := &model.ProfileSettings{} result.Privacy, result.PrivacyErr = c.GetProfilePrivacySettings(ctx, profileId) result.Logs, result.LogsErr = c.GetProfileLogsSettings(ctx, profileId) result.DNSSEC, result.DNSSECErr = c.GetProfileDNSSECSettings(ctx, profileId) result.Advanced, result.AdvancedErr = c.GetProfileAdvancedSettings(ctx, profileId) + result.Statistics, result.StatisticsErr = c.GetProfileStatisticsSettings(ctx, profileId) + result.Blocklists, result.BlocklistsErr = c.GetProfileBlocklists(ctx, profileId) + result.Services, result.ServicesErr = c.GetProfileServicesBlocked(ctx, profileId) + ruleIDs, err := c.GetCustomRulesHashes(ctx, profileId) + if err != nil { + result.CustomRulesErr = err + return result + } + for _, id := range ruleIDs { + rule, err := c.GetCustomRulesHash(ctx, id) + if err != nil { + result.CustomRulesErr = err + return result + } + result.CustomRules = append(result.CustomRules, rule) + } return result } @@ -155,6 +190,8 @@ func BenchmarkGetProfileSettings(b *testing.B) { require.NotNil(b, ps.Privacy) require.Nil(b, ps.LogsErr) require.NotNil(b, ps.Logs) + require.Len(b, ps.Blocklists, 3) + require.Len(b, ps.CustomRules, len(benchCustomRules)) b.ResetTimer() for i := 0; i < b.N; i++ { @@ -170,6 +207,8 @@ func BenchmarkGetProfileSettings(b *testing.B) { require.NotNil(b, ps.Privacy) require.Nil(b, ps.LogsErr) require.NotNil(b, ps.Logs) + require.Len(b, ps.Blocklists, 3) + require.Len(b, ps.CustomRules, len(benchCustomRules)) b.ResetTimer() for i := 0; i < b.N; i++ { @@ -183,8 +222,9 @@ func BenchmarkGetProfileSettings(b *testing.B) { require.NoError(b, err) require.Nil(b, ps.PrivacyErr) - localCache := gocache.New(30*time.Second, time.Minute) - localCache.Set(benchProfileID, ps, gocache.DefaultExpiration) + localCache, err := settingscache.New(30*time.Second, 1024) + require.NoError(b, err) + localCache.Put(benchProfileID, ps) b.ResetTimer() for i := 0; i < b.N; i++ { diff --git a/proxy/collector/collector.go b/proxy/collector/collector.go index 99642d6f..d2b33948 100644 --- a/proxy/collector/collector.go +++ b/proxy/collector/collector.go @@ -32,13 +32,14 @@ func NewCollector(collectorCfg config.CollectorConfig, collectorType string, sto batchSize := collectorCfg.GetBatchSize() freq := collectorCfg.GetFrequency() statsChan := make(chan (model.EventStatistics), batchSize) - return &StatisticsCollector{ + return &ServiceStatisticsCollector{ Type: collectorType, StopChan: stopChan, Frequency: freq, BatchSize: batchSize, StatsChan: statsChan, Emitter: emitter, + Pop: collectorCfg.GetPopName(), }, nil default: return nil, errors.New("unknown collector type") diff --git a/proxy/collector/service_statistics.go b/proxy/collector/service_statistics.go new file mode 100644 index 00000000..8a7194b8 --- /dev/null +++ b/proxy/collector/service_statistics.go @@ -0,0 +1,108 @@ +package collector + +import ( + "context" + "sort" + "time" + + "github.com/ivpn/dns/proxy/collector/channel" + "github.com/ivpn/dns/proxy/emitter" + "github.com/ivpn/dns/proxy/model" + "github.com/rs/zerolog/log" +) + +// ServiceStatisticsCollector sums per-query counter events into one +// service-wide document per hour for this PoP and hands the open documents to +// the emitter on every flush. Only the Collect goroutine touches the +// accumulator, so no locking is needed. +type ServiceStatisticsCollector struct { + Type string + BatchSize int + StopChan chan struct{} + Frequency time.Duration + StatsChan chan model.EventStatistics + Emitter emitter.Emitter + Pop string + // Now places events into their hour; tests inject a clock. + Now func() time.Time + + buckets map[time.Time]*model.ServiceStatistics + counter int +} + +func (c *ServiceStatisticsCollector) Collect() error { + ticker := time.NewTicker(c.Frequency) + defer ticker.Stop() + for { + select { + case event, ok := <-c.StatsChan: + if !ok { + log.Debug().Msg("Channel closed or empty") + continue + } + c.add(event) + if c.counter >= c.BatchSize { + c.flush("batch_size") + } + case <-ticker.C: + if c.counter == 0 { + log.Trace().Msg("Postpone stats event emission") + continue + } + c.flush("frequency") + case <-c.StopChan: + log.Info().Msg("Stopping statistics collector") + return nil + } + } +} + +func (c *ServiceStatisticsCollector) GetChannel() channel.CollectorChannel { + return channel.EventStatisticsChannel{Channel: c.StatsChan} +} + +func (c *ServiceStatisticsCollector) now() time.Time { + if c.Now != nil { + return c.Now() + } + return time.Now() +} + +// add sums one event into the document for the hour that is open now. +func (c *ServiceStatisticsCollector) add(event model.EventStatistics) { + if c.buckets == nil { + c.buckets = make(map[time.Time]*model.ServiceStatistics) + } + now := c.now() + bucket := model.BucketStart(now) + doc, ok := c.buckets[bucket] + if !ok { + doc = model.NewServiceStatistics(c.Pop, now) + c.buckets[bucket] = doc + } + doc.Aggregate(event) + c.counter++ +} + +// flush emits every open document as an increment and resets. A failed emit +// is logged and the counters are dropped, as for query logs. +func (c *ServiceStatisticsCollector) flush(trigger string) { + if len(c.buckets) == 0 { + return + } + batch := make([]model.ServiceStatistics, 0, len(c.buckets)) + for _, doc := range c.buckets { + batch = append(batch, *doc) + } + sort.Slice(batch, func(i, j int) bool { return batch[i].Timestamp.Before(batch[j].Timestamp) }) + + ctx, cancel := context.WithTimeout(context.Background(), EmitTimeout) + defer cancel() + log.Info().Str("event_type", c.Type).Str("trigger", trigger).Int("events_number", len(batch)).Msg("Emitting stats events batch") + if err := c.Emitter.EmitServiceStatistics(ctx, batch); err != nil { + log.Error().Err(err).Msg("Failed to emit stats events") + } + + c.buckets = make(map[time.Time]*model.ServiceStatistics) + c.counter = 0 +} diff --git a/proxy/collector/service_statistics_test.go b/proxy/collector/service_statistics_test.go new file mode 100644 index 00000000..298a4254 --- /dev/null +++ b/proxy/collector/service_statistics_test.go @@ -0,0 +1,140 @@ +package collector + +import ( + "testing" + "time" + + "github.com/ivpn/dns/proxy/mocks" + "github.com/ivpn/dns/proxy/model" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +func newTestStatsCollector(t *testing.T, emitter *mocks.Emitter, batchSize int, freq time.Duration) *ServiceStatisticsCollector { + t.Helper() + return &ServiceStatisticsCollector{ + Type: model.TYPE_STATISTICS, + BatchSize: batchSize, + Frequency: freq, + StopChan: make(chan struct{}), + StatsChan: make(chan model.EventStatistics, batchSize), + Emitter: emitter, + Pop: "ams1", + } +} + +func evt(q model.Queries) model.EventStatistics { + return model.EventStatistics{Queries: q} +} + +// specRef: proxy-statistics-behaviour.md #Y5 #Y6 +func TestServiceStatisticsCollector_SumsEventsIntoHourDocuments(t *testing.T) { + emitter := mocks.NewEmitter(t) + c := newTestStatsCollector(t, emitter, 100, time.Minute) + clock := time.Date(2026, 9, 17, 13, 58, 30, 0, time.UTC) + c.Now = func() time.Time { return clock } + + c.add(evt(model.Queries{Total: 1})) + c.add(evt(model.Queries{Total: 1, Blocked: 1})) + clock = clock.Add(90 * time.Second) // crosses into the next hour + c.add(evt(model.Queries{Total: 1, DNSSEC: 1})) + + var got []model.ServiceStatistics + emitter.On("EmitServiceStatistics", mock.Anything, mock.MatchedBy(func(batch []model.ServiceStatistics) bool { + got = batch + return true + })).Return(nil).Once() + + c.flush("test") + + require.Len(t, got, 2, "one document per hour touched in this flush") + assert.Equal(t, "ams1:2026-09-17T13", got[0].ID) + assert.Equal(t, model.Queries{Total: 2, Blocked: 1}, got[0].Queries) + assert.Equal(t, "ams1:2026-09-17T14", got[1].ID) + assert.Equal(t, model.Queries{Total: 1, DNSSEC: 1}, got[1].Queries) + for _, doc := range got { + assert.Equal(t, "ams1", doc.Pop) + assert.Zero(t, doc.Timestamp.Minute()+doc.Timestamp.Second()+doc.Timestamp.Nanosecond(), "hour start only") + } +} + +// specRef: proxy-statistics-behaviour.md #Y8 +func TestServiceStatisticsCollector_FlushResetsAccumulator(t *testing.T) { + emitter := mocks.NewEmitter(t) + c := newTestStatsCollector(t, emitter, 100, time.Minute) + c.Now = func() time.Time { return time.Date(2026, 9, 17, 13, 10, 0, 0, time.UTC) } + + emitter.On("EmitServiceStatistics", mock.Anything, mock.MatchedBy(func(batch []model.ServiceStatistics) bool { + return len(batch) == 1 && batch[0].Queries.Total == 3 + })).Return(nil).Once() + + for i := 0; i < 3; i++ { + c.add(evt(model.Queries{Total: 1})) + } + c.flush("test") + + assert.Empty(t, c.buckets) + assert.Zero(t, c.counter) + c.flush("test") // nothing pending: no emit (the mock would fail on a second call) +} + +// specRef: proxy-statistics-behaviour.md #Y8 +func TestServiceStatisticsCollector_Collect_FlushesOnBatchSizeAndInterval(t *testing.T) { + emitter := mocks.NewEmitter(t) + c := newTestStatsCollector(t, emitter, 2, 50*time.Millisecond) + + batches := make(chan []model.ServiceStatistics, 4) + emitter.On("EmitServiceStatistics", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + batches <- args.Get(1).([]model.ServiceStatistics) + }).Return(nil) + + done := make(chan struct{}) + go func() { _ = c.Collect(); close(done) }() + + // Two events reach the batch size and flush immediately. + c.StatsChan <- evt(model.Queries{Total: 1}) + c.StatsChan <- evt(model.Queries{Total: 1, Blocked: 1}) + select { + case batch := <-batches: + require.Len(t, batch, 1) + assert.Equal(t, model.Queries{Total: 2, Blocked: 1}, batch[0].Queries) + case <-time.After(time.Second): + t.Fatal("batch-size flush did not happen") + } + + // One event below the batch size is flushed by the ticker. + c.StatsChan <- evt(model.Queries{Total: 1}) + select { + case batch := <-batches: + require.Len(t, batch, 1) + assert.Equal(t, model.Queries{Total: 1}, batch[0].Queries) + case <-time.After(time.Second): + t.Fatal("interval flush did not happen") + } + + close(c.StopChan) + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("collector did not stop") + } +} + +// specRef: proxy-statistics-behaviour.md #Y9 +func TestServiceStatisticsCollector_EmitErrorDropsBatchAndContinues(t *testing.T) { + emitter := mocks.NewEmitter(t) + c := newTestStatsCollector(t, emitter, 100, time.Minute) + c.Now = func() time.Time { return time.Date(2026, 9, 17, 13, 10, 0, 0, time.UTC) } + + emitter.On("EmitServiceStatistics", mock.Anything, mock.Anything).Return(assert.AnError).Once() + c.add(evt(model.Queries{Total: 1})) + assert.NotPanics(t, func() { c.flush("test") }) + assert.Empty(t, c.buckets, "a failed batch is dropped, not retried") + + emitter.On("EmitServiceStatistics", mock.Anything, mock.MatchedBy(func(batch []model.ServiceStatistics) bool { + return len(batch) == 1 && batch[0].Queries.Total == 1 + })).Return(nil).Once() + c.add(evt(model.Queries{Total: 1})) + c.flush("test") +} diff --git a/proxy/collector/statistics.go b/proxy/collector/statistics.go deleted file mode 100644 index 2ef90e83..00000000 --- a/proxy/collector/statistics.go +++ /dev/null @@ -1,104 +0,0 @@ -package collector - -import ( - "context" - "sync" - "time" - - "github.com/ivpn/dns/proxy/collector/channel" - "github.com/ivpn/dns/proxy/emitter" - "github.com/ivpn/dns/proxy/model" - "github.com/rs/zerolog/log" -) - -type StatisticsCollector struct { - Type string - BatchSize int - StopChan chan struct{} - Frequency time.Duration - StatsChan chan model.EventStatistics - Emitter emitter.Emitter - mu sync.Mutex - statsAggregated sync.Map - statsToEmit []model.EventStatistics -} - -func (c *StatisticsCollector) Collect() error { - ticker := time.NewTicker(c.Frequency) - counter := 0 - ctx := context.Background() - for { - select { - case statsEvent, ok := <-c.StatsChan: - if !ok { - log.Debug().Msg("Channel closed or empty") - continue - } - - c.mu.Lock() - stats, ok := c.statsAggregated.Load(statsEvent.Statistics.ProfileID) - if ok { - stats.(*model.Statistics).Aggregate(statsEvent.Statistics) - c.statsAggregated.Store(statsEvent.Statistics.ProfileID, stats) - } else { - c.statsAggregated.Store(statsEvent.Statistics.ProfileID, statsEvent.Statistics) - } - counter++ - c.mu.Unlock() - - if counter == c.BatchSize { - timeoutCtx, cancel := context.WithTimeout(ctx, EmitTimeout) - c.statsAggregated.Range(func(key, value interface{}) bool { - stats, _ := value.(*model.Statistics) - c.statsToEmit = append(c.statsToEmit, model.EventStatistics{ - Statistics: stats, - }) - return true - }) - log.Info().Str("event_type", c.Type).Str("trigger", "batch_size").Int("events_number", len(c.statsToEmit)).Msg("Emitting stats events batch") - if err := c.Emitter.EmitStatistics(timeoutCtx, c.statsToEmit); err != nil { - log.Error().Err(err).Msg("Failed to emit stats events") - } - cancel() - - // reset for next batch - c.mu.Lock() - c.statsToEmit = make([]model.EventStatistics, 0) - c.statsAggregated.Clear() - counter = 0 - c.mu.Unlock() - } - case <-ticker.C: - c.mu.Lock() - c.statsAggregated.Range(func(key, value interface{}) bool { - stats, _ := value.(*model.Statistics) - c.statsToEmit = append(c.statsToEmit, model.EventStatistics{ - Statistics: stats, - }) - return true - }) - if len(c.statsToEmit) > 0 { - timeoutCtx, cancel := context.WithTimeout(ctx, EmitTimeout) - log.Info().Str("event_type", c.Type).Int("events_number", len(c.statsToEmit)).Str("trigger", "frequency").Msg("Emitting stats events batch") - if err := c.Emitter.EmitStatistics(timeoutCtx, c.statsToEmit); err != nil { - log.Error().Err(err).Msg("Failed to emit events") - } - cancel() - - c.statsToEmit = make([]model.EventStatistics, 0) - c.statsAggregated.Clear() - counter = 0 - } - c.mu.Unlock() - log.Trace().Msg("Postpone stats event emission") - case <-c.StopChan: - log.Info().Msg("Stopping statistics collector") - ticker.Stop() - return nil - } - } -} - -func (c *StatisticsCollector) GetChannel() channel.CollectorChannel { - return channel.EventStatisticsChannel{Channel: c.StatsChan} -} diff --git a/proxy/config/collector.go b/proxy/config/collector.go index c62a91d6..de0b1593 100644 --- a/proxy/config/collector.go +++ b/proxy/config/collector.go @@ -4,6 +4,7 @@ import ( "errors" "os" "strconv" + "strings" "time" "github.com/ivpn/dns/proxy/model" @@ -12,12 +13,16 @@ import ( type CollectorConfig interface { GetBatchSize() int GetFrequency() time.Duration + GetPopName() string } type BatchCollectorConfig struct { Type string BatchSize int Frequency time.Duration + // PopName labels service-wide statistics documents; set for the + // statistics collector only. + PopName string } func (b *BatchCollectorConfig) GetBatchSize() int { @@ -28,6 +33,10 @@ func (b *BatchCollectorConfig) GetFrequency() time.Duration { return b.Frequency } +func (b *BatchCollectorConfig) GetPopName() string { + return b.PopName +} + func NewCollectorConfig(collectorType string) (CollectorConfig, error) { switch collectorType { case model.TYPE_QUERY_LOGS: @@ -66,7 +75,7 @@ func loadQueryLogsCollectorConfig() (*BatchCollectorConfig, error) { } func loadStatisticsCollectorConfig() (*BatchCollectorConfig, error) { - bs := os.Getenv("COLLECTOR_STATISTICS_BATCH_SIZE") + bs := os.Getenv("COLLECTOR_SERVICE_STATISTICS_BATCH_SIZE") if bs == "" { bs = "10000" } @@ -75,7 +84,7 @@ func loadStatisticsCollectorConfig() (*BatchCollectorConfig, error) { return nil, err } - freq := os.Getenv("COLLECTOR_STATISTICS_BATCH_INTERVAL") + freq := os.Getenv("COLLECTOR_SERVICE_STATISTICS_BATCH_INTERVAL") if freq == "" { freq = "30s" } @@ -88,5 +97,20 @@ func loadStatisticsCollectorConfig() (*BatchCollectorConfig, error) { Type: model.TYPE_STATISTICS, BatchSize: batchSize, Frequency: interval, + PopName: loadPopName(), }, nil } + +const defaultPopName = "unknown" + +// loadPopName is the PoP label on service-wide statistics: POP_NAME, else the +// host name, else "unknown". It never identifies a profile or a client. +func loadPopName() string { + if v := strings.TrimSpace(os.Getenv("POP_NAME")); v != "" { + return v + } + if h, err := os.Hostname(); err == nil && strings.TrimSpace(h) != "" { + return strings.TrimSpace(h) + } + return defaultPopName +} diff --git a/proxy/config/collector_test.go b/proxy/config/collector_test.go new file mode 100644 index 00000000..9d2b8408 --- /dev/null +++ b/proxy/config/collector_test.go @@ -0,0 +1,48 @@ +package config + +import ( + "os" + "testing" + + "github.com/ivpn/dns/proxy/model" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// specRef: proxy-statistics-behaviour.md #Y7 +func TestLoadPopName(t *testing.T) { + host, err := os.Hostname() + require.NoError(t, err) + + tests := []struct { + name string + env string + want string + }{ + {name: "explicit value", env: "ams1", want: "ams1"}, + {name: "surrounding whitespace is trimmed", env: " fra1\t", want: "fra1"}, + {name: "unset falls back to the host name", env: "", want: host}, + {name: "blank falls back to the host name", env: " ", want: host}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Setenv("POP_NAME", tt.env) + assert.Equal(t, tt.want, loadPopName()) + }) + } +} + +// specRef: proxy-statistics-behaviour.md #Y7 +func TestNewCollectorConfig_StatisticsCarriesPopName(t *testing.T) { + t.Setenv("POP_NAME", "syd1") + t.Setenv("COLLECTOR_SERVICE_STATISTICS_BATCH_SIZE", "") + t.Setenv("COLLECTOR_SERVICE_STATISTICS_BATCH_INTERVAL", "") + + cfg, err := NewCollectorConfig(model.TYPE_STATISTICS) + require.NoError(t, err) + assert.Equal(t, "syd1", cfg.GetPopName()) + + logsCfg, err := NewCollectorConfig(model.TYPE_QUERY_LOGS) + require.NoError(t, err) + assert.Empty(t, logsCfg.GetPopName(), "only statistics documents are labelled") +} diff --git a/proxy/config/config.go b/proxy/config/config.go index 5eb0ea1d..495aa507 100644 --- a/proxy/config/config.go +++ b/proxy/config/config.go @@ -11,6 +11,25 @@ import ( "github.com/ivpn/dns/proxy/model" ) +// Response modes for the rate-limit layers (RATELIMIT_PER_IP_RESPONSE, +// RATELIMIT_PER_PROFILE_RESPONSE). +const ( + RateLimitResponseDrop = "drop" + RateLimitResponseRefuse = "refuse" +) + +// Defaults for the profile settings cache and the Redis client. +const ( + defaultProfileSettingsCacheTTL = 30 * time.Second + // defaultProfileSettingsCacheSize counts profiles, not bytes: stale entries are + // kept until evicted, and an entry holds the profile's custom rules, so a + // profile at the 10k-rule ceiling is a few MB while a typical one is a few KB. + defaultProfileSettingsCacheSize = 20_000 + // defaultCacheCommandTimeout suits a PoP-local replica: one dial, read or + // write may take at most this long, with a single retry. + defaultCacheCommandTimeout = time.Second +) + // Config represents the application configuration type Config struct { Server *ServerConfig @@ -57,11 +76,6 @@ type DNSCacheConfig struct { } // Rate limit response modes. -const ( - RateLimitResponseDrop = "drop" - RateLimitResponseRefuse = "refuse" -) - // RateLimitConfig holds rate limiter settings. type RateLimitConfig struct { PerIPEnabled bool @@ -100,7 +114,11 @@ type ServerConfig struct { DnsCheckDomain string DnsCheckPort string ProfileSettingsCacheTTL time.Duration - MaxGoroutines uint // MAX_GOROUTINES - cap on concurrent request-processing goroutines (0 disables) + // ProfileSettingsCacheSize bounds the in-process settings cache (LRU, in + // profiles); entries past the TTL stay until evicted and serve as + // last-known-good. PROFILE_SETTINGS_CACHE_SIZE. + ProfileSettingsCacheSize int + MaxGoroutines uint // MAX_GOROUTINES - cap on concurrent request-processing goroutines (0 disables) } // ServicesConfig configures ASN-based services blocking. @@ -348,8 +366,8 @@ func New() (*Config, error) { dnsCacheCfg := loadDNSCacheConfig() rebindingCfg := loadRebindingConfig() - // Profile settings in-memory cache TTL (default 30s, "0" disables expiration) - profileSettingsCacheTTL := 30 * time.Second + // Profile settings in-memory cache TTL ("0" disables expiration) + profileSettingsCacheTTL := defaultProfileSettingsCacheTTL if v := os.Getenv("PROFILE_SETTINGS_CACHE_TTL"); v != "" { parsed, err := time.ParseDuration(v) if err != nil { @@ -357,6 +375,7 @@ func New() (*Config, error) { } profileSettingsCacheTTL = parsed } + profileSettingsCacheSize := loadProfileSettingsCacheSize() // Get AdGuard log level (default to "info" if not set or invalid) adguardLogLevel := strings.ToLower(os.Getenv("LOG_LEVEL_ADGUARD")) @@ -385,11 +404,12 @@ func New() (*Config, error) { return &Config{ Server: &ServerConfig{ - Names: parseCSV(os.Getenv("SERVER_NAME")), - DnsCheckDomain: dnsCheckDomain, - DnsCheckPort: os.Getenv("DNS_CHECK_PORT"), - ProfileSettingsCacheTTL: profileSettingsCacheTTL, - MaxGoroutines: loadMaxGoroutines(), + Names: parseCSV(os.Getenv("SERVER_NAME")), + DnsCheckDomain: dnsCheckDomain, + DnsCheckPort: os.Getenv("DNS_CHECK_PORT"), + ProfileSettingsCacheTTL: profileSettingsCacheTTL, + ProfileSettingsCacheSize: profileSettingsCacheSize, + MaxGoroutines: loadMaxGoroutines(), }, Services: &ServicesConfig{ CatalogPath: servicesCatalogPath, @@ -404,6 +424,7 @@ func New() (*Config, error) { TrustedProxies: trustedProxies, ProfileIDMinLength: profileIdMinLen, Cache: &cache.Config{ + CommandTimeout: loadCacheCommandTimeout(), Address: os.Getenv("CACHE_ADDRESS"), FailoverAddresses: cacheAddrs, Username: os.Getenv("CACHE_USERNAME"), @@ -514,3 +535,31 @@ func loadMetricsConfig() *MetricsConfig { } return cfg } + +// loadProfileSettingsCacheSize reads PROFILE_SETTINGS_CACHE_SIZE; a missing, +// non-numeric or non-positive value keeps the default. +func loadProfileSettingsCacheSize() int { + v := os.Getenv("PROFILE_SETTINGS_CACHE_SIZE") + if v == "" { + return defaultProfileSettingsCacheSize + } + n, err := strconv.Atoi(v) + if err != nil || n <= 0 { + return defaultProfileSettingsCacheSize + } + return n +} + +// loadCacheCommandTimeout reads CACHE_COMMAND_TIMEOUT (Go duration). Unset or +// invalid keeps the default; "0" hands control back to the go-redis defaults. +func loadCacheCommandTimeout() time.Duration { + v := os.Getenv("CACHE_COMMAND_TIMEOUT") + if v == "" { + return defaultCacheCommandTimeout + } + d, err := time.ParseDuration(v) + if err != nil || d < 0 { + return defaultCacheCommandTimeout + } + return d +} diff --git a/proxy/config/config_test.go b/proxy/config/config_test.go index 1a4a648a..8dfd862f 100644 --- a/proxy/config/config_test.go +++ b/proxy/config/config_test.go @@ -2,6 +2,7 @@ package config import ( "testing" + "time" "github.com/stretchr/testify/assert" ) @@ -26,3 +27,45 @@ func TestLoadMaxGoroutines(t *testing.T) { }) } } + +// specRef: proxy-request-admission-behaviour.md #Q13 +func TestLoadProfileSettingsCacheSize(t *testing.T) { + tests := []struct { + name string + env string + want int + }{ + {name: "default when unset", env: "", want: 20_000}, + {name: "override", env: "5000", want: 5000}, + {name: "zero keeps default", env: "0", want: 20_000}, + {name: "negative keeps default", env: "-1", want: 20_000}, + {name: "non-numeric keeps default", env: "lots", want: 20_000}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Setenv("PROFILE_SETTINGS_CACHE_SIZE", tt.env) + assert.Equal(t, tt.want, loadProfileSettingsCacheSize()) + }) + } +} + +// specRef: proxy-request-admission-behaviour.md #Q12 +func TestLoadCacheCommandTimeout(t *testing.T) { + tests := []struct { + name string + env string + want time.Duration + }{ + {name: "default when unset", env: "", want: time.Second}, + {name: "override", env: "250ms", want: 250 * time.Millisecond}, + {name: "zero defers to client defaults", env: "0", want: 0}, + {name: "invalid keeps default", env: "soon", want: time.Second}, + {name: "negative keeps default", env: "-1s", want: time.Second}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Setenv("CACHE_COMMAND_TIMEOUT", tt.env) + assert.Equal(t, tt.want, loadCacheCommandTimeout()) + }) + } +} diff --git a/proxy/emitter/emitter.go b/proxy/emitter/emitter.go index f00e6536..443e9505 100644 --- a/proxy/emitter/emitter.go +++ b/proxy/emitter/emitter.go @@ -12,7 +12,7 @@ import ( type Emitter interface { EmitQueryLogs(ctx context.Context, data []model.EventQueryLog) error - EmitStatistics(ctx context.Context, data []model.EventStatistics) error + EmitServiceStatistics(ctx context.Context, data []model.ServiceStatistics) error Disconnect() error } diff --git a/proxy/emitter/mongodb/emitter.go b/proxy/emitter/mongodb/emitter.go index 063d3995..8d68226d 100644 --- a/proxy/emitter/mongodb/emitter.go +++ b/proxy/emitter/mongodb/emitter.go @@ -32,11 +32,11 @@ func NewMongoDBEmitter(dbCfg *store.Config) (*MongoDBEmitter, error) { } func (e *MongoDBEmitter) EmitQueryLogs(ctx context.Context, data []model.EventQueryLog) error { - return e.DB.QueryLogsRepository.InsertBatch(ctx, data) + return e.DB.InsertBatch(ctx, data) } -func (e *MongoDBEmitter) EmitStatistics(ctx context.Context, data []model.EventStatistics) error { - return e.DB.StatisticsRepository.InsertBatch(ctx, data) +func (e *MongoDBEmitter) EmitServiceStatistics(ctx context.Context, data []model.ServiceStatistics) error { + return e.DB.AddBatch(ctx, data) } func (e *MongoDBEmitter) Disconnect() error { diff --git a/proxy/emitter/mongodb/mongodb.go b/proxy/emitter/mongodb/mongodb.go index 030d282c..2b1af1c6 100644 --- a/proxy/emitter/mongodb/mongodb.go +++ b/proxy/emitter/mongodb/mongodb.go @@ -7,8 +7,8 @@ import ( ) const ( - collNameQueryLogs = "query_logs" - collNameStats = "statistics" + collNameQueryLogs = "query_logs" + collNameServiceStats = "service_statistics" ) // MongoDB is a MongoDB database instance @@ -17,7 +17,7 @@ type MongoDB struct { dbConfig *store.Config client *mongo.Client *QueryLogsRepository - *StatisticsRepository + *ServiceStatisticsRepository } // NewMongoDB creates a new MongoDB instance @@ -37,9 +37,9 @@ func (db *MongoDB) RegisterRepositories() error { log.Error().Err(err).Msg("Failed to create query logs repository") return err } - db.StatisticsRepository, err = NewStatisticsRepository(db.client, db.dbConfig.Name) + db.ServiceStatisticsRepository, err = NewServiceStatisticsRepository(db.client, db.dbConfig.Name, collNameServiceStats) if err != nil { - log.Error().Err(err).Msg("Failed to create statistics repository") + log.Error().Err(err).Msg("Failed to create service statistics repository") return err } return nil diff --git a/proxy/emitter/mongodb/service_statistics.go b/proxy/emitter/mongodb/service_statistics.go new file mode 100644 index 00000000..c66dcd7e --- /dev/null +++ b/proxy/emitter/mongodb/service_statistics.go @@ -0,0 +1,62 @@ +package mongodb + +import ( + "context" + + "github.com/ivpn/dns/proxy/model" + "github.com/rs/zerolog/log" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +// ServiceStatisticsRepository adds service-wide query counters into one +// document per PoP and hour. It is a regular collection: a time-series +// collection cannot increment measurement fields in place. +type ServiceStatisticsRepository struct { + collName string + coll *mongo.Collection +} + +// NewServiceStatisticsRepository binds the repository to its collection. The +// collection needs no explicit creation, indexes or TTL: documents are keyed +// by _id and kept for good. +func NewServiceStatisticsRepository(client *mongo.Client, dbName, collName string) (*ServiceStatisticsRepository, error) { + return &ServiceStatisticsRepository{ + collName: collName, + coll: client.Database(dbName).Collection(collName), + }, nil +} + +// AddBatch increments each document's counters, creating it on first sight. +// Increments are atomic, so several proxy instances and restarts converge on +// the exact hourly total. +func (r *ServiceStatisticsRepository) AddBatch(ctx context.Context, batch []model.ServiceStatistics) error { + if len(batch) == 0 { + return nil + } + writes := make([]mongo.WriteModel, 0, len(batch)) + for _, doc := range batch { + writes = append(writes, mongo.NewUpdateOneModel(). + SetFilter(bson.D{{Key: "_id", Value: doc.ID}}). + SetUpdate(bson.D{ + {Key: "$inc", Value: bson.D{ + {Key: "queries.total", Value: doc.Queries.Total}, + {Key: "queries.blocked", Value: doc.Queries.Blocked}, + {Key: "queries.dnssec", Value: doc.Queries.DNSSEC}, + }}, + {Key: "$setOnInsert", Value: bson.D{ + {Key: "timestamp", Value: doc.Timestamp}, + {Key: "pop", Value: doc.Pop}, + }}, + }). + SetUpsert(true)) + } + + _, err := r.coll.BulkWrite(ctx, writes, options.BulkWrite().SetOrdered(false)) + if err != nil { + return err + } + log.Info().Str("collection_name", r.collName).Int("batch_size", len(writes)).Msg("Added batch of service stats") + return nil +} diff --git a/proxy/emitter/mongodb/statistics.go b/proxy/emitter/mongodb/statistics.go deleted file mode 100644 index f7439633..00000000 --- a/proxy/emitter/mongodb/statistics.go +++ /dev/null @@ -1,104 +0,0 @@ -package mongodb - -import ( - "context" - - "github.com/ivpn/dns/proxy/model" - "github.com/rs/zerolog/log" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" -) - -const ( - statisticsCollName = "statistics" -) - -// StatisticsRepository is a MongoDB repository for statistics collection -type StatisticsRepository struct { - client *mongo.Client - database *mongo.Database - DbName string - statisticsColl *mongo.Collection -} - -// NewStatisticsRepository creates a new statistics instance -func NewStatisticsRepository(client *mongo.Client, dbName string) (*StatisticsRepository, error) { - database := client.Database(dbName) - - coll := client.Database(dbName).Collection(statisticsCollName) - - repo := &StatisticsRepository{ - client: client, - database: database, - DbName: dbName, - statisticsColl: coll, - } - if err := repo.createStatisticsCollection(context.Background()); err != nil { - return nil, err - } - - return repo, nil -} - -// InsertBatch upserts a batch of profile statistics -func (r *StatisticsRepository) InsertBatch(ctx context.Context, batch []model.EventStatistics) error { - statsDocs := make([]any, 0) - - for _, event := range batch { - event.Statistics.ID = primitive.NewObjectID() - statsDocs = append(statsDocs, event.Statistics) - } - - if len(statsDocs) > 0 { - _, err := r.statisticsColl.InsertMany(ctx, statsDocs, &options.InsertManyOptions{ - Ordered: new(bool), - }) - if err != nil { - return err - } - log.Info().Str("collection_name", statisticsCollName).Int("batch_size", len(statsDocs)).Msgf("Inserted batch of user stats") - } - - return nil -} - -func (r *StatisticsRepository) createStatisticsCollection(ctx context.Context) error { - existingCollNames, err := r.database.ListCollectionNames(ctx, bson.D{}, nil) - if err != nil { - log.Err(err).Msg("Error listing collection names") - return err - } - defer func() { - if err == nil { - r.statisticsColl = r.client.Database(r.DbName).Collection(statisticsCollName) - } - }() - - // Timeseries collections must be explicitly created - collExists := contains(existingCollNames, statisticsCollName) - if collExists { - log.Info().Msgf("%s collection already exists. continuing.", statisticsCollName) - return nil - } - err = r.database.CreateCollection( - ctx, - statisticsCollName, - &options.CreateCollectionOptions{ - TimeSeriesOptions: &options.TimeSeriesOptions{ - TimeField: timeField, - MetaField: &metafieldProfileId, - Granularity: &granularityMinutes, - }, - ExpireAfterSeconds: &expirationOneMonth, - }, - ) - if err != nil { - log.Err(err).Msgf("Error creating collection [%s]", statisticsCollName) - return err - } else { - log.Info().Msgf("Successfully created %s collection for the first time.", statisticsCollName) - } - return nil -} diff --git a/proxy/filter/blocklists.go b/proxy/filter/blocklists.go index 1e5cb93f..0972b88b 100644 --- a/proxy/filter/blocklists.go +++ b/proxy/filter/blocklists.go @@ -34,51 +34,96 @@ type blocklistMatch struct { // and the IP phase (CNAME targets). func matchDomainAgainstBlocklists(ctx context.Context, c cache.Cache, reqCtx *requestcontext.RequestContext, blocklists []string, fqdn string) (*blocklistMatch, error) { for _, blocklistId := range blocklists { - // check exact match first - blocklisted, err := c.GetBlocklistEntry(ctx, blocklistId, fqdn) + match, err := matchDomainAgainstBlocklist(ctx, c, reqCtx, blocklistId, fqdn) if err != nil { return nil, err } - if blocklisted { - return &blocklistMatch{blocklistID: blocklistId}, nil + if match == nil { + continue } - if reqCtx.PrivacySettings[SUBDOMAINS_RULE] == RULE_BLOCK { - // iterate over all parent domains, excluding the TLD and the full - // FQDN (already covered by the exact-match check above) - parts := strings.Split(fqdn, ".") - var candidate string - for i := len(parts) - 2; i >= 1; i-- { - // Build candidate incrementally by prepending current part - if i == len(parts)-2 { - candidate = parts[i] + "." + parts[i+1] - } else { - candidate = parts[i] + "." + candidate - } - - // now, check if candidate domain is part of any blocklist entry - blocklisted, err = c.GetBlocklistEntry(ctx, blocklistId, candidate) - if err != nil { - return nil, err - } - e := reqCtx.Logger.Trace().Bool("blocklisted", blocklisted).Str("blocklist", blocklistId) - reqCtx.MaybeDomain(e, "candidate", candidate).Msg("Candidate domain") - - if blocklisted { - return &blocklistMatch{blocklistID: blocklistId, viaParent: true}, nil - } - } + // The list's own exception set (its @@ rules) withdraws the match; + // scoped per source, so the remaining subscribed lists still get + // checked. This consult lives inside the stage on purpose: the + // aggregator resolves any Allow over every Block, so a list-level + // allow stage would override user custom block rules. + excepted, err := matchDomainAgainstExceptions(ctx, c, blocklistId, fqdn) + if err != nil { + return nil, err } + if excepted { + e := reqCtx.Logger.Debug().Str("blocklist", blocklistId) + reqCtx.MaybeDomain(e, "domain", fqdn).Msg("Blocklist match withdrawn by the list's exception") + continue + } + return match, nil } return nil, nil } -func (f *DomainFilter) filterBlocklists(reqCtx *requestcontext.RequestContext, dctx *proxy.DNSContext) (*model.StageResult, error) { - defer sentry.Recover() - blocklists, err := f.Cache.GetProfileBlocklists(context.Background(), reqCtx.ProfileId) +// matchDomainAgainstBlocklist checks fqdn against a single list: exact +// membership first, then the parent-domain walk when the profile's subdomains +// rule is set to block. +func matchDomainAgainstBlocklist(ctx context.Context, c cache.Cache, reqCtx *requestcontext.RequestContext, blocklistId string, fqdn string) (*blocklistMatch, error) { + // check exact match first + blocklisted, err := c.GetBlocklistEntry(ctx, blocklistId, fqdn) if err != nil { return nil, err } + if blocklisted { + return &blocklistMatch{blocklistID: blocklistId}, nil + } + + if reqCtx.PrivacySettings[SUBDOMAINS_RULE] == RULE_BLOCK { + // iterate over all parent domains, excluding the TLD and the full + // FQDN (already covered by the exact-match check above) + parts := strings.Split(fqdn, ".") + var candidate string + for i := len(parts) - 2; i >= 1; i-- { + // Build candidate incrementally by prepending current part + if i == len(parts)-2 { + candidate = parts[i] + "." + parts[i+1] + } else { + candidate = parts[i] + "." + candidate + } + + // now, check if candidate domain is part of any blocklist entry + blocklisted, err = c.GetBlocklistEntry(ctx, blocklistId, candidate) + if err != nil { + return nil, err + } + e := reqCtx.Logger.Trace().Bool("blocklisted", blocklisted).Str("blocklist", blocklistId) + reqCtx.MaybeDomain(e, "candidate", candidate).Msg("Candidate domain") + + if blocklisted { + return &blocklistMatch{blocklistID: blocklistId, viaParent: true}, nil + } + } + } + return nil, nil +} + +// matchDomainAgainstExceptions reports whether fqdn or any of its parent +// domains (down to two labels) is in the list's exception set. The walk is +// unconditional: an adblock exception (`@@||d^`) covers d and its subdomains +// regardless of the profile's blocklists_subdomains_rule. Runs only on the +// would-block path, so its lookups never touch the common allow path. +func matchDomainAgainstExceptions(ctx context.Context, c cache.Cache, blocklistId string, fqdn string) (bool, error) { + parts := strings.Split(fqdn, ".") + for i := 0; i <= len(parts)-2; i++ { + excepted, err := c.GetBlocklistExceptionEntry(ctx, blocklistId, strings.Join(parts[i:], ".")) + if err != nil { + return false, err + } + if excepted { + return true, nil + } + } + return false, nil +} + +func (f *DomainFilter) filterBlocklists(ctx context.Context, reqCtx *requestcontext.RequestContext, dctx *proxy.DNSContext) (*model.StageResult, error) { + defer sentry.Recover() question := dctx.Req.Question[0].Name // answer only first question - google dns does the same @@ -91,7 +136,7 @@ func (f *DomainFilter) filterBlocklists(reqCtx *requestcontext.RequestContext, d result := &model.StageResult{Decision: model.DecisionNone, Tier: TierBlocklists} - match, err := matchDomainAgainstBlocklists(context.Background(), f.Cache, reqCtx, blocklists, fqdn) + match, err := matchDomainAgainstBlocklists(ctx, f.Cache, reqCtx, reqCtx.Blocklists, fqdn) if err != nil { return nil, err } diff --git a/proxy/filter/blocklists_benchmark_test.go b/proxy/filter/blocklists_benchmark_test.go index c78f22df..13ca4b21 100644 --- a/proxy/filter/blocklists_benchmark_test.go +++ b/proxy/filter/blocklists_benchmark_test.go @@ -88,3 +88,21 @@ func TestSubdomainCandidatesEquivalence(t *testing.T) { } } } + +// BenchmarkExceptionWalkCandidates isolates the string construction of the +// exception walk (full FQDN plus every parent down to two labels), which runs +// only on the would-block path; the Redis SISMEMBER per candidate dominates +// in production, exactly as with the block walk above. +func BenchmarkExceptionWalkCandidates(b *testing.B) { + for _, tc := range subdomainBenchDomains { + b.Run(tc.name, func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + parts := strings.Split(tc.fqdn, ".") + for j := 0; j <= len(parts)-2; j++ { + benchCandidateSink = strings.Join(parts[j:], ".") + } + } + }) + } +} diff --git a/proxy/filter/blocklists_case_test.go b/proxy/filter/blocklists_case_test.go index 494b5aaa..51e13728 100644 --- a/proxy/filter/blocklists_case_test.go +++ b/proxy/filter/blocklists_case_test.go @@ -1,6 +1,7 @@ package filter import ( + "context" "testing" "github.com/AdguardTeam/dnsproxy/proxy" @@ -96,8 +97,6 @@ func TestFilterBlocklistsIsCaseInsensitive(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { mockCache := new(mocks.Cache) - mockCache.On("GetProfileBlocklists", mock.Anything, "profile1"). - Return([]string{blocklistID}, nil) // Model Redis SISMEMBER: exact byte match on the stored (lowercase) // members, everything else is a miss. Specific expectations are @@ -108,6 +107,9 @@ func TestFilterBlocklistsIsCaseInsensitive(t *testing.T) { } mockCache.On("GetBlocklistEntry", mock.Anything, blocklistID, mock.Anything). Return(false, nil) + // No exceptions published in these cases (specRef: #X5). + mockCache.On("GetBlocklistExceptionEntry", mock.Anything, mock.Anything, mock.Anything). + Return(false, nil).Maybe() fm := NewDomainFilter(&proxy.Proxy{}, mockCache, nil) @@ -117,11 +119,12 @@ func TestFilterBlocklistsIsCaseInsensitive(t *testing.T) { loggerFactory := logging.NewFactory(zerolog.DebugLevel) reqCtx := &requestcontext.RequestContext{ ProfileId: "profile1", + Blocklists: []string{blocklistID}, PrivacySettings: tt.privacySettings, Logger: loggerFactory.ForProfile("profile1", true), } - result, err := fm.filterBlocklists(reqCtx, &proxy.DNSContext{Req: msg}) + result, err := fm.filterBlocklists(context.Background(), reqCtx, &proxy.DNSContext{Req: msg}) assert.NoError(t, err) assert.NotNil(t, result) @@ -195,17 +198,14 @@ func TestServiceDomainMatchingIsCaseInsensitive(t *testing.T) { for _, tt := range tests { t.Run(tt.qname, func(t *testing.T) { - mockCache := new(mocks.Cache) - mockCache.On("GetProfileServicesBlocked", mock.Anything, "test-profile"). - Return([]string{"microsoft"}, nil) - - fm := &DomainFilter{Cache: mockCache, ServicesCatalog: staticCatalog{cat: catalog}} + fm := &DomainFilter{Cache: mocks.NewCache(t), ServicesCatalog: staticCatalog{cat: catalog}} msg := new(dns.Msg) msg.SetQuestion(tt.qname, dns.TypeA) - result, err := fm.filterServiceDomains(newTestReqCtx(t, "test-profile"), - &proxy.DNSContext{Req: msg}) + reqCtx := newTestReqCtx(t, "test-profile") + reqCtx.BlockedServices = []string{"microsoft"} + result, err := fm.filterServiceDomains(context.Background(), reqCtx, &proxy.DNSContext{Req: msg}) assert.NoError(t, err) assert.Equal(t, tt.expect, result.Decision, diff --git a/proxy/filter/blocklists_test.go b/proxy/filter/blocklists_test.go index bd83335a..16ba2c4b 100644 --- a/proxy/filter/blocklists_test.go +++ b/proxy/filter/blocklists_test.go @@ -1,6 +1,7 @@ package filter import ( + "context" "errors" "testing" @@ -108,18 +109,8 @@ func TestFilterBlocklists(t *testing.T) { expectErr: false, }, { - name: "Cache error on GetProfileBlocklists", - profileID: "profile6", - questionDomain: "foo.com", - blocklists: nil, - blocklistEntries: map[string]map[string]bool{}, - privacySettings: map[string]string{}, - expectBlocked: false, - expectReasons: nil, - expectErr: true, - cacheErr: errors.New("cache error"), - }, - { + // The subscription list travels on the request context; the only + // store read left in the stage is the membership lookup. name: "Cache error on GetBlocklistEntry", profileID: "profile7", questionDomain: "foo.com", @@ -139,15 +130,6 @@ func TestFilterBlocklists(t *testing.T) { t.Run(tt.name, func(t *testing.T) { mockCache := new(mocks.Cache) - // Setup GetProfileBlocklists - if tt.cacheErr != nil && (tt.name == "Cache error on GetProfileBlocklists") { - mockCache.On("GetProfileBlocklists", mock.Anything, tt.profileID). - Return(nil, tt.cacheErr) - } else { - mockCache.On("GetProfileBlocklists", mock.Anything, tt.profileID). - Return(tt.blocklists, nil) - } - if tt.name == "Multiple blocklists - first blocks" { entries := tt.blocklistEntries[blocklistID1] var blocked bool @@ -181,6 +163,11 @@ func TestFilterBlocklists(t *testing.T) { } } + // No case in this table publishes exceptions; the consult on the + // would-block path must see an absent set (specRef: #X5). + mockCache.On("GetBlocklistExceptionEntry", mock.Anything, mock.Anything, mock.Anything). + Return(false, nil).Maybe() + dnsProxy := &proxy.Proxy{} fm := NewDomainFilter(dnsProxy, mockCache, nil) @@ -193,6 +180,7 @@ func TestFilterBlocklists(t *testing.T) { reqCtx := &requestcontext.RequestContext{ ProfileId: tt.profileID, + Blocklists: tt.blocklists, PrivacySettings: tt.privacySettings, Logger: testLogger, } @@ -200,7 +188,7 @@ func TestFilterBlocklists(t *testing.T) { Req: msg, } - result, err := fm.filterBlocklists(reqCtx, dnsCtx) + result, err := fm.filterBlocklists(context.Background(), reqCtx, dnsCtx) if tt.expectErr { assert.Error(t, err) return @@ -220,3 +208,143 @@ func TestFilterBlocklists(t *testing.T) { }) } } + +// TestFilterBlocklists_Exceptions covers the list-level exception consult: +// a match from list L is withdrawn when L's own exception set covers the +// query name, without creating an Allow and without weakening other lists. +func TestFilterBlocklists_Exceptions(t *testing.T) { + const ( + listL = "adguard_dns_filter" + listM = "hagezi_pro" + ) + + tests := []struct { + name string + questionDomain string + blocklists []string + blockEntries map[string]map[string]bool // list -> domain -> blocked + exceptions map[string]map[string]bool // list -> domain -> excepted + privacySettings map[string]string + exceptionErr error + expectBlocked bool + expectReasons []string + expectErr bool + }{ + { + // specRef: #X1 — the list's own unblock withdraws the exact match. + name: "exact match suppressed by same-list exception", + questionDomain: "data.orders.costco.com", + blocklists: []string{listL}, + blockEntries: map[string]map[string]bool{listL: {"data.orders.costco.com": true}}, + exceptions: map[string]map[string]bool{listL: {"data.orders.costco.com": true}}, + expectBlocked: false, + }, + { + // specRef: #X2 — exception on the query name suppresses a + // parent-walk hit under blocklists_subdomains_rule = block. + name: "parent-walk match suppressed by exception on query name", + questionDomain: "sbs.demdex.net", + blocklists: []string{listL}, + blockEntries: map[string]map[string]bool{listL: {"demdex.net": true}}, + exceptions: map[string]map[string]bool{listL: {"sbs.demdex.net": true}}, + privacySettings: map[string]string{SUBDOMAINS_RULE: RULE_BLOCK}, + expectBlocked: false, + }, + { + // specRef: #X3 — the exception walk covers subdomains of the + // excepted name regardless of the subdomains rule. + name: "exact match suppressed by exception on parent", + questionDomain: "x.sbs.demdex.net", + blocklists: []string{listL}, + blockEntries: map[string]map[string]bool{listL: {"x.sbs.demdex.net": true}}, + exceptions: map[string]map[string]bool{listL: {"sbs.demdex.net": true}}, + expectBlocked: false, + }, + { + // specRef: #X4 — per-source scoping: L's exception cannot weaken + // M; scanning continues and M's block stands. + name: "exception scoped to its list, other list still blocks", + questionDomain: "tracker.example.com", + blocklists: []string{listL, listM}, + blockEntries: map[string]map[string]bool{ + listL: {"tracker.example.com": true}, + listM: {"tracker.example.com": true}, + }, + exceptions: map[string]map[string]bool{listL: {"tracker.example.com": true}}, + expectBlocked: true, + expectReasons: []string{"blocklist: " + listM}, + }, + { + // specRef: #X5 — no exception set published: unchanged blocking. + name: "no exceptions published, block stands", + questionDomain: "blocked.example.com", + blocklists: []string{listL}, + blockEntries: map[string]map[string]bool{listL: {"blocked.example.com": true}}, + expectBlocked: true, + expectReasons: []string{"blocklist: " + listL}, + }, + { + // specRef: #X6 — exception lookup errors propagate like block + // lookup errors. + name: "exception lookup error propagates", + questionDomain: "blocked.example.com", + blocklists: []string{listL}, + blockEntries: map[string]map[string]bool{listL: {"blocked.example.com": true}}, + exceptionErr: errors.New("exception lookup error"), + expectErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mockCache := new(mocks.Cache) + for _, blID := range tt.blocklists { + entries := tt.blockEntries[blID] + mockCache.On("GetBlocklistEntry", mock.Anything, blID, mock.MatchedBy(func(string) bool { return true })). + Return(func(_ context.Context, blocklistId, domain string) (bool, error) { + return entries[domain], nil + }) + excepted := tt.exceptions[blID] + if tt.exceptionErr != nil { + mockCache.On("GetBlocklistExceptionEntry", mock.Anything, blID, mock.Anything). + Return(false, tt.exceptionErr) + } else { + mockCache.On("GetBlocklistExceptionEntry", mock.Anything, blID, mock.Anything). + Return(func(_ context.Context, blocklistId, domain string) (bool, error) { + return excepted[domain], nil + }).Maybe() + } + } + + fm := NewDomainFilter(&proxy.Proxy{}, mockCache, nil) + + msg := new(dns.Msg) + msg.SetQuestion(tt.questionDomain+".", dns.TypeA) + loggerFactory := logging.NewFactory(zerolog.DebugLevel) + reqCtx := &requestcontext.RequestContext{ + ProfileId: "profileX", + PrivacySettings: tt.privacySettings, + Blocklists: tt.blocklists, + Logger: loggerFactory.ForProfile("profileX", true), + } + dnsCtx := &proxy.DNSContext{Req: msg} + + result, err := fm.filterBlocklists(context.Background(), reqCtx, dnsCtx) + if tt.expectErr { + assert.Error(t, err) + return + } + assert.NoError(t, err) + assert.NotNil(t, result) + if tt.expectBlocked { + assert.Equal(t, model.DecisionBlock, result.Decision) + assert.Equal(t, tt.expectReasons, result.Reasons) + } else { + // A suppressed match must leave the decision at None — an + // exception never produces an Allow. + assert.Equal(t, model.DecisionNone, result.Decision) + assert.Empty(t, result.Reasons) + } + }) + } +} diff --git a/proxy/filter/cname.go b/proxy/filter/cname.go index 8f7940fc..5bfcd14c 100644 --- a/proxy/filter/cname.go +++ b/proxy/filter/cname.go @@ -51,7 +51,7 @@ func extractCNAMETargets(answers []dns.RR, qname string) []string { // custom Block (T200) > blocklist Block (T100). Custom rules are therefore // always evaluated, while blocklist lookups are skipped once a custom rule // has decided. -func (f *IPFilter) filterCNAME(reqCtx *requestcontext.RequestContext, dctx *proxy.DNSContext) (*model.StageResult, error) { +func (f *IPFilter) filterCNAME(ctx context.Context, reqCtx *requestcontext.RequestContext, dctx *proxy.DNSContext) (*model.StageResult, error) { defer sentry.Recover() result := &model.StageResult{Decision: model.DecisionNone, Tier: TierBlocklists} @@ -68,16 +68,8 @@ func (f *IPFilter) filterCNAME(reqCtx *requestcontext.RequestContext, dctx *prox return result, nil } - customRuleHashes, err := f.Cache.GetCustomRulesHashes(context.Background(), reqCtx.ProfileId) - if err != nil { - return nil, err - } allowMatched, blockMatched := false, false - for _, customRuleHash := range customRuleHashes { - hash, err := f.Cache.GetCustomRulesHash(context.Background(), customRuleHash) - if err != nil { - return nil, err - } + for _, hash := range reqCtx.CustomRules { for _, target := range targets { if matchDomainPattern(&f.patternCache, target, hash["value"]) { switch hash["action"] { @@ -105,12 +97,8 @@ func (f *IPFilter) filterCNAME(reqCtx *requestcontext.RequestContext, dctx *prox return result, nil } - blocklists, err := f.Cache.GetProfileBlocklists(context.Background(), reqCtx.ProfileId) - if err != nil { - return nil, err - } for _, target := range targets { - match, err := matchDomainAgainstBlocklists(context.Background(), f.Cache, reqCtx, blocklists, target) + match, err := matchDomainAgainstBlocklists(ctx, f.Cache, reqCtx, reqCtx.Blocklists, target) if err != nil { return nil, err } diff --git a/proxy/filter/cname_benchmark_test.go b/proxy/filter/cname_benchmark_test.go index 304d1aef..a24e7c78 100644 --- a/proxy/filter/cname_benchmark_test.go +++ b/proxy/filter/cname_benchmark_test.go @@ -1,6 +1,7 @@ package filter import ( + "context" "testing" "github.com/AdguardTeam/dnsproxy/proxy" @@ -49,7 +50,7 @@ func BenchmarkFilterCNAME_EarlyExit(b *testing.B) { b.ReportAllocs() for i := 0; i < b.N; i++ { - result, err := f.filterCNAME(reqCtx, dnsCtx) + result, err := f.filterCNAME(context.Background(), reqCtx, dnsCtx) if err != nil || result == nil { b.Fatal("unexpected filterCNAME result") } diff --git a/proxy/filter/cname_test.go b/proxy/filter/cname_test.go index e7ed6945..2df8f153 100644 --- a/proxy/filter/cname_test.go +++ b/proxy/filter/cname_test.go @@ -1,6 +1,7 @@ package filter import ( + "context" "net" "testing" @@ -137,9 +138,9 @@ func TestFilterCNAME(t *testing.T) { wantDecision: model.DecisionNone, }, { - name: "U2 — target on subscribed blocklist: Block T100", - tableRef: "F/U2", - response: buildCNAMEChainResponse("metrics.shop.example", dns.TypeA, []string{"tracker.evil.net"}, "1.2.3.4"), + name: "U2 — target on subscribed blocklist: Block T100", + tableRef: "F/U2", + response: buildCNAMEChainResponse("metrics.shop.example", dns.TypeA, []string{"tracker.evil.net"}, "1.2.3.4"), blocklists: []string{blocklistID}, blocklistEntries: map[string]map[string]bool{ blocklistID: {"tracker.evil.net": true}, @@ -151,9 +152,9 @@ func TestFilterCNAME(t *testing.T) { wantReasons: []string{"blocklist: " + blocklistID, REASON_CNAME_UNCLOAKING}, }, { - name: "U3 — intermediate chain name on blocklist: Block T100", - tableRef: "F/U3", - response: buildCNAMEChainResponse("metrics.shop.example", dns.TypeA, []string{"tracker.evil.net", "edge.clean-cdn.example"}, "1.2.3.4"), + name: "U3 — intermediate chain name on blocklist: Block T100", + tableRef: "F/U3", + response: buildCNAMEChainResponse("metrics.shop.example", dns.TypeA, []string{"tracker.evil.net", "edge.clean-cdn.example"}, "1.2.3.4"), blocklists: []string{blocklistID}, blocklistEntries: map[string]map[string]bool{ blocklistID: {"tracker.evil.net": true}, @@ -165,9 +166,9 @@ func TestFilterCNAME(t *testing.T) { wantReasons: []string{"blocklist: " + blocklistID, REASON_CNAME_UNCLOAKING}, }, { - name: "U4 — parent of target on blocklist, subdomains rule on: Block T100 + subdomains reason", - tableRef: "F/U4", - response: buildCNAMEChainResponse("metrics.shop.example", dns.TypeA, []string{"sub.tracker-park.net"}, "1.2.3.4"), + name: "U4 — parent of target on blocklist, subdomains rule on: Block T100 + subdomains reason", + tableRef: "F/U4", + response: buildCNAMEChainResponse("metrics.shop.example", dns.TypeA, []string{"sub.tracker-park.net"}, "1.2.3.4"), blocklists: []string{blocklistID}, blocklistEntries: map[string]map[string]bool{ blocklistID: {"tracker-park.net": true}, @@ -179,9 +180,9 @@ func TestFilterCNAME(t *testing.T) { wantReasons: []string{"blocklist: " + blocklistID, SUBDOMAINS_RULE, REASON_CNAME_UNCLOAKING}, }, { - name: "U5 — parent of target on blocklist, subdomains rule off: None", - tableRef: "F/U5", - response: buildCNAMEChainResponse("metrics.shop.example", dns.TypeA, []string{"sub.tracker-park.net"}, "1.2.3.4"), + name: "U5 — parent of target on blocklist, subdomains rule off: None", + tableRef: "F/U5", + response: buildCNAMEChainResponse("metrics.shop.example", dns.TypeA, []string{"sub.tracker-park.net"}, "1.2.3.4"), blocklists: []string{blocklistID}, blocklistEntries: map[string]map[string]bool{ blocklistID: {"tracker-park.net": true}, @@ -191,9 +192,9 @@ func TestFilterCNAME(t *testing.T) { wantDecision: model.DecisionNone, }, { - name: "U6 — target only on an unsubscribed list: None", - tableRef: "F/U6", - response: buildCNAMEChainResponse("metrics.shop.example", dns.TypeA, []string{"tracker.evil.net"}, "1.2.3.4"), + name: "U6 — target only on an unsubscribed list: None", + tableRef: "F/U6", + response: buildCNAMEChainResponse("metrics.shop.example", dns.TypeA, []string{"tracker.evil.net"}, "1.2.3.4"), blocklists: []string{blocklistID}, blocklistEntries: map[string]map[string]bool{ blocklistID: {}, // subscribed list does not contain the target @@ -203,9 +204,9 @@ func TestFilterCNAME(t *testing.T) { wantDecision: model.DecisionNone, }, { - name: "U7 — target matches custom Block rule (wildcard): Block T200", - tableRef: "F/U7", - response: buildCNAMEChainResponse("metrics.shop.example", dns.TypeA, []string{"sub.tracker.net"}, "1.2.3.4"), + name: "U7 — target matches custom Block rule (wildcard): Block T200", + tableRef: "F/U7", + response: buildCNAMEChainResponse("metrics.shop.example", dns.TypeA, []string{"sub.tracker.net"}, "1.2.3.4"), blocklists: []string{}, privacySettings: map[string]string{}, customHashes: []string{"h1"}, @@ -217,9 +218,9 @@ func TestFilterCNAME(t *testing.T) { wantReasons: []string{REASON_CUSTOM_RULES, REASON_CNAME_UNCLOAKING}, }, { - name: "U8 — target matches custom Allow rule and a blocklist: Allow T200 wins", - tableRef: "F/U8", - response: buildCNAMEChainResponse("metrics.shop.example", dns.TypeA, []string{"tracker.evil.net"}, "1.2.3.4"), + name: "U8 — target matches custom Allow rule and a blocklist: Allow T200 wins", + tableRef: "F/U8", + response: buildCNAMEChainResponse("metrics.shop.example", dns.TypeA, []string{"tracker.evil.net"}, "1.2.3.4"), blocklists: []string{blocklistID}, blocklistEntries: map[string]map[string]bool{ blocklistID: {"tracker.evil.net": true}, @@ -251,9 +252,9 @@ func TestFilterCNAME(t *testing.T) { wantDecision: model.DecisionNone, }, { - name: "U12 — HTTPS qtype answer carrying a CNAME: Block T100", - tableRef: "F/U12", - response: buildCNAMEChainResponse("metrics.shop.example", dns.TypeHTTPS, []string{"tracker.evil.net"}, ""), + name: "U12 — HTTPS qtype answer carrying a CNAME: Block T100", + tableRef: "F/U12", + response: buildCNAMEChainResponse("metrics.shop.example", dns.TypeHTTPS, []string{"tracker.evil.net"}, ""), blocklists: []string{blocklistID}, blocklistEntries: map[string]map[string]bool{ blocklistID: {"tracker.evil.net": true}, @@ -288,14 +289,6 @@ func TestFilterCNAME(t *testing.T) { mockCache := new(mocks.Cache) if !tt.expectNoCacheCalls { - mockCache.On("GetCustomRulesHashes", mock.Anything, profileID). - Return(tt.customHashes, nil).Maybe() - for hash, rule := range tt.customRules { - mockCache.On("GetCustomRulesHash", mock.Anything, hash). - Return(rule, nil).Maybe() - } - mockCache.On("GetProfileBlocklists", mock.Anything, profileID). - Return(tt.blocklists, nil).Maybe() for blID, entries := range tt.blocklistEntries { for domain, blocked := range entries { mockCache.On("GetBlocklistEntry", mock.Anything, blID, domain). @@ -304,6 +297,8 @@ func TestFilterCNAME(t *testing.T) { } mockCache.On("GetBlocklistEntry", mock.Anything, mock.Anything, mock.Anything). Return(false, nil).Maybe() + mockCache.On("GetBlocklistExceptionEntry", mock.Anything, mock.Anything, mock.Anything). + Return(false, nil).Maybe() } f := NewIPFilter(&proxy.Proxy{}, mockCache, nil, nil, nil, @@ -312,6 +307,8 @@ func TestFilterCNAME(t *testing.T) { loggerFactory := logging.NewFactory(zerolog.DebugLevel) reqCtx := &requestcontext.RequestContext{ ProfileId: profileID, + Blocklists: tt.blocklists, + CustomRules: orderedRules(tt.customHashes, tt.customRules), PrivacySettings: tt.privacySettings, Logger: loggerFactory.ForProfile(profileID, true), } @@ -327,7 +324,7 @@ func TestFilterCNAME(t *testing.T) { dnsCtx.Res = tt.response } - result, err := f.filterCNAME(reqCtx, dnsCtx) + result, err := f.filterCNAME(context.Background(), reqCtx, dnsCtx) assert.NoError(t, err) assert.NotNil(t, result) assert.Equal(t, tt.wantDecision, result.Decision, "tableRef %s", tt.tableRef) @@ -375,20 +372,17 @@ func TestIPFilter_CrossPhase_CNAMEUncloaking(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { mockCache := new(mocks.Cache) - mockCache.On("GetProfileServicesBlocked", mock.Anything, profileID). - Return([]string{}, nil) - mockCache.On("GetCustomRulesHashes", mock.Anything, profileID). - Return([]string{}, nil) - mockCache.On("GetProfileBlocklists", mock.Anything, profileID). - Return([]string{blocklistID}, nil) mockCache.On("GetBlocklistEntry", mock.Anything, blocklistID, "tracker.evil.net"). Return(true, nil) mockCache.On("GetBlocklistEntry", mock.Anything, mock.Anything, mock.Anything). Return(false, nil).Maybe() + mockCache.On("GetBlocklistExceptionEntry", mock.Anything, mock.Anything, mock.Anything). + Return(false, nil).Maybe() ipFilter := NewIPFilter(&proxy.Proxy{}, mockCache, nil, nil, nil, nil) reqCtx := newTestReqCtx(t, profileID) + reqCtx.Blocklists = []string{blocklistID} reqCtx.PartialFilteringResults = append( reqCtx.PartialFilteringResults, tt.domainResults..., ) @@ -398,7 +392,7 @@ func TestIPFilter_CrossPhase_CNAMEUncloaking(t *testing.T) { req.SetQuestion("metrics.shop.example.", dns.TypeA) dnsCtx := &proxy.DNSContext{Req: req, Res: res} - err := ipFilter.Execute(reqCtx, dnsCtx) + err := ipFilter.Execute(context.Background(), reqCtx, dnsCtx) assert.NoError(t, err) assert.Equal(t, tt.wantStatus, reqCtx.FilterResult.Status, "tableRef %s", tt.tableRef) for _, r := range tt.wantContains { diff --git a/proxy/filter/cross_phase_aggregation_test.go b/proxy/filter/cross_phase_aggregation_test.go index c2d37bd1..a24b3383 100644 --- a/proxy/filter/cross_phase_aggregation_test.go +++ b/proxy/filter/cross_phase_aggregation_test.go @@ -1,6 +1,7 @@ package filter import ( + "context" "net" "net/netip" "testing" @@ -10,7 +11,6 @@ import ( "github.com/ivpn/dns/proxy/model" "github.com/miekg/dns" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" ) // domainAllowResult returns a domain-phase Allow StageResult at TierCustomRules. @@ -336,34 +336,21 @@ func TestIPFilter_CrossPhaseAggregation(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - mockCache := new(mocks.Cache) - - // Services cache setup - if len(tt.blockedServiceIDs) > 0 { - mockCache.On("GetProfileServicesBlocked", mock.Anything, profileID). - Return(tt.blockedServiceIDs, nil) - } else { - mockCache.On("GetProfileServicesBlocked", mock.Anything, profileID). - Return([]string{}, nil) - } - - // Custom rules cache setup - mockCache.On("GetCustomRulesHashes", mock.Anything, profileID). - Return(tt.customHashes, nil) - for hash, rule := range tt.customRules { - mockCache.On("GetCustomRulesHash", mock.Anything, hash). - Return(rule, nil).Maybe() - } + // Per-profile inputs travel on the request context; the strict mock + // fails if any stage reaches for the store. + mockCache := mocks.NewCache(t) ipFilter := NewIPFilter(&proxy.Proxy{}, mockCache, tt.catalog, tt.asnLookup, nil, nil) reqCtx := newTestReqCtx(t, profileID) + reqCtx.BlockedServices = tt.blockedServiceIDs + reqCtx.CustomRules = orderedRules(tt.customHashes, tt.customRules) // Pre-populate with domain-phase results to simulate the real pipeline. reqCtx.PartialFilteringResults = append( reqCtx.PartialFilteringResults, tt.domainResults..., ) - err := ipFilter.Execute(reqCtx, tt.dnsCtx) + err := ipFilter.Execute(context.Background(), reqCtx, tt.dnsCtx) assert.NoError(t, err) assert.Equal(t, tt.wantStatus, reqCtx.FilterResult.Status, "table %s: expected status %s", tt.tableRef, tt.wantStatus) @@ -383,8 +370,6 @@ func TestIPFilter_CrossPhaseAggregation(t *testing.T) { assert.Contains(t, reqCtx.PartialFilteringResults, dr, "table %s: domain result should remain in PartialFilteringResults", tt.tableRef) } - - mockCache.AssertExpectations(t) }) } } @@ -417,19 +402,13 @@ func TestIPFilter_RebindingCrossPhase(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - mockCache := new(mocks.Cache) - mockCache.On("GetProfileServicesBlocked", mock.Anything, profileID). - Return([]string{}, nil).Maybe() - mockCache.On("GetCustomRulesHashes", mock.Anything, profileID). - Return([]string{}, nil).Maybe() - - ipFilter := NewIPFilter(&proxy.Proxy{}, mockCache, nil, nil, defaultRebindingConfig(), nil) + ipFilter := NewIPFilter(&proxy.Proxy{}, mocks.NewCache(t), nil, nil, defaultRebindingConfig(), nil) reqCtx := newTestReqCtx(t, profileID) reqCtx.RebindingProtectionSettings = map[string]string{"enabled": "1"} reqCtx.PartialFilteringResults = append(reqCtx.PartialFilteringResults, tt.domainResults...) - err := ipFilter.Execute(reqCtx, dnsCtxWithAAnswer(t, "192.168.1.1")) + err := ipFilter.Execute(context.Background(), reqCtx, dnsCtxWithAAnswer(t, "192.168.1.1")) assert.NoError(t, err) assert.Equal(t, tt.wantStatus, reqCtx.FilterResult.Status, "table %s: expected status %s", tt.tableRef, tt.wantStatus) @@ -476,13 +455,7 @@ func TestIPFilter_NilResponse_PreservesDomainBlock(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - mockCache := new(mocks.Cache) - mockCache.On("GetProfileServicesBlocked", mock.Anything, profileID). - Return([]string{}, nil).Maybe() - mockCache.On("GetCustomRulesHashes", mock.Anything, profileID). - Return([]string{}, nil).Maybe() - - ipFilter := NewIPFilter(&proxy.Proxy{}, mockCache, nil, nil, nil, nil) + ipFilter := NewIPFilter(&proxy.Proxy{}, mocks.NewCache(t), nil, nil, nil, nil) reqCtx := newTestReqCtx(t, profileID) reqCtx.PartialFilteringResults = append( @@ -497,7 +470,7 @@ func TestIPFilter_NilResponse_PreservesDomainBlock(t *testing.T) { req.SetQuestion("blocked.example.com.", dns.TypeA) dnsCtx := &proxy.DNSContext{Req: req, Res: nil} - err := ipFilter.Execute(reqCtx, dnsCtx) + err := ipFilter.Execute(context.Background(), reqCtx, dnsCtx) assert.NoError(t, err) // With unified aggregation, domain-phase Block results propagate @@ -584,19 +557,13 @@ func TestIPFilter_NilResponse_IPAllowInert(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - mockCache := new(mocks.Cache) - mockCache.On("GetProfileServicesBlocked", mock.Anything, profileID). - Return(tt.blockedServiceIDs, nil).Maybe() - mockCache.On("GetCustomRulesHashes", mock.Anything, profileID). - Return(tt.customHashes, nil) - for hash, rule := range tt.customRules { - mockCache.On("GetCustomRulesHash", mock.Anything, hash). - Return(rule, nil).Maybe() - } + mockCache := mocks.NewCache(t) ipFilter := NewIPFilter(&proxy.Proxy{}, mockCache, tt.catalog, tt.asnLookup, nil, nil) reqCtx := newTestReqCtx(t, profileID) + reqCtx.BlockedServices = tt.blockedServiceIDs + reqCtx.CustomRules = orderedRules(tt.customHashes, tt.customRules) reqCtx.PartialFilteringResults = append( reqCtx.PartialFilteringResults, tt.domainResults..., ) @@ -607,15 +574,13 @@ func TestIPFilter_NilResponse_IPAllowInert(t *testing.T) { req.SetQuestion("blocked.example.com.", dns.TypeA) dnsCtx := &proxy.DNSContext{Req: req, Res: nil} - err := ipFilter.Execute(reqCtx, dnsCtx) + err := ipFilter.Execute(context.Background(), reqCtx, dnsCtx) assert.NoError(t, err) // With unified aggregation, domain-phase Block propagates. IP allow // rules are inert (nil Res, can't match IPs), so domain Block wins. assert.Equal(t, model.StatusBlocked, reqCtx.FilterResult.Status, "table %s: domain block preserved — IP allow inert with nil Res", tt.tableRef) - - mockCache.AssertExpectations(t) }) } } @@ -630,17 +595,7 @@ func TestIPFilter_CrossPhaseAggregation_PartialResultsGrow(t *testing.T) { answerIP = "1.1.1.1" ) - mockCache := new(mocks.Cache) - mockCache.On("GetProfileServicesBlocked", mock.Anything, profileID). - Return([]string{"google"}, nil) - mockCache.On("GetCustomRulesHashes", mock.Anything, profileID). - Return([]string{"h_block_ip"}, nil) - mockCache.On("GetCustomRulesHash", mock.Anything, "h_block_ip"). - Return(map[string]string{ - "action": ACTION_BLOCK, "value": answerIP, "syntax": "ip4_addr", - }, nil) - - ipFilter := NewIPFilter(&proxy.Proxy{}, mockCache, + ipFilter := NewIPFilter(&proxy.Proxy{}, mocks.NewCache(t), staticCatalog{cat: googleCatalogWithASN(asn)}, staticASNLookup{asn: asn}, nil, @@ -648,11 +603,15 @@ func TestIPFilter_CrossPhaseAggregation_PartialResultsGrow(t *testing.T) { ) reqCtx := newTestReqCtx(t, profileID) + reqCtx.BlockedServices = []string{"google"} + reqCtx.CustomRules = []map[string]string{ + {"action": ACTION_BLOCK, "value": answerIP, "syntax": "ip4_addr"}, + } // Start with one domain-phase result. reqCtx.PartialFilteringResults = []model.StageResult{domainAllowResult()} dnsCtx := dnsCtxWithAAnswer(t, answerIP) - err := ipFilter.Execute(reqCtx, dnsCtx) + err := ipFilter.Execute(context.Background(), reqCtx, dnsCtx) assert.NoError(t, err) // Domain (1) + services (1) + rebinding (1) + custom rules (1) + cname (1) @@ -671,20 +630,16 @@ func TestIPFilter_CrossPhaseAggregation_PartialResultsGrow(t *testing.T) { func TestIPFilter_NilResponse_SubFiltersReturnNone(t *testing.T) { const profileID = "nil-res-subfilters" - mockCache := new(mocks.Cache) - mockCache.On("GetCustomRulesHashes", mock.Anything, profileID). - Return([]string{"h1"}, nil) - mockCache.On("GetCustomRulesHash", mock.Anything, "h1"). - Return(map[string]string{ - "action": ACTION_BLOCK, "value": "1.1.1.1", "syntax": "ip4_addr", - }, nil) - mockCache.On("GetProfileServicesBlocked", mock.Anything, profileID). - Return([]string{"google"}, nil) + mockCache := mocks.NewCache(t) req := new(dns.Msg) req.SetQuestion("example.com.", dns.TypeA) dnsCtx := &proxy.DNSContext{Req: req, Res: nil} reqCtx := newTestReqCtx(t, profileID) + reqCtx.BlockedServices = []string{"google"} + reqCtx.CustomRules = []map[string]string{ + {"action": ACTION_BLOCK, "value": "1.1.1.1", "syntax": "ip4_addr"}, + } // filterServices with nil Res svcFilter := &IPFilter{ @@ -692,7 +647,7 @@ func TestIPFilter_NilResponse_SubFiltersReturnNone(t *testing.T) { ServicesCatalog: staticCatalog{cat: googleCatalogWithASN(15169)}, ASNLookup: staticASNLookup{asn: 15169}, } - svcResult, err := svcFilter.filterServices(reqCtx, dnsCtx) + svcResult, err := svcFilter.filterServices(context.Background(), reqCtx, dnsCtx) assert.NoError(t, err) assert.Equal(t, model.DecisionNone, svcResult.Decision, "filterServices must return None for nil Res") @@ -701,7 +656,7 @@ func TestIPFilter_NilResponse_SubFiltersReturnNone(t *testing.T) { Cache: mockCache, ASNLookup: staticASNLookup{asn: 15169}, } - crResult, err := crFilter.filterCustomRules(reqCtx, dnsCtx) + crResult, err := crFilter.filterCustomRules(context.Background(), reqCtx, dnsCtx) assert.NoError(t, err) assert.Equal(t, model.DecisionNone, crResult.Decision, "filterCustomRules must return None for nil Res") } @@ -714,19 +669,12 @@ func TestIPFilter_DnsCtxWithAddr(t *testing.T) { answerIP = "1.1.1.1" ) - mockCache := new(mocks.Cache) - mockCache.On("GetProfileServicesBlocked", mock.Anything, profileID). - Return([]string{}, nil) - mockCache.On("GetCustomRulesHashes", mock.Anything, profileID). - Return([]string{"h_block_ip"}, nil) - mockCache.On("GetCustomRulesHash", mock.Anything, "h_block_ip"). - Return(map[string]string{ - "action": ACTION_BLOCK, "value": answerIP, "syntax": "ip4_addr", - }, nil) - - ipFilter := NewIPFilter(&proxy.Proxy{}, mockCache, nil, nil, nil, nil) + ipFilter := NewIPFilter(&proxy.Proxy{}, mocks.NewCache(t), nil, nil, nil, nil) reqCtx := newTestReqCtx(t, profileID) + reqCtx.CustomRules = []map[string]string{ + {"action": ACTION_BLOCK, "value": answerIP, "syntax": "ip4_addr"}, + } reqCtx.PartialFilteringResults = []model.StageResult{domainAllowResult()} req := new(dns.Msg) @@ -743,7 +691,7 @@ func TestIPFilter_DnsCtxWithAddr(t *testing.T) { addr := netip.MustParseAddrPort("10.0.0.1:12345") dnsCtx := &proxy.DNSContext{Req: req, Res: res, Addr: addr} - err := ipFilter.Execute(reqCtx, dnsCtx) + err := ipFilter.Execute(context.Background(), reqCtx, dnsCtx) assert.NoError(t, err) // Domain allow (T200) overrides IP block (T200) — allow always wins. diff --git a/proxy/filter/custom_rules.go b/proxy/filter/custom_rules.go index c1bacdc5..463b606e 100644 --- a/proxy/filter/custom_rules.go +++ b/proxy/filter/custom_rules.go @@ -107,12 +107,8 @@ func matchDomainPattern(patternCache *sync.Map, domain, pattern string) bool { } // filterCustomRules checks if the domain is allowed or blocked by custom rules; method is executed before the DNS request is sent. -func (f *DomainFilter) filterCustomRules(reqCtx *requestcontext.RequestContext, dctx *proxy.DNSContext) (*model.StageResult, error) { +func (f *DomainFilter) filterCustomRules(ctx context.Context, reqCtx *requestcontext.RequestContext, dctx *proxy.DNSContext) (*model.StageResult, error) { defer sentry.Recover() - customRuleHashes, err := f.Cache.GetCustomRulesHashes(context.Background(), reqCtx.ProfileId) - if err != nil { - return nil, err - } question := dctx.Req.Question[0].Name fqdn, _ := strings.CutSuffix(question, ".") @@ -120,12 +116,7 @@ func (f *DomainFilter) filterCustomRules(reqCtx *requestcontext.RequestContext, result := &model.StageResult{Decision: model.DecisionNone, Tier: TierCustomRules} allowMatched := false - for _, customRuleHash := range customRuleHashes { - hash, err := f.Cache.GetCustomRulesHash(context.Background(), customRuleHash) - if err != nil { - return nil, err - } - + for _, hash := range reqCtx.CustomRules { if f.matchDomain(fqdn, hash["value"]) { switch hash["action"] { case ACTION_BLOCK: @@ -159,14 +150,9 @@ func (f *DomainFilter) filterCustomRules(reqCtx *requestcontext.RequestContext, } // filterCustomRules checks if the IP address is allowed or blocked by custom rules; method is executed after the DNS request is sent. -func (f *IPFilter) filterCustomRules(reqCtx *requestcontext.RequestContext, dctx *proxy.DNSContext) (*model.StageResult, error) { +func (f *IPFilter) filterCustomRules(ctx context.Context, reqCtx *requestcontext.RequestContext, dctx *proxy.DNSContext) (*model.StageResult, error) { defer sentry.Recover() - customRuleHashes, err := f.Cache.GetCustomRulesHashes(context.Background(), reqCtx.ProfileId) - if err != nil { - return nil, err - } - result := &model.StageResult{Decision: model.DecisionNone, Tier: TierCustomRules} allowMatched := false blockMatched := false @@ -176,14 +162,10 @@ func (f *IPFilter) filterCustomRules(reqCtx *requestcontext.RequestContext, dctx } ips := extractIPsFromAnswer(dctx.Res.Answer) - for _, customRuleHash := range customRuleHashes { - hash, err := f.Cache.GetCustomRulesHash(context.Background(), customRuleHash) - if err != nil { - return nil, err - } + for _, hash := range reqCtx.CustomRules { syntax, ok := hash["syntax"] if !ok || syntax == "" { - log.Debug().Str("hash", customRuleHash).Msg("Old custom rule detected, syntax is empty") + log.Debug().Msg("Old custom rule detected, syntax is empty") continue } @@ -200,7 +182,7 @@ func (f *IPFilter) filterCustomRules(reqCtx *requestcontext.RequestContext, dctx } ruleASN, ok := parseCustomRuleASN(hash["value"]) if !ok { - log.Debug().Str("hash", customRuleHash).Str("value", hash["value"]).Msg("Invalid ASN custom rule value") + log.Debug().Str("value", hash["value"]).Msg("Invalid ASN custom rule value") continue } for _, ip := range ips { diff --git a/proxy/filter/custom_rules_benchmark_test.go b/proxy/filter/custom_rules_benchmark_test.go index 3067aab1..921cbdf9 100644 --- a/proxy/filter/custom_rules_benchmark_test.go +++ b/proxy/filter/custom_rules_benchmark_test.go @@ -1,6 +1,7 @@ package filter import ( + "context" "fmt" "testing" @@ -10,7 +11,6 @@ import ( "github.com/ivpn/dns/proxy/requestcontext" "github.com/miekg/dns" "github.com/rs/zerolog" - "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) @@ -52,17 +52,10 @@ func BenchmarkFilterCustomRules(b *testing.B) { for _, tc := range testCases { b.Run(tc.name, func(b *testing.B) { - mockCache := &mocks.Cache{} - filterManager := &DomainFilter{Cache: mockCache} + filterManager := &DomainFilter{Cache: &mocks.Cache{}} hashes, rulesMap := setupTestData(tc.rulesSize) - // Setup mock expectations - mockCache.On("GetCustomRulesHashes", mock.Anything, mock.Anything).Return(hashes, nil) - for hash, rule := range rulesMap { - mockCache.On("GetCustomRulesHash", mock.Anything, hash).Return(rule, nil) - } - // Create DNS request context msg := new(dns.Msg) msg.SetQuestion(tc.domain+".", dns.TypeA) @@ -71,14 +64,15 @@ func BenchmarkFilterCustomRules(b *testing.B) { } loggerFactory := logging.NewFactory(zerolog.Disabled) reqCtx := &requestcontext.RequestContext{ - ProfileId: "test-profile", - Logger: loggerFactory.ForProfile("test-profile", true), + ProfileId: "test-profile", + CustomRules: orderedRules(hashes, rulesMap), + Logger: loggerFactory.ForProfile("test-profile", true), } // Reset timer and run benchmark b.ResetTimer() for i := 0; i < b.N; i++ { - result, err := filterManager.filterCustomRules(reqCtx, dnsCtx) + result, err := filterManager.filterCustomRules(context.Background(), reqCtx, dnsCtx) require.NoError(b, err) require.NotNil(b, result) } diff --git a/proxy/filter/custom_rules_test.go b/proxy/filter/custom_rules_test.go index 690c74d7..be5cd475 100644 --- a/proxy/filter/custom_rules_test.go +++ b/proxy/filter/custom_rules_test.go @@ -2,6 +2,7 @@ package filter import ( "bytes" + "context" "errors" "net" "testing" @@ -15,7 +16,6 @@ import ( "github.com/rs/zerolog" "github.com/rs/zerolog/log" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" ) func TestFilterCustomRules(t *testing.T) { @@ -108,19 +108,9 @@ func TestFilterCustomRules(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - // Create mock cache - mockCache := new(mocks.Cache) + // The stage has no store dependency: a strict mock fails on any call. + mockCache := mocks.NewCache(t) - // Setup mock expectations - mockCache.On("GetCustomRulesHashes", mock.Anything, tt.profileID). - Return(tt.customRuleHashes, nil) - - for hash, rule := range tt.customRules { - mockCache.On("GetCustomRulesHash", mock.Anything, hash). - Return(rule, nil).Maybe() - } - - // Create filter manager with mock cache dnsProxy := &proxy.Proxy{} fm := NewDomainFilter(dnsProxy, mockCache, nil) @@ -134,15 +124,16 @@ func TestFilterCustomRules(t *testing.T) { // Create request context reqCtx := &requestcontext.RequestContext{ - ProfileId: tt.profileID, - Logger: testLogger, + ProfileId: tt.profileID, + CustomRules: orderedRules(tt.customRuleHashes, tt.customRules), + Logger: testLogger, } dnsCtx := &proxy.DNSContext{ Req: msg, } // Call the function - got, err := fm.filterCustomRules(reqCtx, dnsCtx) + got, err := fm.filterCustomRules(context.Background(), reqCtx, dnsCtx) // Assert results if tt.wantErr { assert.Error(t, err) @@ -152,9 +143,6 @@ func TestFilterCustomRules(t *testing.T) { assert.NoError(t, err) assert.NotNil(t, got) assert.Equal(t, tt.expectedFltrResult, got) - - // Verify all mock expectations were met - mockCache.AssertExpectations(t) }) } } @@ -496,15 +484,15 @@ func TestIPFilter_FilterCustomRules_ASN_Table(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - mockCache := new(mocks.Cache) - mockCache.On("GetCustomRulesHashes", mock.Anything, profileID).Return(tt.customRuleHashes, nil) - for hash, rule := range tt.customRules { - mockCache.On("GetCustomRulesHash", mock.Anything, hash).Return(rule, nil).Maybe() - } + mockCache := mocks.NewCache(t) loggerFactory := logging.NewFactory(zerolog.DebugLevel) testLogger := loggerFactory.ForProfile(profileID, true) - reqCtx := &requestcontext.RequestContext{ProfileId: profileID, Logger: testLogger} + reqCtx := &requestcontext.RequestContext{ + ProfileId: profileID, + CustomRules: orderedRules(tt.customRuleHashes, tt.customRules), + Logger: testLogger, + } var asnLookup ASNLookup if tt.setupASNLookup != nil { @@ -512,7 +500,7 @@ func TestIPFilter_FilterCustomRules_ASN_Table(t *testing.T) { } ipFilter := &IPFilter{Cache: mockCache, ASNLookup: asnLookup} - got, err := ipFilter.filterCustomRules(reqCtx, tt.dnsCtx) + got, err := ipFilter.filterCustomRules(context.Background(), reqCtx, tt.dnsCtx) assert.NoError(t, err) assert.NotNil(t, got) assert.Equal(t, TierCustomRules, got.Tier) @@ -522,8 +510,6 @@ func TestIPFilter_FilterCustomRules_ASN_Table(t *testing.T) { } else { assert.NotContains(t, got.Reasons, REASON_CUSTOM_RULES) } - - mockCache.AssertExpectations(t) }) } } @@ -541,22 +527,17 @@ func TestFilterCustomRulesDomainNotLoggedWhenGateOff(t *testing.T) { }) log.Logger = orig - mockCache := new(mocks.Cache) - mockCache.On("GetCustomRulesHashes", mock.Anything, "prof-1"). - Return([]string{"h1"}, nil) - mockCache.On("GetCustomRulesHash", mock.Anything, "h1"). - Return(map[string]string{"action": ACTION_BLOCK, "value": "blocked.example.com"}, nil) - - fm := NewDomainFilter(&proxy.Proxy{}, mockCache, nil) + fm := NewDomainFilter(&proxy.Proxy{}, mocks.NewCache(t), nil) msg := new(dns.Msg) msg.SetQuestion("blocked.example.com.", dns.TypeA) reqCtx := &requestcontext.RequestContext{ ProfileId: "prof-1", + CustomRules: []map[string]string{{"action": ACTION_BLOCK, "value": "blocked.example.com"}}, Logger: logger, LoggerConfig: logger.Config(), } - got, err := fm.filterCustomRules(reqCtx, &proxy.DNSContext{Req: msg}) + got, err := fm.filterCustomRules(context.Background(), reqCtx, &proxy.DNSContext{Req: msg}) assert.NoError(t, err) assert.Equal(t, model.DecisionBlock, got.Decision) diff --git a/proxy/filter/default_rule.go b/proxy/filter/default_rule.go index be8c399e..b1f845e7 100644 --- a/proxy/filter/default_rule.go +++ b/proxy/filter/default_rule.go @@ -2,7 +2,6 @@ package filter import ( "context" - "github.com/AdguardTeam/dnsproxy/proxy" "github.com/getsentry/sentry-go" "github.com/ivpn/dns/proxy/model" @@ -15,15 +14,12 @@ const ( DEFAULT_RULE = "default_rule" ) -func (f *DomainFilter) applyDefaultRule(reqCtx *requestcontext.RequestContext, dctx *proxy.DNSContext) (*model.StageResult, error) { +func (f *DomainFilter) applyDefaultRule(ctx context.Context, reqCtx *requestcontext.RequestContext, dctx *proxy.DNSContext) (*model.StageResult, error) { defer sentry.Recover() - prvSettings, err := f.Cache.GetProfilePrivacySettings(context.Background(), reqCtx.ProfileId) - if err != nil { - return nil, err - } + // Privacy settings already travel on the request context; no store read. result := &model.StageResult{Decision: model.DecisionNone, Tier: TierDefaultRule} - if prvSettings[DEFAULT_RULE] == RULE_BLOCK { + if reqCtx.PrivacySettings[DEFAULT_RULE] == RULE_BLOCK { result.Decision = model.DecisionBlock result.Reasons = append(result.Reasons, DEFAULT_RULE) reqCtx.Logger.Debug().Msg("Applied default block rule") diff --git a/proxy/filter/domain.go b/proxy/filter/domain.go index 1249a794..924b9826 100644 --- a/proxy/filter/domain.go +++ b/proxy/filter/domain.go @@ -11,15 +11,16 @@ import ( "github.com/ivpn/dns/proxy/model" "github.com/ivpn/dns/proxy/requestcontext" "github.com/miekg/dns" - "golang.org/x/sync/errgroup" ) type DomainFilter struct { Proxy *proxy.Proxy Cache cache.Cache ServicesCatalog ServicesCatalogGetter - patternCache sync.Map - FilteringFuncs []func(reqCtx *requestcontext.RequestContext, dctx *proxy.DNSContext) (*model.StageResult, error) + // Metrics receives per-stage failures; nil disables the metric. + Metrics StageErrorRecorder + patternCache sync.Map + stages []stage } // NewDomainFilter creates a new DomainFilter instance. @@ -30,47 +31,33 @@ func NewDomainFilter(dnsProxy *proxy.Proxy, cache cache.Cache, servicesCatalog S Proxy: dnsProxy, ServicesCatalog: servicesCatalog, } - fltrManager.FilteringFuncs = []func(reqCtx *requestcontext.RequestContext, dctx *proxy.DNSContext) (*model.StageResult, error){ - fltrManager.filterBlocklists, - fltrManager.filterCustomRules, - fltrManager.filterServiceDomains, - fltrManager.applyDefaultRule, + fltrManager.stages = []stage{ + {StageBlocklists, fltrManager.filterBlocklists}, + {StageCustomRules, fltrManager.filterCustomRules}, + {StageServiceDomains, fltrManager.filterServiceDomains}, + {StageDefaultRule, fltrManager.applyDefaultRule}, } return fltrManager } -// Execute performs all stages of filtering DNS requests -func (f *DomainFilter) Execute(reqCtx *requestcontext.RequestContext, dctx *proxy.DNSContext) (err error) { - ctx := context.Background() - eg, egCtx := errgroup.WithContext(ctx) - resultChan := make(chan *model.StageResult, len(f.FilteringFuncs)) - for _, fltrFunc := range f.FilteringFuncs { - func(ctx context.Context, reqCtx *requestcontext.RequestContext) { - eg.Go(func() error { - fltrRes, err := fltrFunc(reqCtx, dctx) - if err != nil { - return err - } - resultChan <- fltrRes - return nil - }) - }(egCtx, reqCtx) - } - if err := eg.Wait(); err != nil { - reqCtx.Logger.Err(err).Msg("Error filtering DNS requests") - } - close(resultChan) +// Execute performs all stages of filtering DNS requests. Any stage failure +// yields StatusUnavailable: the partial results of the other stages are kept +// for logging but never aggregated into a decision. +func (f *DomainFilter) Execute(ctx context.Context, reqCtx *requestcontext.RequestContext, dctx *proxy.DNSContext) (err error) { + err = runStages(ctx, FilterTypeDomain, f.stages, f.Metrics, reqCtx, dctx) - for res := range resultChan { - reqCtx.PartialFilteringResults = append(reqCtx.PartialFilteringResults, *res) + var finalFltrRes model.FilterResult + if err != nil { + finalFltrRes = model.FilterResult{Status: model.StatusUnavailable} + } else { + finalFltrRes = getFinalFilteringResult(reqCtx.PartialFilteringResults) } - finalFltrRes := getFinalFilteringResult(reqCtx.PartialFilteringResults) e := reqCtx.Logger.Debug().Str("Query status", string(finalFltrRes.Status)).Strs("reasons", finalFltrRes.Reasons).Str("qtype", dns.Type(dctx.Req.Question[0].Qtype).String()).Str("filter_type", FilterTypeDomain) reqCtx.AddClientIP(e, dctx.Addr.Addr().String()) reqCtx.AddDomain(e, dctx.Req.Question[0].Name).Msg("Final filtering result") reqCtx.FilterResult = finalFltrRes - return nil + return err } // filterServiceDomains blocks queries for domains listed in the services @@ -78,7 +65,7 @@ func (f *DomainFilter) Execute(reqCtx *requestcontext.RequestContext, dctx *prox // that ASN-based blocking misses when services use third-party CDNs. // Subdomain matching is always on: listing "microsoft.com" also blocks // "www.microsoft.com", "login.microsoft.com", etc. -func (f *DomainFilter) filterServiceDomains(reqCtx *requestcontext.RequestContext, dctx *proxy.DNSContext) (*model.StageResult, error) { +func (f *DomainFilter) filterServiceDomains(ctx context.Context, reqCtx *requestcontext.RequestContext, dctx *proxy.DNSContext) (*model.StageResult, error) { defer sentry.Recover() result := &model.StageResult{Decision: model.DecisionNone, Tier: TierServices} @@ -86,11 +73,13 @@ func (f *DomainFilter) filterServiceDomains(reqCtx *requestcontext.RequestContex return result, nil } - blockedServices, err := f.Cache.GetProfileServicesBlocked(context.Background(), reqCtx.ProfileId) - if err != nil || len(blockedServices) == 0 { + blockedServices := reqCtx.BlockedServices + if len(blockedServices) == 0 { return result, nil } + // The catalog is a local file, not the settings store: failing to load it + // leaves the stage inert instead of failing the query. cat, err := f.ServicesCatalog.Get() if err != nil || cat == nil { return result, nil diff --git a/proxy/filter/e2e_pipeline_benchmark_test.go b/proxy/filter/e2e_pipeline_benchmark_test.go new file mode 100644 index 00000000..c4396a01 --- /dev/null +++ b/proxy/filter/e2e_pipeline_benchmark_test.go @@ -0,0 +1,148 @@ +package filter + +import ( + "context" + "fmt" + "net" + "net/netip" + "strconv" + "strings" + "testing" + + "github.com/AdguardTeam/dnsproxy/proxy" + libscache "github.com/ivpn/dns/libs/cache" + "github.com/ivpn/dns/libs/logging" + "github.com/ivpn/dns/proxy/cache" + "github.com/ivpn/dns/proxy/requestcontext" + "github.com/miekg/dns" + goredis "github.com/redis/go-redis/v9" + "github.com/stretchr/testify/require" + tcredis "github.com/testcontainers/testcontainers-go/modules/redis" +) + +const benchE2EProfile = "benche2eprofile1" + +// startBenchRedis runs a throwaway Redis seeded with one realistic profile: +// 3 blocklists, 5 custom rules, subdomain blocking on, default rule allow. +func startBenchRedis(b *testing.B) (cache.Cache, *goredis.Client) { + b.Helper() + ctx := context.Background() + redisC, err := tcredis.Run(ctx, "redis:7") + if err != nil { + b.Skipf("redis container unavailable: %v", err) + } + b.Cleanup(func() { _ = redisC.Terminate(ctx) }) + host, err := redisC.Host(ctx) + require.NoError(b, err) + port, err := redisC.MappedPort(ctx, "6379") + require.NoError(b, err) + addr := fmt.Sprintf("%s:%s", host, port.Port()) + + rdb := goredis.NewClient(&goredis.Options{Addr: addr}) + b.Cleanup(func() { _ = rdb.Close() }) + k := "settings:" + benchE2EProfile + require.NoError(b, rdb.HSet(ctx, k+":privacy", map[string]string{"default_rule": "allow", "blocklists_subdomains_rule": "block"}).Err()) + require.NoError(b, rdb.HSet(ctx, k+":logs", map[string]string{"enabled": "true"}).Err()) + require.NoError(b, rdb.HSet(ctx, k+":security:dnssec", map[string]string{"enabled": "true", "send_do_bit": "true"}).Err()) + require.NoError(b, rdb.HSet(ctx, k+":advanced", map[string]string{"recursor": "default"}).Err()) + require.NoError(b, rdb.HSet(ctx, k+":statistics", map[string]string{"enabled": "false"}).Err()) + require.NoError(b, rdb.RPush(ctx, k+":blocklists", "bl_ads", "bl_malware", "bl_tracking").Err()) + for _, bl := range []string{"bl_ads", "bl_malware", "bl_tracking"} { + require.NoError(b, rdb.SAdd(ctx, "blocklist:"+bl, "blocked.example", "ads.tracker.example").Err()) + } + rules := []map[string]string{ + {"value": "ads.example", "action": "block", "syntax": "domain"}, + {"value": "*.cdn.example", "action": "allow", "syntax": "domain"}, + {"value": "203.0.113.9", "action": "block", "syntax": "ip"}, + {"value": "AS64496", "action": "block", "syntax": "asn"}, + {"value": "*tracker*", "action": "block", "syntax": "domain"}, + } + for i, r := range rules { + h := fmt.Sprintf("%s:custom_rule:%d", k, i) + require.NoError(b, rdb.HSet(ctx, h, r).Err()) + require.NoError(b, rdb.SAdd(ctx, k+":custom_rules", h).Err()) + } + + c, err := cache.NewRedisCache(&libscache.Config{Address: addr}) + require.NoError(b, err) + b.Cleanup(c.Close) + return c, rdb +} + +// redisCalls sums cmdstat call counters from INFO commandstats. +func redisCalls(b *testing.B, rdb *goredis.Client) (total int64) { + b.Helper() + info, err := rdb.Info(context.Background(), "commandstats").Result() + require.NoError(b, err) + for _, line := range strings.Split(info, "\n") { + if !strings.HasPrefix(line, "cmdstat_") { + continue + } + for _, kv := range strings.Split(strings.SplitN(line, ":", 2)[1], ",") { + if strings.HasPrefix(kv, "calls=") { + n, _ := strconv.ParseInt(strings.TrimSpace(strings.TrimPrefix(kv, "calls=")), 10, 64) + total += n + } + } + } + return total +} + +func benchLogger() logging.LoggerInterface { + return logging.NewDefaultFactory().ForRequest(logging.LoggingConfig{Enabled: false, ProfileID: benchE2EProfile}) +} + +// benchQuery is a no-match query: every stage runs to completion, blocklist +// membership walks 3 lists × 3 labels. +func benchQuery() *proxy.DNSContext { + req := new(dns.Msg) + req.SetQuestion("www.news.example.", dns.TypeA) + res := new(dns.Msg) + res.SetReply(req) + res.Answer = []dns.RR{&dns.A{Hdr: dns.RR_Header{Name: req.Question[0].Name, Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 60}, A: net.ParseIP("198.51.100.7")}} + return &proxy.DNSContext{Req: req, Res: res, Addr: netip.MustParseAddrPort("192.0.2.1:53"), Proto: proxy.ProtoUDP} +} + +// BenchmarkE2EPipeline_RealRedis measures one full query through both filter +// phases against a real Redis. "warm" = settings already cached (the common +// case); "cold" = settings batch fetched first (cache miss). Reports Redis +// commands per query alongside ns/op. +func BenchmarkE2EPipeline_RealRedis(b *testing.B) { + c, rdb := startBenchRedis(b) + ctx := context.Background() + domainF := NewDomainFilter(nil, c, nil) + ipF := NewIPFilter(nil, c, nil, nil, nil, nil) + settings, err := c.GetProfileSettingsBatch(ctx, benchE2EProfile) + require.NoError(b, err) + require.NoError(b, settings.StoreError()) + logger := benchLogger() + + b.Run("warm", func(b *testing.B) { + require.NoError(b, rdb.ConfigResetStat(ctx).Err()) + b.ResetTimer() + for i := 0; i < b.N; i++ { + reqCtx := requestcontext.NewRequestContext(ctx, nil, benchE2EProfile, "", settings, logger) + dctx := benchQuery() + _ = domainF.Execute(ctx, reqCtx, dctx) + _ = ipF.Execute(ctx, reqCtx, dctx) + } + b.StopTimer() + b.ReportMetric(float64(redisCalls(b, rdb)-1)/float64(b.N), "redis-cmds/op") + }) + b.Run("cold", func(b *testing.B) { + require.NoError(b, rdb.ConfigResetStat(ctx).Err()) + b.ResetTimer() + for i := 0; i < b.N; i++ { + s, err := c.GetProfileSettingsBatch(ctx, benchE2EProfile) + if err != nil { + b.Fatal(err) + } + reqCtx := requestcontext.NewRequestContext(ctx, nil, benchE2EProfile, "", s, logger) + dctx := benchQuery() + _ = domainF.Execute(ctx, reqCtx, dctx) + _ = ipF.Execute(ctx, reqCtx, dctx) + } + b.StopTimer() + b.ReportMetric(float64(redisCalls(b, rdb)-1)/float64(b.N), "redis-cmds/op") + }) +} diff --git a/proxy/filter/filter.go b/proxy/filter/filter.go index 2b0a0586..9fe4b069 100644 --- a/proxy/filter/filter.go +++ b/proxy/filter/filter.go @@ -1,12 +1,16 @@ package filter import ( + "context" + "github.com/AdguardTeam/dnsproxy/proxy" "github.com/ivpn/dns/proxy/requestcontext" ) type Filter interface { - Execute(reqCtx *requestcontext.RequestContext, dctx *proxy.DNSContext) error + // Execute runs one filter phase; ctx is the request context and bounds + // the phase's live store reads. + Execute(ctx context.Context, reqCtx *requestcontext.RequestContext, dctx *proxy.DNSContext) error } const ( diff --git a/proxy/filter/ip.go b/proxy/filter/ip.go index 200fec0f..ab80352d 100644 --- a/proxy/filter/ip.go +++ b/proxy/filter/ip.go @@ -10,7 +10,6 @@ import ( "github.com/ivpn/dns/proxy/model" "github.com/ivpn/dns/proxy/requestcontext" "github.com/miekg/dns" - "golang.org/x/sync/errgroup" ) type IPFilter struct { @@ -20,8 +19,10 @@ type IPFilter struct { ASNLookup ASNLookup RebindingConfig *config.RebindingConfig FilteringConfig *config.FilteringConfig - patternCache sync.Map - FilteringFuncs []func(reqCtx *requestcontext.RequestContext, dctx *proxy.DNSContext) (*model.StageResult, error) + // Metrics receives per-stage failures; nil disables the metric. + Metrics StageErrorRecorder + patternCache sync.Map + stages []stage } // NewIPFilter creates a new IPFilter instance. A nil filteringConfig means all @@ -35,47 +36,31 @@ func NewIPFilter(dnsProxy *proxy.Proxy, cache cache.Cache, servicesCatalog Servi RebindingConfig: rebindingConfig, FilteringConfig: filteringConfig, } - fltrManager.FilteringFuncs = []func(reqCtx *requestcontext.RequestContext, dctx *proxy.DNSContext) (*model.StageResult, error){ - fltrManager.filterServices, - fltrManager.filterRebinding, - fltrManager.filterCustomRules, - fltrManager.filterCNAME, + fltrManager.stages = []stage{ + {StageServices, fltrManager.filterServices}, + {StageRebinding, fltrManager.filterRebinding}, + {StageCustomRules, fltrManager.filterCustomRules}, + {StageCNAME, fltrManager.filterCNAME}, } return fltrManager } -// Execute performs all stages of filtering DNS requests -func (f *IPFilter) Execute(reqCtx *requestcontext.RequestContext, dctx *proxy.DNSContext) (err error) { - ctx := context.Background() - eg, egCtx := errgroup.WithContext(ctx) - resultChan := make(chan *model.StageResult, len(f.FilteringFuncs)) - for _, fltrFunc := range f.FilteringFuncs { - func(ctx context.Context, reqCtx *requestcontext.RequestContext) { - eg.Go(func() error { - fltrRes, err := fltrFunc(reqCtx, dctx) - if err != nil { - return err - } - resultChan <- fltrRes - return nil - }) - }(egCtx, reqCtx) - } - if err := eg.Wait(); err != nil { - reqCtx.Logger.Err(err).Msg("Error filtering IP address in DNS response") - } - close(resultChan) +// Execute performs all stages of filtering DNS responses. Any stage failure +// yields StatusUnavailable even though an upstream answer exists: an answer +// that could not be checked is not returned. +func (f *IPFilter) Execute(ctx context.Context, reqCtx *requestcontext.RequestContext, dctx *proxy.DNSContext) (err error) { + err = runStages(ctx, FilterTypeIP, f.stages, f.Metrics, reqCtx, dctx) - for res := range resultChan { - reqCtx.PartialFilteringResults = append(reqCtx.PartialFilteringResults, *res) + var finalFltrRes model.FilterResult + if err != nil { + finalFltrRes = model.FilterResult{Status: model.StatusUnavailable} + } else { + finalFltrRes = getFinalFilteringResult(reqCtx.PartialFilteringResults) } - - finalFltrRes := getFinalFilteringResult(reqCtx.PartialFilteringResults) e := reqCtx.Logger.Debug().Str("Query status", string(finalFltrRes.Status)).Strs("Reasons", finalFltrRes.Reasons).Str("qtype", dns.Type(dctx.Req.Question[0].Qtype).String()).Str("filter_type", FilterTypeIP) reqCtx.AddClientIP(e, dctx.Addr.Addr().String()) reqCtx.AddDomain(e, dctx.Req.Question[0].Name).Msg("Final filtering result") - // save the final filtering result to the request context once, only in IP filtering phase? reqCtx.FilterResult = finalFltrRes - return nil + return err } diff --git a/proxy/filter/ip_custom_rules_asn_test.go b/proxy/filter/ip_custom_rules_asn_test.go index 52eb39fc..821bbc1f 100644 --- a/proxy/filter/ip_custom_rules_asn_test.go +++ b/proxy/filter/ip_custom_rules_asn_test.go @@ -1,6 +1,7 @@ package filter import ( + "context" "net" "testing" @@ -12,7 +13,6 @@ import ( "github.com/miekg/dns" "github.com/rs/zerolog" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" ) func TestIPFilter_BlockWinsOnConflict_CustomRules_ASN(t *testing.T) { @@ -22,19 +22,11 @@ func TestIPFilter_BlockWinsOnConflict_CustomRules_ASN(t *testing.T) { allowIP := "1.1.1.1" blockIP := "2.2.2.2" - mockCache := new(mocks.Cache) - customRuleHashes := []string{"hash_allow_asn", "hash_block_asn"} - mockCache.On("GetCustomRulesHashes", mock.Anything, profileID).Return(customRuleHashes, nil) - mockCache.On("GetCustomRulesHash", mock.Anything, "hash_allow_asn").Return(map[string]string{ - "action": ACTION_ALLOW, - "value": "AS15169", - "syntax": "asn", - }, nil) - mockCache.On("GetCustomRulesHash", mock.Anything, "hash_block_asn").Return(map[string]string{ - "action": ACTION_BLOCK, - "value": "15169", - "syntax": "asn", - }, nil) + mockCache := mocks.NewCache(t) + customRules := []map[string]string{ + {"action": ACTION_ALLOW, "value": "AS15169", "syntax": "asn"}, + {"action": ACTION_BLOCK, "value": "15169", "syntax": "asn"}, + } mockASN := mocks.NewASNLookup(t) mockASN.On("ASN", net.ParseIP(allowIP)).Return(allowASN, nil) @@ -56,9 +48,9 @@ func TestIPFilter_BlockWinsOnConflict_CustomRules_ASN(t *testing.T) { dnsCtx := &proxy.DNSContext{Req: req, Res: res} loggerFactory := logging.NewFactory(zerolog.DebugLevel) testLogger := loggerFactory.ForProfile(profileID, true) - reqCtx := &requestcontext.RequestContext{ProfileId: profileID, Logger: testLogger} + reqCtx := &requestcontext.RequestContext{ProfileId: profileID, CustomRules: customRules, Logger: testLogger} - err := ipFilter.Execute(reqCtx, dnsCtx) + err := ipFilter.Execute(context.Background(), reqCtx, dnsCtx) assert.NoError(t, err) assert.Equal(t, model.StatusBlocked, reqCtx.FilterResult.Status) assert.Contains(t, reqCtx.FilterResult.Reasons, REASON_CUSTOM_RULES) @@ -70,14 +62,10 @@ func TestIPFilter_BlockByASN_CustomRules(t *testing.T) { ipStr := "1.1.1.1" - mockCache := new(mocks.Cache) - customRuleHashes := []string{"hash_block_asn"} - mockCache.On("GetCustomRulesHashes", mock.Anything, profileID).Return(customRuleHashes, nil) - mockCache.On("GetCustomRulesHash", mock.Anything, "hash_block_asn").Return(map[string]string{ - "action": ACTION_BLOCK, - "value": "AS15169", - "syntax": "asn", - }, nil) + mockCache := mocks.NewCache(t) + customRules := []map[string]string{ + {"action": ACTION_BLOCK, "value": "AS15169", "syntax": "asn"}, + } mockASN := mocks.NewASNLookup(t) mockASN.On("ASN", net.ParseIP(ipStr)).Return(asn, nil) @@ -97,9 +85,9 @@ func TestIPFilter_BlockByASN_CustomRules(t *testing.T) { dnsCtx := &proxy.DNSContext{Req: req, Res: res} loggerFactory := logging.NewFactory(zerolog.DebugLevel) testLogger := loggerFactory.ForProfile(profileID, true) - reqCtx := &requestcontext.RequestContext{ProfileId: profileID, Logger: testLogger} + reqCtx := &requestcontext.RequestContext{ProfileId: profileID, CustomRules: customRules, Logger: testLogger} - err := ipFilter.Execute(reqCtx, dnsCtx) + err := ipFilter.Execute(context.Background(), reqCtx, dnsCtx) assert.NoError(t, err) assert.Equal(t, model.StatusBlocked, reqCtx.FilterResult.Status) assert.Contains(t, reqCtx.FilterResult.Reasons, REASON_CUSTOM_RULES) diff --git a/proxy/filter/ip_custom_rules_precedence_test.go b/proxy/filter/ip_custom_rules_precedence_test.go index fab2192f..bdf33807 100644 --- a/proxy/filter/ip_custom_rules_precedence_test.go +++ b/proxy/filter/ip_custom_rules_precedence_test.go @@ -1,6 +1,7 @@ package filter import ( + "context" "net" "testing" @@ -12,7 +13,6 @@ import ( "github.com/miekg/dns" "github.com/rs/zerolog" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" ) func TestIPFilter_BlockWinsOnConflict_CustomRules_IP(t *testing.T) { @@ -21,30 +21,14 @@ func TestIPFilter_BlockWinsOnConflict_CustomRules_IP(t *testing.T) { allowIP := "1.1.1.1" blockIP := "2.2.2.2" - // Create mock cache - mockCache := new(mocks.Cache) - - customRuleHashes := []string{"hash_allow", "hash_block"} - mockCache.On("GetCustomRulesHashes", mock.Anything, profileID). - Return(customRuleHashes, nil) - - mockCache.On("GetCustomRulesHash", mock.Anything, "hash_allow"). - Return(map[string]string{ - "action": ACTION_ALLOW, - "value": allowIP, - "syntax": "ip4_addr", - }, nil) - - mockCache.On("GetCustomRulesHash", mock.Anything, "hash_block"). - Return(map[string]string{ - "action": ACTION_BLOCK, - "value": blockIP, - "syntax": "ip4_addr", - }, nil) + // Custom rules travel on the request context; the store is never read here. + customRules := []map[string]string{ + {"action": ACTION_ALLOW, "value": allowIP, "syntax": "ip4_addr"}, + {"action": ACTION_BLOCK, "value": blockIP, "syntax": "ip4_addr"}, + } - // Create filter manager with mock cache dnsProxy := &proxy.Proxy{} - ipFilter := NewIPFilter(dnsProxy, mockCache, nil, nil, nil, nil) + ipFilter := NewIPFilter(dnsProxy, mocks.NewCache(t), nil, nil, nil, nil) // Create DNS request/response with two A answers. req := new(dns.Msg) @@ -67,9 +51,9 @@ func TestIPFilter_BlockWinsOnConflict_CustomRules_IP(t *testing.T) { loggerFactory := logging.NewFactory(zerolog.DebugLevel) testLogger := loggerFactory.ForProfile(profileID, true) - reqCtx := &requestcontext.RequestContext{ProfileId: profileID, Logger: testLogger} + reqCtx := &requestcontext.RequestContext{ProfileId: profileID, CustomRules: customRules, Logger: testLogger} - err := ipFilter.Execute(reqCtx, dnsCtx) + err := ipFilter.Execute(context.Background(), reqCtx, dnsCtx) assert.NoError(t, err) // When both allow and block custom rules match within a single response, block wins. diff --git a/proxy/filter/ip_custom_rules_test.go b/proxy/filter/ip_custom_rules_test.go index 56b3b1cc..20ed2310 100644 --- a/proxy/filter/ip_custom_rules_test.go +++ b/proxy/filter/ip_custom_rules_test.go @@ -1,7 +1,7 @@ package filter import ( - "errors" + "context" "net" "testing" @@ -13,7 +13,6 @@ import ( "github.com/miekg/dns" "github.com/rs/zerolog" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" ) // buildDNSResponse creates a dns.Msg response with the given A and AAAA answer records. @@ -243,20 +242,13 @@ func TestIPFilterCustomRules(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - mockCache := new(mocks.Cache) - - mockCache.On("GetCustomRulesHashes", mock.Anything, tt.profileID). - Return(tt.customRuleHashes, nil) - for hash, rule := range tt.customRules { - mockCache.On("GetCustomRulesHash", mock.Anything, hash). - Return(rule, nil).Maybe() - } - - fm := NewIPFilter(&proxy.Proxy{}, mockCache, nil, nil, nil, nil) + // The stage has no store dependency: a strict mock fails on any call. + fm := NewIPFilter(&proxy.Proxy{}, mocks.NewCache(t), nil, nil, nil, nil) reqCtx := &requestcontext.RequestContext{ - ProfileId: tt.profileID, - Logger: loggerFactory.ForProfile(tt.profileID, true), + ProfileId: tt.profileID, + CustomRules: orderedRules(tt.customRuleHashes, tt.customRules), + Logger: loggerFactory.ForProfile(tt.profileID, true), } msg := new(dns.Msg) @@ -266,7 +258,7 @@ func TestIPFilterCustomRules(t *testing.T) { Res: tt.response, } - got, err := fm.filterCustomRules(reqCtx, dctx) + got, err := fm.filterCustomRules(context.Background(), reqCtx, dctx) if tt.wantErr { assert.Error(t, err) @@ -276,60 +268,6 @@ func TestIPFilterCustomRules(t *testing.T) { assert.NoError(t, err) assert.NotNil(t, got) assert.Equal(t, tt.expectedResult, got) - mockCache.AssertExpectations(t) - }) - } -} - -func TestIPFilterCustomRules_CacheErrors(t *testing.T) { - tests := []struct { - name string - setupMock func(*mocks.Cache) - }{ - { - name: "GetCustomRulesHashes returns error", - setupMock: func(m *mocks.Cache) { - m.On("GetCustomRulesHashes", mock.Anything, "test-profile"). - Return([]string(nil), errors.New("redis connection refused")) - }, - }, - { - name: "GetCustomRulesHash returns error", - setupMock: func(m *mocks.Cache) { - m.On("GetCustomRulesHashes", mock.Anything, "test-profile"). - Return([]string{"hash1"}, nil) - m.On("GetCustomRulesHash", mock.Anything, "hash1"). - Return(map[string]string(nil), errors.New("redis timeout")) - }, - }, - } - - loggerFactory := logging.NewFactory(zerolog.Disabled) - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - mockCache := new(mocks.Cache) - tt.setupMock(mockCache) - - fm := NewIPFilter(&proxy.Proxy{}, mockCache, nil, nil, nil, nil) - - reqCtx := &requestcontext.RequestContext{ - ProfileId: "test-profile", - Logger: loggerFactory.ForProfile("test-profile", true), - } - - msg := new(dns.Msg) - msg.SetQuestion("example.com.", dns.TypeA) - dctx := &proxy.DNSContext{ - Req: msg, - Res: buildDNSResponse("example.com", []string{"1.2.3.4"}, nil), - } - - got, err := fm.filterCustomRules(reqCtx, dctx) - - assert.Error(t, err) - assert.Nil(t, got) - mockCache.AssertExpectations(t) }) } } diff --git a/proxy/filter/rebinding.go b/proxy/filter/rebinding.go index 6594dc0e..0abb96ab 100644 --- a/proxy/filter/rebinding.go +++ b/proxy/filter/rebinding.go @@ -1,6 +1,7 @@ package filter import ( + "context" "strconv" "strings" @@ -26,7 +27,7 @@ const ( // It is per-profile opt-in: the profile must have rebinding_protection enabled, and // the global master switch must be on. Names matching an operator allow-suffix // (e.g. .local) are never blocked. -func (f *IPFilter) filterRebinding(reqCtx *requestcontext.RequestContext, dctx *proxy.DNSContext) (*model.StageResult, error) { +func (f *IPFilter) filterRebinding(ctx context.Context, reqCtx *requestcontext.RequestContext, dctx *proxy.DNSContext) (*model.StageResult, error) { defer sentry.Recover() result := &model.StageResult{Decision: model.DecisionNone, Tier: TierRebinding} diff --git a/proxy/filter/rebinding_benchmark_test.go b/proxy/filter/rebinding_benchmark_test.go index 42af40f0..3207b195 100644 --- a/proxy/filter/rebinding_benchmark_test.go +++ b/proxy/filter/rebinding_benchmark_test.go @@ -1,6 +1,7 @@ package filter import ( + "context" "net" "testing" @@ -60,7 +61,7 @@ func BenchmarkFilterRebinding(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { - res, err := f.filterRebinding(reqCtx, dctx) + res, err := f.filterRebinding(context.Background(), reqCtx, dctx) require.NoError(b, err) require.NotNil(b, res) } diff --git a/proxy/filter/rebinding_test.go b/proxy/filter/rebinding_test.go index f5ef9dd4..f9541b5b 100644 --- a/proxy/filter/rebinding_test.go +++ b/proxy/filter/rebinding_test.go @@ -1,6 +1,7 @@ package filter import ( + "context" "net" "testing" @@ -120,7 +121,7 @@ func TestFilterRebinding(t *testing.T) { reqCtx.RebindingProtectionSettings = tt.settings f := &IPFilter{RebindingConfig: tt.cfg} - res, err := f.filterRebinding(reqCtx, tt.dctx) + res, err := f.filterRebinding(context.Background(), reqCtx, tt.dctx) assert.NoError(t, err, "row %s", tt.tableRef) assert.NotNil(t, res, "row %s", tt.tableRef) assert.Equal(t, tt.want, res.Decision, "row %s: %s", tt.tableRef, tt.name) @@ -142,7 +143,7 @@ func TestFilterRebinding_ReasonReachesFinalFilterResult(t *testing.T) { reqCtx.RebindingProtectionSettings = map[string]string{"enabled": "1"} f := &IPFilter{RebindingConfig: defaultRebindingConfig()} - res, err := f.filterRebinding(reqCtx, dnsCtxNameA(t, "evil.com.", "192.168.1.1")) + res, err := f.filterRebinding(context.Background(), reqCtx, dnsCtxNameA(t, "evil.com.", "192.168.1.1")) assert.NoError(t, err) final := getFinalFilteringResult(append(reqCtx.PartialFilteringResults, *res)) @@ -159,7 +160,7 @@ func TestFilterRebinding_HTTPSHint(t *testing.T) { f := &IPFilter{RebindingConfig: defaultRebindingConfig()} dctx := dnsCtxWithHTTPSAnswer(t, "evil.com.", []net.IP{net.ParseIP("192.168.1.1")}, nil) - res, err := f.filterRebinding(reqCtx, dctx) + res, err := f.filterRebinding(context.Background(), reqCtx, dctx) assert.NoError(t, err) assert.Equal(t, model.DecisionBlock, res.Decision) assert.Contains(t, res.Reasons, REASON_REBINDING) diff --git a/proxy/filter/service_domains_test.go b/proxy/filter/service_domains_test.go index 3858fd13..072d57ad 100644 --- a/proxy/filter/service_domains_test.go +++ b/proxy/filter/service_domains_test.go @@ -1,6 +1,7 @@ package filter import ( + "context" "testing" "github.com/AdguardTeam/dnsproxy/proxy" @@ -9,7 +10,6 @@ import ( "github.com/ivpn/dns/proxy/model" "github.com/miekg/dns" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) @@ -114,21 +114,18 @@ func TestFilterServiceDomains(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - mockCache := new(mocks.Cache) - mockCache.On("GetProfileServicesBlocked", mock.Anything, "test-profile"). - Return(tt.blockedIDs, nil) - fm := &DomainFilter{ - Cache: mockCache, + Cache: mocks.NewCache(t), ServicesCatalog: staticCatalog{cat: catalog}, } reqCtx := newTestReqCtx(t, "test-profile") + reqCtx.BlockedServices = tt.blockedIDs msg := new(dns.Msg) msg.SetQuestion(tt.domain, dns.TypeA) dctx := &proxy.DNSContext{Req: msg} - result, err := fm.filterServiceDomains(reqCtx, dctx) + result, err := fm.filterServiceDomains(context.Background(), reqCtx, dctx) require.NoError(t, err) assert.Equal(t, tt.expectedDecision, result.Decision) assert.Equal(t, TierServices, result.Tier) @@ -150,26 +147,23 @@ func TestFilterServiceDomains_NilCatalog(t *testing.T) { msg.SetQuestion("microsoft.com.", dns.TypeA) dctx := &proxy.DNSContext{Req: msg} - result, err := fm.filterServiceDomains(reqCtx, dctx) + result, err := fm.filterServiceDomains(context.Background(), reqCtx, dctx) require.NoError(t, err) assert.Equal(t, model.DecisionNone, result.Decision) } func TestFilterServiceDomains_CatalogError(t *testing.T) { - mockCache := new(mocks.Cache) - mockCache.On("GetProfileServicesBlocked", mock.Anything, "test-profile"). - Return([]string{"microsoft"}, nil) - fm := &DomainFilter{ - Cache: mockCache, + Cache: mocks.NewCache(t), ServicesCatalog: staticCatalogErr{err: assert.AnError}, } reqCtx := newTestReqCtx(t, "test-profile") + reqCtx.BlockedServices = []string{"microsoft"} msg := new(dns.Msg) msg.SetQuestion("microsoft.com.", dns.TypeA) dctx := &proxy.DNSContext{Req: msg} - result, err := fm.filterServiceDomains(reqCtx, dctx) + result, err := fm.filterServiceDomains(context.Background(), reqCtx, dctx) require.NoError(t, err) assert.Equal(t, model.DecisionNone, result.Decision) } diff --git a/proxy/filter/services.go b/proxy/filter/services.go index 39b70108..de7a41ad 100644 --- a/proxy/filter/services.go +++ b/proxy/filter/services.go @@ -24,7 +24,7 @@ type ASNLookup interface { ASN(ip net.IP) (uint, error) } -func (f *IPFilter) filterServices(reqCtx *requestcontext.RequestContext, dctx *proxy.DNSContext) (*model.StageResult, error) { +func (f *IPFilter) filterServices(ctx context.Context, reqCtx *requestcontext.RequestContext, dctx *proxy.DNSContext) (*model.StageResult, error) { defer sentry.Recover() result := &model.StageResult{Decision: model.DecisionNone, Tier: TierServices} @@ -35,18 +35,15 @@ func (f *IPFilter) filterServices(reqCtx *requestcontext.RequestContext, dctx *p return result, nil } - blockedServices, err := f.Cache.GetProfileServicesBlocked(context.Background(), reqCtx.ProfileId) - if err != nil { - // Missing key should be non-fatal; treat as disabled. - return result, nil - } + blockedServices := reqCtx.BlockedServices if len(blockedServices) == 0 { return result, nil } + // The catalog is a local file, not the settings store: failing to load it + // leaves the stage inert instead of failing the query. cat, err := f.ServicesCatalog.Get() if err != nil || cat == nil { - // Catalog load failure should not break DNS. return result, nil } diff --git a/proxy/filter/services_test.go b/proxy/filter/services_test.go index b7846f24..b96a9ec4 100644 --- a/proxy/filter/services_test.go +++ b/proxy/filter/services_test.go @@ -1,6 +1,7 @@ package filter import ( + "context" "errors" "net" "testing" @@ -14,7 +15,6 @@ import ( "github.com/miekg/dns" "github.com/rs/zerolog" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" ) type staticCatalog struct{ cat *servicescatalog.Catalog } @@ -54,6 +54,18 @@ func newTestReqCtx(t *testing.T, profileID string) *requestcontext.RequestContex return &requestcontext.RequestContext{ProfileId: profileID, Logger: testLogger} } +// orderedRules builds the request-context rule list the settings batch would +// carry: the rule hashes named by ids, in that order. +func orderedRules(ids []string, rules map[string]map[string]string) []map[string]string { + out := make([]map[string]string, 0, len(ids)) + for _, id := range ids { + if r, ok := rules[id]; ok { + out = append(out, r) + } + } + return out +} + func dnsCtxWithAAnswer(t *testing.T, ipStr string) *proxy.DNSContext { t.Helper() req := new(dns.Msg) @@ -133,7 +145,6 @@ func TestIPFilter_filterServices_Table(t *testing.T) { servicesGetter ServicesCatalogGetter asnLookup ASNLookup blockedIDs []string - cacheErr error dnsCtx *proxy.DNSContext wantDecision model.Decision wantReasons []string @@ -170,15 +181,6 @@ func TestIPFilter_filterServices_Table(t *testing.T) { dnsCtx: &proxy.DNSContext{Req: new(dns.Msg), Res: nil}, wantDecision: model.DecisionNone, }, - { - name: "cache error treated as disabled", - servicesGetter: staticCatalog{cat: googleCatalogWithASN(asn)}, - asnLookup: staticASNLookup{asn: asn}, - blockedIDs: nil, - cacheErr: errors.New("cache error"), - dnsCtx: dnsCtxWithAAnswer(t, "1.1.1.1"), - wantDecision: model.DecisionNone, - }, { name: "no blocked services", servicesGetter: staticCatalog{cat: googleCatalogWithASN(asn)}, @@ -314,15 +316,8 @@ func TestIPFilter_filterServices_Table(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - mockCache := new(mocks.Cache) - shouldCallCache := tt.servicesGetter != nil && tt.asnLookup != nil && tt.dnsCtx != nil && tt.dnsCtx.Res != nil - if shouldCallCache { - if tt.cacheErr != nil { - mockCache.On("GetProfileServicesBlocked", mock.Anything, profileID).Return(nil, tt.cacheErr) - } else { - mockCache.On("GetProfileServicesBlocked", mock.Anything, profileID).Return(tt.blockedIDs, nil) - } - } + // The stage has no store dependency: a strict mock fails on any call. + mockCache := mocks.NewCache(t) ipFilter := &IPFilter{ Cache: mockCache, @@ -332,7 +327,8 @@ func TestIPFilter_filterServices_Table(t *testing.T) { } reqCtx := newTestReqCtx(t, profileID) - got, err := ipFilter.filterServices(reqCtx, tt.dnsCtx) + reqCtx.BlockedServices = tt.blockedIDs + got, err := ipFilter.filterServices(context.Background(), reqCtx, tt.dnsCtx) assert.NoError(t, err) assert.NotNil(t, got) assert.Equal(t, TierServices, got.Tier) @@ -340,12 +336,6 @@ func TestIPFilter_filterServices_Table(t *testing.T) { for _, r := range tt.wantReasons { assert.Contains(t, got.Reasons, r) } - - if shouldCallCache { - mockCache.AssertExpectations(t) - } else { - mockCache.AssertNotCalled(t, "GetProfileServicesBlocked", mock.Anything, mock.Anything) - } }) } } @@ -411,18 +401,15 @@ func TestIPFilter_ServicesBlocking_Integration_Table(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - mockCache := new(mocks.Cache) - mockCache.On("GetProfileServicesBlocked", mock.Anything, tt.profileID).Return(tt.blockedIDs, nil) - mockCache.On("GetCustomRulesHashes", mock.Anything, tt.profileID).Return(tt.customHashes, nil) - for hash, rule := range tt.customRules { - mockCache.On("GetCustomRulesHash", mock.Anything, hash).Return(rule, nil) - } + mockCache := mocks.NewCache(t) dnsProxy := &proxy.Proxy{} ipFilter := NewIPFilter(dnsProxy, mockCache, staticCatalog{cat: tt.catalog}, tt.asnLookup, nil, nil) reqCtx := newTestReqCtx(t, tt.profileID) + reqCtx.BlockedServices = tt.blockedIDs + reqCtx.CustomRules = orderedRules(tt.customHashes, tt.customRules) - err := ipFilter.Execute(reqCtx, tt.dnsCtx) + err := ipFilter.Execute(context.Background(), reqCtx, tt.dnsCtx) assert.NoError(t, err) assert.Equal(t, tt.wantStatus, reqCtx.FilterResult.Status) for _, s := range tt.wantContains { @@ -431,8 +418,6 @@ func TestIPFilter_ServicesBlocking_Integration_Table(t *testing.T) { for _, s := range tt.wantNotContains { assert.NotContains(t, reqCtx.FilterResult.Reasons, s) } - - mockCache.AssertExpectations(t) }) } } diff --git a/proxy/filter/stage_errors_test.go b/proxy/filter/stage_errors_test.go new file mode 100644 index 00000000..211a0bfe --- /dev/null +++ b/proxy/filter/stage_errors_test.go @@ -0,0 +1,331 @@ +package filter + +// Tests for the settings-store error policy in DomainFilter.Execute and +// IPFilter.Execute: a failing stage yields StatusUnavailable instead of an +// accidental fail-open. Rows: docs/specs/proxy-filtering-behaviour.md Section I. +// +// Per-profile inputs (blocklist subscriptions, custom rules, blocked services, +// privacy settings) travel on the request context, so the only stage read that +// can fail is the live blocklist membership lookup (GetBlocklistEntry), used by +// the domain-phase blocklists stage and the IP-phase CNAME stage. + +import ( + "context" + "errors" + "sync" + "testing" + + "github.com/AdguardTeam/dnsproxy/proxy" + "github.com/ivpn/dns/libs/logging" + "github.com/ivpn/dns/proxy/mocks" + "github.com/ivpn/dns/proxy/model" + "github.com/ivpn/dns/proxy/requestcontext" + "github.com/miekg/dns" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +// stageErrorRecorder records (phase, stage) pairs; stages run concurrently so +// it is mutex-guarded. +type stageErrorRecorder struct { + mu sync.Mutex + pairs [][2]string +} + +func (r *stageErrorRecorder) RecordFilterStageError(phase, stage string) { + r.mu.Lock() + defer r.mu.Unlock() + r.pairs = append(r.pairs, [2]string{phase, stage}) +} + +func (r *stageErrorRecorder) has(phase, stage string) bool { + r.mu.Lock() + defer r.mu.Unlock() + for _, p := range r.pairs { + if p[0] == phase && p[1] == stage { + return true + } + } + return false +} + +func (r *stageErrorRecorder) count() int { + r.mu.Lock() + defer r.mu.Unlock() + return len(r.pairs) +} + +const stageErrBlocklistID = "bl-stage-err" + +func stageErrReqCtx(t *testing.T, profileID string, privacy map[string]string) *requestcontext.RequestContext { + t.Helper() + logger := logging.NewFactory(zerolog.DebugLevel).ForProfile(profileID, true) + return &requestcontext.RequestContext{ + ProfileId: profileID, + Blocklists: []string{stageErrBlocklistID}, + PrivacySettings: privacy, + Logger: logger, + } +} + +func stageErrDomainDctx(qname string) *proxy.DNSContext { + msg := new(dns.Msg) + msg.SetQuestion(dns.Fqdn(qname), dns.TypeA) + return &proxy.DNSContext{Req: msg} +} + +// stageErrCNAMEDctx builds a resolved answer with a CNAME hop so the IP-phase +// CNAME stage performs a live blocklist lookup on the target. +func stageErrCNAMEDctx(qname string) *proxy.DNSContext { + res := buildCNAMEChainResponse(qname, dns.TypeA, []string{"tracker.evil.net"}, "93.184.216.34") + req := new(dns.Msg) + req.SetQuestion(dns.Fqdn(qname), dns.TypeA) + return &proxy.DNSContext{Req: req, Res: res} +} + +// failingMembershipCache returns a mock whose blocklist membership lookup fails. +func failingMembershipCache() *mocks.Cache { + mockCache := new(mocks.Cache) + mockCache.On("GetBlocklistEntry", mock.Anything, stageErrBlocklistID, mock.Anything). + Return(false, errStore).Maybe() + return mockCache +} + +// hasDecision reports whether any partial result carries the given decision at +// the given tier. +func hasDecision(results []model.StageResult, decision model.Decision, tier int) bool { + for _, r := range results { + if r.Decision == decision && r.Tier == tier { + return true + } + } + return false +} + +var errStore = errors.New("dial tcp 10.0.0.6:6379: i/o timeout") + +// specRef: proxy-filtering-behaviour.md #I1 +// specRef: proxy-filtering-behaviour.md #I8 +func TestDomainFilterExecute_StageError_Unavailable(t *testing.T) { + const profileID = "stage-error-i1" + rec := &stageErrorRecorder{} + f := NewDomainFilter(nil, failingMembershipCache(), nil) + f.Metrics = rec + + reqCtx := stageErrReqCtx(t, profileID, map[string]string{}) + err := f.Execute(context.Background(), reqCtx, stageErrDomainDctx("example.com")) + + require.Error(t, err, "Execute must surface the stage error") + assert.Equal(t, model.StatusUnavailable, reqCtx.FilterResult.Status) + assert.Nil(t, reqCtx.FilterResult.Reasons, "an unavailable result carries no reasons") + assert.True(t, rec.has(FilterTypeDomain, StageBlocklists), "recorder pairs: %v", rec.pairs) + assert.Equal(t, 1, rec.count(), "only the erroring stage is recorded") + // Successful stages still report their (None) results for logging. + assert.True(t, hasDecision(reqCtx.PartialFilteringResults, model.DecisionNone, TierCustomRules)) +} + +// specRef: proxy-filtering-behaviour.md #I2 +func TestDomainFilterExecute_StageError_WinsOverBlock(t *testing.T) { + const profileID = "stage-error-i2" + rec := &stageErrorRecorder{} + f := NewDomainFilter(nil, failingMembershipCache(), nil) + f.Metrics = rec + + reqCtx := stageErrReqCtx(t, profileID, map[string]string{}) + reqCtx.CustomRules = []map[string]string{{"action": ACTION_BLOCK, "value": "ads.example.com"}} + err := f.Execute(context.Background(), reqCtx, stageErrDomainDctx("ads.example.com")) + + require.Error(t, err) + assert.Equal(t, model.StatusUnavailable, reqCtx.FilterResult.Status, "a stage error must not be downgraded to the partial Block") + assert.True(t, hasDecision(reqCtx.PartialFilteringResults, model.DecisionBlock, TierCustomRules), + "the successful custom-rules Block is still recorded as a partial result") + assert.True(t, rec.has(FilterTypeDomain, StageBlocklists)) +} + +// specRef: proxy-filtering-behaviour.md #I3 +func TestDomainFilterExecute_StageError_WinsOverAllow(t *testing.T) { + const profileID = "stage-error-i3" + rec := &stageErrorRecorder{} + f := NewDomainFilter(nil, failingMembershipCache(), nil) + f.Metrics = rec + + reqCtx := stageErrReqCtx(t, profileID, map[string]string{}) + reqCtx.CustomRules = []map[string]string{{"action": ACTION_ALLOW, "value": "ok.example.com"}} + err := f.Execute(context.Background(), reqCtx, stageErrDomainDctx("ok.example.com")) + + require.Error(t, err) + assert.Equal(t, model.StatusUnavailable, reqCtx.FilterResult.Status, "a stage error must not be downgraded to the partial Allow") + assert.True(t, hasDecision(reqCtx.PartialFilteringResults, model.DecisionAllow, TierCustomRules)) + assert.True(t, rec.has(FilterTypeDomain, StageBlocklists)) +} + +// specRef: proxy-filtering-behaviour.md #I4 +// specRef: proxy-filtering-behaviour.md #I8 +func TestIPFilterExecute_StageError_DiscardsUpstreamAnswer(t *testing.T) { + const profileID = "stage-error-i4" + rec := &stageErrorRecorder{} + // nil catalog/ASN lookup: filterServices is inert; nil rebinding config: inert. + f := NewIPFilter(nil, failingMembershipCache(), nil, nil, nil, nil) + f.Metrics = rec + + reqCtx := stageErrReqCtx(t, profileID, map[string]string{}) + // Domain phase ran and produced a Processed (None) result. + reqCtx.PartialFilteringResults = []model.StageResult{{Decision: model.DecisionNone, Tier: TierBlocklists}} + reqCtx.FilterResult = model.FilterResult{Status: model.StatusProcessed} + + err := f.Execute(context.Background(), reqCtx, stageErrCNAMEDctx("metrics.shop.example")) + + require.Error(t, err) + assert.Equal(t, model.StatusUnavailable, reqCtx.FilterResult.Status, "IP-phase store error must not fall through to Processed") + assert.True(t, rec.has(FilterTypeIP, StageCNAME), "recorder pairs: %v", rec.pairs) + assert.Equal(t, 1, rec.count()) +} + +// specRef: proxy-filtering-behaviour.md #I5 +func TestExecute_NoErrors_NoMatches_Processed(t *testing.T) { + const profileID = "stage-error-i5" + rec := &stageErrorRecorder{} + + t.Run("domain phase", func(t *testing.T) { + mockCache := mocks.NewCache(t) + mockCache.EXPECT().GetBlocklistEntry(mock.Anything, stageErrBlocklistID, mock.Anything).Return(false, nil).Maybe() + + f := NewDomainFilter(nil, mockCache, nil) + f.Metrics = rec + reqCtx := stageErrReqCtx(t, profileID, map[string]string{}) + + err := f.Execute(context.Background(), reqCtx, stageErrDomainDctx("example.com")) + require.NoError(t, err) + assert.Equal(t, model.StatusProcessed, reqCtx.FilterResult.Status) + }) + + t.Run("ip phase", func(t *testing.T) { + // No CNAME in the answer: the IP phase never touches the store. + f := NewIPFilter(nil, mocks.NewCache(t), nil, nil, nil, nil) + f.Metrics = rec + reqCtx := stageErrReqCtx(t, profileID, map[string]string{}) + + err := f.Execute(context.Background(), reqCtx, dnsCtxWithAAnswer(t, "93.184.216.34")) + require.NoError(t, err) + assert.Equal(t, model.StatusProcessed, reqCtx.FilterResult.Status) + }) + + assert.Equal(t, 0, rec.count(), "no stage error must be recorded when nothing failed") +} + +// specRef: proxy-filtering-behaviour.md #I7 +func TestIPFilterExecute_CatalogUnavailable_Inert(t *testing.T) { + const ( + profileID = "stage-error-i7" + asn = uint(15169) + ) + rec := &stageErrorRecorder{} + f := NewIPFilter(nil, mocks.NewCache(t), staticCatalogErr{err: errors.New("catalog load")}, staticASNLookup{asn: asn}, nil, nil) + f.Metrics = rec + + reqCtx := stageErrReqCtx(t, profileID, map[string]string{}) + reqCtx.BlockedServices = []string{"google"} + err := f.Execute(context.Background(), reqCtx, dnsCtxWithAAnswer(t, "8.8.8.8")) + + require.NoError(t, err, "a local catalog load failure is not a store error") + assert.Equal(t, model.StatusProcessed, reqCtx.FilterResult.Status) + assert.Equal(t, 0, rec.count()) +} + +// Execute must tolerate a nil metrics recorder (tests and tools construct +// filters without one). +// specRef: proxy-filtering-behaviour.md #I1 +func TestDomainFilterExecute_StageError_NilRecorder(t *testing.T) { + const profileID = "stage-error-nil-recorder" + f := NewDomainFilter(nil, failingMembershipCache(), nil) + reqCtx := stageErrReqCtx(t, profileID, map[string]string{}) + + require.NotPanics(t, func() { _ = f.Execute(context.Background(), reqCtx, stageErrDomainDctx("example.com")) }) + assert.Equal(t, model.StatusUnavailable, reqCtx.FilterResult.Status) +} + +// applyDefaultRule reads the privacy settings already carried by the request +// context; a strict mock with no expectations fails the test if the stage +// reaches for the store. +// specRef: proxy-filtering-behaviour.md #I5 +func TestApplyDefaultRule_ReadsRequestContext_NoCacheCall(t *testing.T) { + const profileID = "default-rule-no-redis" + f := NewDomainFilter(nil, mocks.NewCache(t), nil) + + // No blocklist subscriptions, so the blocklists stage makes no lookups either. + noBlocklists := func(privacy map[string]string) *requestcontext.RequestContext { + reqCtx := stageErrReqCtx(t, profileID, privacy) + reqCtx.Blocklists = nil + return reqCtx + } + + t.Run("stage direct", func(t *testing.T) { + res, err := f.applyDefaultRule(context.Background(), noBlocklists(map[string]string{DEFAULT_RULE: RULE_BLOCK}), stageErrDomainDctx("anything.example.com")) + require.NoError(t, err) + assert.Equal(t, model.DecisionBlock, res.Decision) + assert.Equal(t, TierDefaultRule, res.Tier) + assert.Contains(t, res.Reasons, DEFAULT_RULE) + }) + + t.Run("stage direct allow", func(t *testing.T) { + res, err := f.applyDefaultRule(context.Background(), noBlocklists(map[string]string{DEFAULT_RULE: RULE_ALLOW}), stageErrDomainDctx("anything.example.com")) + require.NoError(t, err) + assert.Equal(t, model.DecisionNone, res.Decision) + }) + + t.Run("through Execute", func(t *testing.T) { + reqCtx := noBlocklists(map[string]string{DEFAULT_RULE: RULE_BLOCK}) + err := f.Execute(context.Background(), reqCtx, stageErrDomainDctx("anything.example.com")) + require.NoError(t, err) + assert.Equal(t, model.StatusBlocked, reqCtx.FilterResult.Status) + assert.Contains(t, reqCtx.FilterResult.Reasons, DEFAULT_RULE) + }) +} + +// The whole filter path — both phases, every stage armed — reaches the store +// only for blocklist membership. Every other per-profile input comes from the +// settings batch on the request context (Section I note). A strict mockery +// mock fails the test on any call without an expectation. +// specRef: proxy-filtering-behaviour.md #I8 +func TestFilterPath_OnlyBlocklistMembershipHitsStore(t *testing.T) { + const ( + profileID = "store-boundary" + asn = uint(15169) + answerIP = "93.184.216.34" + ) + blocklists := []string{"bl1", "bl2"} + + mockCache := mocks.NewCache(t) + for _, bl := range blocklists { + mockCache.EXPECT().GetBlocklistEntry(mock.Anything, bl, mock.Anything).Return(false, nil) + } + + domainFilter := NewDomainFilter(nil, mockCache, staticCatalog{cat: googleCatalogWithASN(asn)}) + ipFilter := NewIPFilter(nil, mockCache, staticCatalog{cat: googleCatalogWithASN(asn)}, staticASNLookup{asn: asn + 1}, nil, nil) + + reqCtx := stageErrReqCtx(t, profileID, map[string]string{SUBDOMAINS_RULE: RULE_BLOCK}) + reqCtx.Blocklists = blocklists + reqCtx.BlockedServices = []string{"google"} + reqCtx.CustomRules = []map[string]string{ + {"action": ACTION_BLOCK, "value": "ads.other.example", "syntax": "domain"}, + {"action": ACTION_ALLOW, "value": "10.9.8.7", "syntax": "ip4_addr"}, + {"action": ACTION_ALLOW, "value": "AS64496", "syntax": "asn"}, + } + + // Domain phase: QNAME plus parent-walk candidates, each against both lists. + dctx := stageErrDomainDctx("www.shop.example.com") + require.NoError(t, domainFilter.Execute(context.Background(), reqCtx, dctx)) + assert.Equal(t, model.StatusProcessed, reqCtx.FilterResult.Status) + + // IP phase with a CNAME hop: only the target's membership is looked up. + dctx.Res = buildCNAMEChainResponse("www.shop.example.com", dns.TypeA, []string{"edge.cdn.example"}, answerIP) + require.NoError(t, ipFilter.Execute(context.Background(), reqCtx, dctx)) + assert.Equal(t, model.StatusProcessed, reqCtx.FilterResult.Status) + + for _, call := range mockCache.Calls { + assert.Equal(t, "GetBlocklistEntry", call.Method, "only blocklist membership may reach the store") + } + assert.NotEmpty(t, mockCache.Calls, "the membership lookup itself must still be live") +} diff --git a/proxy/filter/stages.go b/proxy/filter/stages.go new file mode 100644 index 00000000..b5d5eb69 --- /dev/null +++ b/proxy/filter/stages.go @@ -0,0 +1,93 @@ +package filter + +import ( + "context" + "errors" + "fmt" + "sync" + "time" + + "github.com/AdguardTeam/dnsproxy/proxy" + "github.com/ivpn/dns/proxy/model" + "github.com/ivpn/dns/proxy/requestcontext" +) + +// Stage names double as the `stage` label of +// proxy_dns_filter_stage_errors_total, so they are stable identifiers. +const ( + StageBlocklists = "blocklists" + StageCustomRules = "custom_rules" + StageServiceDomains = "service_domains" + StageDefaultRule = "default_rule" + StageServices = "services" + StageRebinding = "rebinding" + StageCNAME = "cname" +) + +// StoreDeadline bounds the live settings-store reads of one pipeline step +// (admission batch, domain phase, IP phase), each derived from the request +// context so a client hang-up cancels the work. Three steps keep the worst +// case under the 5s most stub resolvers wait before giving up. +const StoreDeadline = time.Second + +// storeContext returns the context for one phase's live store reads. +func storeContext(ctx context.Context) (context.Context, context.CancelFunc) { + return context.WithTimeout(ctx, StoreDeadline) +} + +// StageErrorRecorder receives one call per failed filter stage. A nil recorder +// disables metrics without disabling the failure handling. +type StageErrorRecorder interface { + RecordFilterStageError(phase, stage string) +} + +type stageFunc func(ctx context.Context, reqCtx *requestcontext.RequestContext, dctx *proxy.DNSContext) (*model.StageResult, error) + +type stage struct { + name string + run stageFunc +} + +// runStages executes every stage concurrently and appends the successful +// results to reqCtx.PartialFilteringResults. A stage that returns an error +// contributes no result; every such failure is recorded and the joined error is +// returned, which the caller must treat as StatusUnavailable rather than +// aggregate the partial results (spec: proxy-filtering-behaviour.md Section I). +func runStages(ctx context.Context, phase string, stages []stage, metrics StageErrorRecorder, reqCtx *requestcontext.RequestContext, dctx *proxy.DNSContext) error { + ctx, cancel := storeContext(ctx) + defer cancel() + + type outcome struct { + res *model.StageResult + err error + } + outcomes := make([]outcome, len(stages)) + + var wg sync.WaitGroup + wg.Add(len(stages)) + for i, st := range stages { + go func(i int, st stage) { + defer wg.Done() + res, err := st.run(ctx, reqCtx, dctx) + outcomes[i] = outcome{res: res, err: err} + }(i, st) + } + wg.Wait() + + var errs []error + for i, st := range stages { + out := outcomes[i] + if out.err != nil { + if metrics != nil { + metrics.RecordFilterStageError(phase, st.name) + } + reqCtx.Logger.Debug().Err(out.err).Str("filter_type", phase).Str("stage", st.name).Msg("Filter stage failed") + errs = append(errs, fmt.Errorf("%s/%s: %w", phase, st.name, out.err)) + continue + } + if out.res != nil { + reqCtx.PartialFilteringResults = append(reqCtx.PartialFilteringResults, *out.res) + } + } + return errors.Join(errs...) +} diff --git a/proxy/filter/store_deadline_test.go b/proxy/filter/store_deadline_test.go new file mode 100644 index 00000000..771bae2d --- /dev/null +++ b/proxy/filter/store_deadline_test.go @@ -0,0 +1,58 @@ +package filter + +import ( + "context" + "net/netip" + "testing" + "time" + + "github.com/AdguardTeam/dnsproxy/proxy" + "github.com/ivpn/dns/proxy/mocks" + "github.com/miekg/dns" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +// Live store reads in the filter path must carry a deadline derived from the +// request context, so a dead store cannot hold a query past StoreDeadline and a +// client that hangs up cancels the work. +// specRef: proxy-filtering-behaviour.md #I8 +func TestDomainFilterExecute_MembershipLookupHasRequestDeadline(t *testing.T) { + var seen context.Context + mockCache := mocks.NewCache(t) + mockCache.EXPECT().GetBlocklistEntry(mock.Anything, "bl1", "example.com"). + Run(func(ctx context.Context, _ string, _ string) { seen = ctx }). + Return(false, nil).Once() + + f := NewDomainFilter(nil, mockCache, nil) + reqCtx := newTestReqCtx(t, "deadline-profile") + reqCtx.Blocklists = []string{"bl1"} + + req := new(dns.Msg) + req.SetQuestion("example.com.", dns.TypeA) + dctx := &proxy.DNSContext{Req: req, Addr: netip.MustParseAddrPort("192.0.2.1:53")} + + // A request context that expires sooner than StoreDeadline must win. + parentBudget := StoreDeadline / 4 + parent, cancel := context.WithTimeout(context.Background(), parentBudget) + defer cancel() + + require.NoError(t, f.Execute(parent, reqCtx, dctx)) + require.NotNil(t, seen) + deadline, ok := seen.Deadline() + require.True(t, ok, "membership lookup must run under a deadline") + remaining := time.Until(deadline) + require.Greater(t, remaining, time.Duration(0)) + require.LessOrEqual(t, remaining, parentBudget, "deadline must derive from the request context") + + // Without a parent deadline the phase budget applies. + seen = nil + mockCache.EXPECT().GetBlocklistEntry(mock.Anything, "bl1", "example.com"). + Run(func(ctx context.Context, _ string, _ string) { seen = ctx }). + Return(false, nil).Once() + reqCtx.PartialFilteringResults = nil + require.NoError(t, f.Execute(context.Background(), reqCtx, dctx)) + deadline, ok = seen.Deadline() + require.True(t, ok) + require.LessOrEqual(t, time.Until(deadline), StoreDeadline) +} diff --git a/proxy/go.mod b/proxy/go.mod index 22b88970..83908767 100644 --- a/proxy/go.mod +++ b/proxy/go.mod @@ -11,7 +11,6 @@ require ( github.com/ivpn/dns/libs v0.0.0 github.com/miekg/dns v1.1.72 github.com/oschwald/geoip2-golang v1.13.0 - github.com/patrickmn/go-cache v2.1.0+incompatible github.com/prometheus/client_golang v1.23.2 github.com/quic-go/quic-go v0.60.0 github.com/redis/go-redis/v9 v9.7.3 @@ -112,7 +111,7 @@ require ( golang.org/x/exp v0.0.0-20260611194520-c48552f49976 // indirect golang.org/x/mod v0.38.0 // indirect golang.org/x/net v0.57.0 // indirect - golang.org/x/sync v0.22.0 + golang.org/x/sync v0.22.0 // indirect golang.org/x/sys v0.47.0 // indirect golang.org/x/text v0.40.0 // indirect golang.org/x/tools v0.48.0 // indirect diff --git a/proxy/go.sum b/proxy/go.sum index 45245df8..33d4fd7d 100644 --- a/proxy/go.sum +++ b/proxy/go.sum @@ -150,8 +150,6 @@ github.com/oschwald/geoip2-golang v1.13.0 h1:Q44/Ldc703pasJeP5V9+aFSZFmBN7DKHbNs github.com/oschwald/geoip2-golang v1.13.0/go.mod h1:P9zG+54KPEFOliZ29i7SeYZ/GM6tfEL+rgSn03hYuUo= github.com/oschwald/maxminddb-golang v1.13.0 h1:R8xBorY71s84yO06NgTmQvqvTvlS/bnYZrrWX1MElnU= github.com/oschwald/maxminddb-golang v1.13.0/go.mod h1:BU0z8BfFVhi1LQaonTwwGQlsHUEu9pWNdMfmq4ztm0o= -github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= -github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= diff --git a/proxy/internal/metrics/server_metrics.go b/proxy/internal/metrics/server_metrics.go index c6b8d4e8..62f67f97 100644 --- a/proxy/internal/metrics/server_metrics.go +++ b/proxy/internal/metrics/server_metrics.go @@ -6,6 +6,21 @@ import ( "github.com/prometheus/client_golang/prometheus" ) +// Label values for proxy_dns_filter_stage_errors_total raised before filtering +// starts; the filter phases and stage names are owned by the filter package. +const ( + PhaseAdmission = "admission" + StageProfileSettings = "profile_settings" +) + +// Status label values for proxy_dns_profile_settings_cache_total. +const ( + CacheLookupHit = "hit" + CacheLookupMiss = "miss" + CacheLookupStale = "stale" + CacheLookupUnavailable = "unavailable" +) + // ServerMetrics implements server.Metrics using Prometheus collectors. type ServerMetrics struct { queries *prometheus.CounterVec @@ -15,6 +30,7 @@ type ServerMetrics struct { ipFilterDuration *prometheus.HistogramVec upstreamDuration *prometheus.HistogramVec blocked *prometheus.CounterVec + filterStageErrors *prometheus.CounterVec } // NewServerMetrics creates and registers all server-level Prometheus collectors. @@ -26,7 +42,7 @@ func NewServerMetrics(reg prometheus.Registerer) *ServerMetrics { }, []string{"proto"}), profileCacheLookups: prometheus.NewCounterVec(prometheus.CounterOpts{ Name: "proxy_dns_profile_settings_cache_total", - Help: "Profile settings cache lookups by status.", + Help: "Profile settings lookups by outcome: hit, miss, stale (last-known-good served), unavailable.", }, []string{"status"}), queryDuration: prometheus.NewHistogramVec(prometheus.HistogramOpts{ Name: "proxy_dns_query_duration_seconds", @@ -52,6 +68,10 @@ func NewServerMetrics(reg prometheus.Registerer) *ServerMetrics { Name: "proxy_dns_blocked_total", Help: "Total blocked DNS queries by filter phase.", }, []string{"phase"}), + filterStageErrors: prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "proxy_dns_filter_stage_errors_total", + Help: "Settings-store read failures by pipeline phase and stage; each one answers SERVFAIL.", + }, []string{"phase", "stage"}), } reg.MustRegister( m.queries, @@ -61,6 +81,7 @@ func NewServerMetrics(reg prometheus.Registerer) *ServerMetrics { m.ipFilterDuration, m.upstreamDuration, m.blocked, + m.filterStageErrors, ) return m } @@ -69,11 +90,7 @@ func (m *ServerMetrics) RecordQuery(proto string) { m.queries.WithLabelValues(proto).Inc() } -func (m *ServerMetrics) RecordProfileCacheLookup(hit bool) { - status := "miss" - if hit { - status = "hit" - } +func (m *ServerMetrics) RecordProfileCacheLookup(status string) { m.profileCacheLookups.WithLabelValues(status).Inc() } @@ -96,3 +113,7 @@ func (m *ServerMetrics) RecordUpstreamDuration(upstream string, d time.Duration) func (m *ServerMetrics) RecordBlocked(phase string) { m.blocked.WithLabelValues(phase).Inc() } + +func (m *ServerMetrics) RecordFilterStageError(phase, stage string) { + m.filterStageErrors.WithLabelValues(phase, stage).Inc() +} diff --git a/proxy/internal/metrics/settings_cache.go b/proxy/internal/metrics/settings_cache.go new file mode 100644 index 00000000..4494aff0 --- /dev/null +++ b/proxy/internal/metrics/settings_cache.go @@ -0,0 +1,60 @@ +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +// SettingsCacheSource is what the gauges read; *settingscache.Cache satisfies it. +type SettingsCacheSource interface { + Len() int + Bytes() int64 + StoreAvailable() bool +} + +// SettingsCacheMetrics exposes the state of the in-process profile settings +// cache and the settings-store breaker. Gauges are read at scrape time from +// counters the cache maintains incrementally, so nothing runs per query. +type SettingsCacheMetrics struct { + evictions *prometheus.CounterVec +} + +// NewSettingsCacheMetrics registers the eviction counter. The gauges need the +// cache instance and are registered separately by ObserveSettingsCache. +func NewSettingsCacheMetrics(reg prometheus.Registerer) *SettingsCacheMetrics { + m := &SettingsCacheMetrics{ + evictions: prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "proxy_dns_profile_settings_cache_evictions_total", + Help: "Profile settings cache entries removed, by reason: size (LRU capacity) or deleted (store reported the profile gone).", + }, []string{"reason"}), + } + reg.MustRegister(m.evictions) + return m +} + +// RecordEviction counts one removed entry; matches settingscache's eviction hook. +func (m *SettingsCacheMetrics) RecordEviction(reason string) { + m.evictions.WithLabelValues(reason).Inc() +} + +// ObserveSettingsCache registers gauges that read src at scrape time. +func ObserveSettingsCache(reg prometheus.Registerer, src SettingsCacheSource) { + reg.MustRegister( + prometheus.NewGaugeFunc(prometheus.GaugeOpts{ + Name: "proxy_dns_profile_settings_cache_entries", + Help: "Profiles currently held in the in-process settings cache (fresh and stale).", + }, func() float64 { return float64(src.Len()) }), + prometheus.NewGaugeFunc(prometheus.GaugeOpts{ + Name: "proxy_dns_profile_settings_cache_bytes_estimate", + Help: "Estimated bytes retained by the settings cache; a trend, not heap accounting.", + }, func() float64 { return float64(src.Bytes()) }), + prometheus.NewGaugeFunc(prometheus.GaugeOpts{ + Name: "proxy_dns_settings_store_available", + Help: "1 when the settings store is believed reachable, 0 while the store breaker is open.", + }, func() float64 { + if src.StoreAvailable() { + return 1 + } + return 0 + }), + ) +} diff --git a/proxy/internal/metrics/settings_cache_test.go b/proxy/internal/metrics/settings_cache_test.go new file mode 100644 index 00000000..19f8c9ab --- /dev/null +++ b/proxy/internal/metrics/settings_cache_test.go @@ -0,0 +1,59 @@ +package metrics + +import ( + "testing" + + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type fakeSource struct { + n int + bytes int64 + up bool +} + +func (f fakeSource) Len() int { return f.n } +func (f fakeSource) Bytes() int64 { return f.bytes } +func (f fakeSource) StoreAvailable() bool { return f.up } + +func TestSettingsCacheMetrics_Registration(t *testing.T) { + reg := prometheus.NewRegistry() + m := NewSettingsCacheMetrics(reg) + src := &fakeSource{n: 3, bytes: 4096, up: true} + ObserveSettingsCache(reg, src) + + m.RecordEviction("size") + m.RecordEviction("size") + m.RecordEviction("deleted") + + gathered, err := reg.Gather() + require.NoError(t, err) + got := map[string]float64{} + for _, fam := range gathered { + for _, mt := range fam.GetMetric() { + switch fam.GetType() { + case dto.MetricType_GAUGE: + got[fam.GetName()] = mt.GetGauge().GetValue() + case dto.MetricType_COUNTER: + got[fam.GetName()+"{"+mt.GetLabel()[0].GetValue()+"}"] = mt.GetCounter().GetValue() + } + } + } + assert.Equal(t, 2.0, got["proxy_dns_profile_settings_cache_evictions_total{size}"]) + assert.Equal(t, 1.0, got["proxy_dns_profile_settings_cache_evictions_total{deleted}"]) + assert.Equal(t, 3.0, got["proxy_dns_profile_settings_cache_entries"]) + assert.Equal(t, 4096.0, got["proxy_dns_profile_settings_cache_bytes_estimate"]) + assert.Equal(t, 1.0, got["proxy_dns_settings_store_available"]) + + src.up = false + gathered, err = reg.Gather() + require.NoError(t, err) + for _, fam := range gathered { + if fam.GetName() == "proxy_dns_settings_store_available" { + assert.Equal(t, 0.0, fam.GetMetric()[0].GetGauge().GetValue(), "gauge reads live state at scrape") + } + } +} diff --git a/proxy/internal/settingscache/cache.go b/proxy/internal/settingscache/cache.go new file mode 100644 index 00000000..16a20ef6 --- /dev/null +++ b/proxy/internal/settingscache/cache.go @@ -0,0 +1,184 @@ +// Package settingscache keeps the last successfully fetched settings of each +// profile in process. An entry is fresh for the configured TTL and is kept +// beyond that, as last-known-good, until the LRU evicts it; serving a stale +// entry is how the proxy keeps filtering correctly while the settings store is +// unreachable (spec: proxy-request-admission-behaviour.md Q13). +package settingscache + +import ( + "sync" + "sync/atomic" + "time" + + lru "github.com/hashicorp/golang-lru/v2" + "github.com/ivpn/dns/proxy/model" +) + +// State describes what Get found for a profile. +type State int + +const ( + // Miss means no entry exists; the store must be consulted. + Miss State = iota + // Fresh means the entry is within its TTL and can be used as is. + Fresh + // Stale means the entry is past its TTL: refresh if the store is + // reachable, otherwise serve it as last-known-good. + Stale +) + +// DefaultProbeInterval bounds how often a failing store is retried. +const DefaultProbeInterval = time.Second + +// Eviction reasons reported to the eviction hook. +const ( + EvictionReasonSize = "size" // LRU capacity reached + EvictionReasonDeleted = "deleted" // store reported the profile gone +) + +type entry struct { + settings *model.ProfileSettings + fetchedAt time.Time + // size is the retained-bytes estimate computed once at Put. + size int64 +} + +// Option configures a Cache at construction time. +type Option func(*Cache) + +// WithClock replaces the time source; for tests that age entries. +func WithClock(now func() time.Time) Option { + return func(c *Cache) { c.now = now } +} + +// WithEvictionHook receives one call per evicted entry with its reason. +func WithEvictionHook(hook func(reason string)) Option { + return func(c *Cache) { c.onEvict = hook } +} + +// Cache is safe for concurrent use. +type Cache struct { + ttl time.Duration + probeInterval time.Duration + entries *lru.Cache[string, *entry] + now func() time.Time + onEvict func(reason string) + // mu serialises writers so the bytes total tracks the LRU exactly; Get + // only takes the LRU's own lock. evictReason is set under mu around an + // explicit removal so the LRU callback reports it instead of "size". + mu sync.Mutex + evictReason string + bytes atomic.Int64 + // retryAt is the unix-nano instant before which store fetches are + // refused; zero means the store is believed healthy. + retryAt atomic.Int64 +} + +// New creates a cache holding at most size entries. ttl <= 0 disables +// expiry (every entry stays Fresh), matching PROFILE_SETTINGS_CACHE_TTL=0. +func New(ttl time.Duration, size int, opts ...Option) (*Cache, error) { + c := &Cache{ + ttl: ttl, + probeInterval: DefaultProbeInterval, + now: time.Now, + onEvict: func(string) {}, + } + for _, opt := range opts { + opt(c) + } + entries, err := lru.NewWithEvict[string, *entry](size, func(_ string, e *entry) { + // Runs synchronously inside Add (capacity) and Remove (explicit), both + // under mu, so it is the single place the total is decremented. + c.bytes.Add(-e.size) + reason := c.evictReason + if reason == "" { + reason = EvictionReasonSize + } + c.onEvict(reason) + }) + if err != nil { + return nil, err + } + c.entries = entries + return c, nil +} + +// Get returns the cached settings for id and how current they are. The +// settings are nil only when the state is Miss. +func (c *Cache) Get(id string) (*model.ProfileSettings, State) { + e, ok := c.entries.Get(id) + if !ok { + return nil, Miss + } + if c.ttl > 0 && c.now().Sub(e.fetchedAt) >= c.ttl { + return e.settings, Stale + } + return e.settings, Fresh +} + +// Put stores a successful fetch. The size estimate is taken here, once per +// fill, so the per-query Get path carries no accounting. +func (c *Cache) Put(id string, settings *model.ProfileSettings) { + e := &entry{settings: settings, fetchedAt: c.now(), size: estimateBytes(settings)} + c.mu.Lock() + defer c.mu.Unlock() + if old, ok := c.entries.Peek(id); ok { + // Replacing in place does not fire the LRU's eviction callback. + c.bytes.Add(-old.size) + } + c.entries.Add(id, e) + c.bytes.Add(e.size) +} + +// Evict forgets a profile, e.g. when the store reports it no longer exists. +func (c *Cache) Evict(id string) { + c.mu.Lock() + defer c.mu.Unlock() + c.evictReason = EvictionReasonDeleted + c.entries.Remove(id) // accounting happens in the LRU callback + c.evictReason = "" +} + +// Len returns the number of cached profiles. +func (c *Cache) Len() int { + return c.entries.Len() +} + +// Bytes returns the estimated bytes retained by all cached entries. +func (c *Cache) Bytes() int64 { + return c.bytes.Load() +} + +// StoreAvailable reports whether the settings store is currently believed +// reachable (no failure mark or probe pending). +func (c *Cache) StoreAvailable() bool { + return c.retryAt.Load() == 0 +} + +// FetchAllowed reports whether the store may be queried now. While the store +// is marked failed, exactly one caller per probe interval is let through, so +// an outage costs one probe per interval instead of one timeout per query. +func (c *Cache) FetchAllowed() bool { + retryAt := c.retryAt.Load() + if retryAt == 0 { + return true + } + now := c.now().UnixNano() + if now < retryAt { + return false + } + return c.retryAt.CompareAndSwap(retryAt, now+int64(c.probeInterval)) +} + +// StoreFailed marks the store unreachable after a connection-level error. It +// reports true on the transition from healthy to failed, so callers can log +// the outage once instead of once per query. +func (c *Cache) StoreFailed() (transition bool) { + return c.retryAt.Swap(c.now().UnixNano()+int64(c.probeInterval)) == 0 +} + +// StoreRecovered clears the failure mark after a successful fetch and reports +// true when the store had been marked failed. +func (c *Cache) StoreRecovered() (transition bool) { + return c.retryAt.Swap(0) != 0 +} diff --git a/proxy/internal/settingscache/cache_test.go b/proxy/internal/settingscache/cache_test.go new file mode 100644 index 00000000..7a360632 --- /dev/null +++ b/proxy/internal/settingscache/cache_test.go @@ -0,0 +1,184 @@ +package settingscache + +import ( + "testing" + "time" + + "github.com/ivpn/dns/proxy/model" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func newTestCache(t *testing.T, ttl time.Duration) (*Cache, *time.Time) { + t.Helper() + now := time.Unix(1_700_000_000, 0) + c, err := New(ttl, 8, WithClock(func() time.Time { return now })) + require.NoError(t, err) + return c, &now +} + +func settings(rule string) *model.ProfileSettings { + return &model.ProfileSettings{Privacy: map[string]string{"default_rule": rule}} +} + +// specRef: proxy-request-admission-behaviour.md #S1 #S2 #S3 #S6 +func TestGet_MissFreshStale(t *testing.T) { + c, now := newTestCache(t, 30*time.Second) + + got, state := c.Get("p1") + assert.Nil(t, got) + assert.Equal(t, Miss, state) + + c.Put("p1", settings("allow")) + got, state = c.Get("p1") + assert.Equal(t, Fresh, state) + assert.Equal(t, "allow", got.Privacy["default_rule"]) + + *now = now.Add(29 * time.Second) + _, state = c.Get("p1") + assert.Equal(t, Fresh, state) + + // Past the TTL the entry is kept and reported as stale, not dropped. + *now = now.Add(time.Second) + got, state = c.Get("p1") + assert.Equal(t, Stale, state) + assert.Equal(t, "allow", got.Privacy["default_rule"]) + + *now = now.Add(48 * time.Hour) + got, state = c.Get("p1") + assert.Equal(t, Stale, state) + assert.NotNil(t, got) +} + +// specRef: proxy-request-admission-behaviour.md #S5 +func TestGet_ZeroTTLNeverExpires(t *testing.T) { + c, now := newTestCache(t, 0) + c.Put("p1", settings("block")) + *now = now.Add(365 * 24 * time.Hour) + _, state := c.Get("p1") + assert.Equal(t, Fresh, state) +} + +// specRef: proxy-request-admission-behaviour.md #S2 +func TestPut_RefreshResetsAge(t *testing.T) { + c, now := newTestCache(t, 30*time.Second) + c.Put("p1", settings("allow")) + *now = now.Add(time.Minute) + _, state := c.Get("p1") + assert.Equal(t, Stale, state) + + c.Put("p1", settings("block")) + got, state := c.Get("p1") + assert.Equal(t, Fresh, state) + assert.Equal(t, "block", got.Privacy["default_rule"]) +} + +// specRef: proxy-request-admission-behaviour.md #S8 +func TestEvict(t *testing.T) { + c, _ := newTestCache(t, 30*time.Second) + c.Put("p1", settings("allow")) + c.Evict("p1") + _, state := c.Get("p1") + assert.Equal(t, Miss, state) + assert.Equal(t, 0, c.Len()) +} + +// specRef: proxy-request-admission-behaviour.md #S7 +func TestSizeBound(t *testing.T) { + c, err := New(time.Minute, 2) + require.NoError(t, err) + c.Put("a", settings("allow")) + c.Put("b", settings("allow")) + c.Put("c", settings("allow")) + assert.Equal(t, 2, c.Len()) + _, state := c.Get("a") + assert.Equal(t, Miss, state, "least recently used entry is evicted") +} + +// specRef: proxy-request-admission-behaviour.md #S9 +func TestBreaker_OneProbePerInterval(t *testing.T) { + c, now := newTestCache(t, 30*time.Second) + assert.True(t, c.FetchAllowed(), "healthy store: every fetch allowed") + assert.True(t, c.FetchAllowed()) + + assert.True(t, c.StoreFailed(), "first failure is the healthy→failed transition") + assert.False(t, c.StoreFailed(), "repeated failures are not transitions") + assert.False(t, c.FetchAllowed(), "just failed: no fetch until the probe interval passes") + + *now = now.Add(DefaultProbeInterval - time.Millisecond) + assert.False(t, c.FetchAllowed()) + + *now = now.Add(time.Millisecond) + assert.True(t, c.FetchAllowed(), "first caller after the interval probes") + assert.False(t, c.FetchAllowed(), "second caller in the same interval does not") + + *now = now.Add(DefaultProbeInterval) + assert.True(t, c.FetchAllowed(), "probe re-arms once per interval while failing") + + assert.True(t, c.StoreRecovered(), "recovery after a failure is a transition") + assert.False(t, c.StoreRecovered(), "recovery while healthy is not") + assert.True(t, c.FetchAllowed()) + assert.True(t, c.FetchAllowed(), "recovered: no gating") +} + +// specRef: proxy-request-admission-behaviour.md #S7 #S8 #S11 +func TestBytesAccounting(t *testing.T) { + var reasons []string + c, err := New(time.Minute, 2, WithEvictionHook(func(r string) { reasons = append(reasons, r) })) + require.NoError(t, err) + assert.Zero(t, c.Bytes()) + + small := settings("allow") + c.Put("a", small) + sizeA := c.Bytes() + assert.Greater(t, sizeA, int64(entryOverhead), "an entry costs more than its fixed overhead") + + heavy := &model.ProfileSettings{Privacy: map[string]string{"default_rule": "block"}} + for i := 0; i < 100; i++ { + heavy.CustomRules = append(heavy.CustomRules, map[string]string{"value": "*.tracker.example", "action": "block", "syntax": "domain"}) + } + c.Put("b", heavy) + sizeB := c.Bytes() - sizeA + assert.Greater(t, sizeB, 100*int64(mapOverhead), "rules dominate a heavy entry") + + // Replacing in place swaps the old size for the new one. + c.Put("a", heavy) + assert.Equal(t, 2*sizeB, c.Bytes()) + assert.Empty(t, reasons, "in-place replacement is not an eviction") + + // Capacity eviction removes the least recently used entry and reports it. + c.Put("c", small) + assert.Equal(t, 2, c.Len()) + assert.Equal(t, sizeB+sizeA, c.Bytes()) + assert.Equal(t, []string{EvictionReasonSize}, reasons) + + // Explicit eviction reports "deleted"; evicting an unknown key is a no-op. + c.Evict("c") + c.Evict("missing") + assert.Equal(t, sizeB, c.Bytes()) + assert.Equal(t, []string{EvictionReasonSize, EvictionReasonDeleted}, reasons) + + c.Evict("a") + assert.Zero(t, c.Bytes()) + assert.Zero(t, c.Len()) +} + +// specRef: proxy-request-admission-behaviour.md #S9 +func TestStoreAvailable(t *testing.T) { + c, now := newTestCache(t, time.Minute) + assert.True(t, c.StoreAvailable()) + c.StoreFailed() + assert.False(t, c.StoreAvailable()) + *now = now.Add(2 * DefaultProbeInterval) + assert.False(t, c.StoreAvailable(), "a due probe does not mean the store is back") + c.StoreRecovered() + assert.True(t, c.StoreAvailable()) +} + +// specRef: proxy-request-admission-behaviour.md #S11 +func TestEstimateBytes_Shapes(t *testing.T) { + assert.Equal(t, int64(entryOverhead), estimateBytes(nil)) + assert.Equal(t, int64(entryOverhead), estimateBytes(&model.ProfileSettings{})) + one := estimateBytes(&model.ProfileSettings{Privacy: map[string]string{"k": "vv"}}) + assert.Equal(t, int64(entryOverhead+mapOverhead+kvOverhead+3), one) +} diff --git a/proxy/internal/settingscache/estimate.go b/proxy/internal/settingscache/estimate.go new file mode 100644 index 00000000..c8eabfc6 --- /dev/null +++ b/proxy/internal/settingscache/estimate.go @@ -0,0 +1,45 @@ +package settingscache + +import "github.com/ivpn/dns/proxy/model" + +// Fixed per-object overheads for the estimate: the entry with its slice +// headers and LRU node, a map header, and a key/value pair of string headers. +// The result is a retained-bytes trend, not heap accounting. +const ( + entryOverhead = 256 + mapOverhead = 48 + kvOverhead = 32 + stringHeader = 16 +) + +// estimateBytes approximates the memory retained by one cached profile. +func estimateBytes(s *model.ProfileSettings) int64 { + if s == nil { + return entryOverhead + } + n := int64(entryOverhead) + for _, m := range []map[string]string{s.Privacy, s.Logs, s.DNSSEC, s.RebindingProtection, s.Advanced, s.Statistics} { + n += mapBytes(m) + } + for _, v := range s.Blocklists { + n += stringHeader + int64(len(v)) + } + for _, v := range s.Services { + n += stringHeader + int64(len(v)) + } + for _, r := range s.CustomRules { + n += mapBytes(r) + } + return n +} + +func mapBytes(m map[string]string) int64 { + if m == nil { + return 0 + } + n := int64(mapOverhead) + for k, v := range m { + n += kvOverhead + int64(len(k)) + int64(len(v)) + } + return n +} diff --git a/proxy/mocks/cache.go b/proxy/mocks/cache.go index 28aa1268..4dda7d77 100644 --- a/proxy/mocks/cache.go +++ b/proxy/mocks/cache.go @@ -143,455 +143,46 @@ func (_c *Cache_GetBlocklistEntry_Call) RunAndReturn(run func(ctx context.Contex return _c } -// GetCustomRulesHash provides a mock function for the type Cache -func (_mock *Cache) GetCustomRulesHash(ctx context.Context, hashId string) (map[string]string, error) { - ret := _mock.Called(ctx, hashId) - - if len(ret) == 0 { - panic("no return value specified for GetCustomRulesHash") - } - - var r0 map[string]string - var r1 error - if returnFunc, ok := ret.Get(0).(func(context.Context, string) (map[string]string, error)); ok { - return returnFunc(ctx, hashId) - } - if returnFunc, ok := ret.Get(0).(func(context.Context, string) map[string]string); ok { - r0 = returnFunc(ctx, hashId) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(map[string]string) - } - } - if returnFunc, ok := ret.Get(1).(func(context.Context, string) error); ok { - r1 = returnFunc(ctx, hashId) - } else { - r1 = ret.Error(1) - } - return r0, r1 -} - -// Cache_GetCustomRulesHash_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetCustomRulesHash' -type Cache_GetCustomRulesHash_Call struct { - *mock.Call -} - -// GetCustomRulesHash is a helper method to define mock.On call -// - ctx context.Context -// - hashId string -func (_e *Cache_Expecter) GetCustomRulesHash(ctx interface{}, hashId interface{}) *Cache_GetCustomRulesHash_Call { - return &Cache_GetCustomRulesHash_Call{Call: _e.mock.On("GetCustomRulesHash", ctx, hashId)} -} - -func (_c *Cache_GetCustomRulesHash_Call) Run(run func(ctx context.Context, hashId string)) *Cache_GetCustomRulesHash_Call { - _c.Call.Run(func(args mock.Arguments) { - var arg0 context.Context - if args[0] != nil { - arg0 = args[0].(context.Context) - } - var arg1 string - if args[1] != nil { - arg1 = args[1].(string) - } - run( - arg0, - arg1, - ) - }) - return _c -} - -func (_c *Cache_GetCustomRulesHash_Call) Return(stringToString map[string]string, err error) *Cache_GetCustomRulesHash_Call { - _c.Call.Return(stringToString, err) - return _c -} - -func (_c *Cache_GetCustomRulesHash_Call) RunAndReturn(run func(ctx context.Context, hashId string) (map[string]string, error)) *Cache_GetCustomRulesHash_Call { - _c.Call.Return(run) - return _c -} - -// GetCustomRulesHashes provides a mock function for the type Cache -func (_mock *Cache) GetCustomRulesHashes(ctx context.Context, profileId string) ([]string, error) { - ret := _mock.Called(ctx, profileId) - - if len(ret) == 0 { - panic("no return value specified for GetCustomRulesHashes") - } - - var r0 []string - var r1 error - if returnFunc, ok := ret.Get(0).(func(context.Context, string) ([]string, error)); ok { - return returnFunc(ctx, profileId) - } - if returnFunc, ok := ret.Get(0).(func(context.Context, string) []string); ok { - r0 = returnFunc(ctx, profileId) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]string) - } - } - if returnFunc, ok := ret.Get(1).(func(context.Context, string) error); ok { - r1 = returnFunc(ctx, profileId) - } else { - r1 = ret.Error(1) - } - return r0, r1 -} - -// Cache_GetCustomRulesHashes_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetCustomRulesHashes' -type Cache_GetCustomRulesHashes_Call struct { - *mock.Call -} - -// GetCustomRulesHashes is a helper method to define mock.On call -// - ctx context.Context -// - profileId string -func (_e *Cache_Expecter) GetCustomRulesHashes(ctx interface{}, profileId interface{}) *Cache_GetCustomRulesHashes_Call { - return &Cache_GetCustomRulesHashes_Call{Call: _e.mock.On("GetCustomRulesHashes", ctx, profileId)} -} - -func (_c *Cache_GetCustomRulesHashes_Call) Run(run func(ctx context.Context, profileId string)) *Cache_GetCustomRulesHashes_Call { - _c.Call.Run(func(args mock.Arguments) { - var arg0 context.Context - if args[0] != nil { - arg0 = args[0].(context.Context) - } - var arg1 string - if args[1] != nil { - arg1 = args[1].(string) - } - run( - arg0, - arg1, - ) - }) - return _c -} - -func (_c *Cache_GetCustomRulesHashes_Call) Return(strings []string, err error) *Cache_GetCustomRulesHashes_Call { - _c.Call.Return(strings, err) - return _c -} - -func (_c *Cache_GetCustomRulesHashes_Call) RunAndReturn(run func(ctx context.Context, profileId string) ([]string, error)) *Cache_GetCustomRulesHashes_Call { - _c.Call.Return(run) - return _c -} - -// GetProfileAdvancedSettings provides a mock function for the type Cache -func (_mock *Cache) GetProfileAdvancedSettings(ctx context.Context, profileId string) (map[string]string, error) { - ret := _mock.Called(ctx, profileId) - - if len(ret) == 0 { - panic("no return value specified for GetProfileAdvancedSettings") - } - - var r0 map[string]string - var r1 error - if returnFunc, ok := ret.Get(0).(func(context.Context, string) (map[string]string, error)); ok { - return returnFunc(ctx, profileId) - } - if returnFunc, ok := ret.Get(0).(func(context.Context, string) map[string]string); ok { - r0 = returnFunc(ctx, profileId) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(map[string]string) - } - } - if returnFunc, ok := ret.Get(1).(func(context.Context, string) error); ok { - r1 = returnFunc(ctx, profileId) - } else { - r1 = ret.Error(1) - } - return r0, r1 -} - -// Cache_GetProfileAdvancedSettings_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetProfileAdvancedSettings' -type Cache_GetProfileAdvancedSettings_Call struct { - *mock.Call -} - -// GetProfileAdvancedSettings is a helper method to define mock.On call -// - ctx context.Context -// - profileId string -func (_e *Cache_Expecter) GetProfileAdvancedSettings(ctx interface{}, profileId interface{}) *Cache_GetProfileAdvancedSettings_Call { - return &Cache_GetProfileAdvancedSettings_Call{Call: _e.mock.On("GetProfileAdvancedSettings", ctx, profileId)} -} - -func (_c *Cache_GetProfileAdvancedSettings_Call) Run(run func(ctx context.Context, profileId string)) *Cache_GetProfileAdvancedSettings_Call { - _c.Call.Run(func(args mock.Arguments) { - var arg0 context.Context - if args[0] != nil { - arg0 = args[0].(context.Context) - } - var arg1 string - if args[1] != nil { - arg1 = args[1].(string) - } - run( - arg0, - arg1, - ) - }) - return _c -} - -func (_c *Cache_GetProfileAdvancedSettings_Call) Return(stringToString map[string]string, err error) *Cache_GetProfileAdvancedSettings_Call { - _c.Call.Return(stringToString, err) - return _c -} - -func (_c *Cache_GetProfileAdvancedSettings_Call) RunAndReturn(run func(ctx context.Context, profileId string) (map[string]string, error)) *Cache_GetProfileAdvancedSettings_Call { - _c.Call.Return(run) - return _c -} - -// GetProfileBlocklists provides a mock function for the type Cache -func (_mock *Cache) GetProfileBlocklists(ctx context.Context, profileId string) ([]string, error) { - ret := _mock.Called(ctx, profileId) - - if len(ret) == 0 { - panic("no return value specified for GetProfileBlocklists") - } - - var r0 []string - var r1 error - if returnFunc, ok := ret.Get(0).(func(context.Context, string) ([]string, error)); ok { - return returnFunc(ctx, profileId) - } - if returnFunc, ok := ret.Get(0).(func(context.Context, string) []string); ok { - r0 = returnFunc(ctx, profileId) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]string) - } - } - if returnFunc, ok := ret.Get(1).(func(context.Context, string) error); ok { - r1 = returnFunc(ctx, profileId) - } else { - r1 = ret.Error(1) - } - return r0, r1 -} - -// Cache_GetProfileBlocklists_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetProfileBlocklists' -type Cache_GetProfileBlocklists_Call struct { - *mock.Call -} - -// GetProfileBlocklists is a helper method to define mock.On call -// - ctx context.Context -// - profileId string -func (_e *Cache_Expecter) GetProfileBlocklists(ctx interface{}, profileId interface{}) *Cache_GetProfileBlocklists_Call { - return &Cache_GetProfileBlocklists_Call{Call: _e.mock.On("GetProfileBlocklists", ctx, profileId)} -} - -func (_c *Cache_GetProfileBlocklists_Call) Run(run func(ctx context.Context, profileId string)) *Cache_GetProfileBlocklists_Call { - _c.Call.Run(func(args mock.Arguments) { - var arg0 context.Context - if args[0] != nil { - arg0 = args[0].(context.Context) - } - var arg1 string - if args[1] != nil { - arg1 = args[1].(string) - } - run( - arg0, - arg1, - ) - }) - return _c -} - -func (_c *Cache_GetProfileBlocklists_Call) Return(strings []string, err error) *Cache_GetProfileBlocklists_Call { - _c.Call.Return(strings, err) - return _c -} - -func (_c *Cache_GetProfileBlocklists_Call) RunAndReturn(run func(ctx context.Context, profileId string) ([]string, error)) *Cache_GetProfileBlocklists_Call { - _c.Call.Return(run) - return _c -} - -// GetProfileDNSSECSettings provides a mock function for the type Cache -func (_mock *Cache) GetProfileDNSSECSettings(ctx context.Context, profileId string) (map[string]string, error) { - ret := _mock.Called(ctx, profileId) - - if len(ret) == 0 { - panic("no return value specified for GetProfileDNSSECSettings") - } - - var r0 map[string]string - var r1 error - if returnFunc, ok := ret.Get(0).(func(context.Context, string) (map[string]string, error)); ok { - return returnFunc(ctx, profileId) - } - if returnFunc, ok := ret.Get(0).(func(context.Context, string) map[string]string); ok { - r0 = returnFunc(ctx, profileId) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(map[string]string) - } - } - if returnFunc, ok := ret.Get(1).(func(context.Context, string) error); ok { - r1 = returnFunc(ctx, profileId) - } else { - r1 = ret.Error(1) - } - return r0, r1 -} - -// Cache_GetProfileDNSSECSettings_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetProfileDNSSECSettings' -type Cache_GetProfileDNSSECSettings_Call struct { - *mock.Call -} - -// GetProfileDNSSECSettings is a helper method to define mock.On call -// - ctx context.Context -// - profileId string -func (_e *Cache_Expecter) GetProfileDNSSECSettings(ctx interface{}, profileId interface{}) *Cache_GetProfileDNSSECSettings_Call { - return &Cache_GetProfileDNSSECSettings_Call{Call: _e.mock.On("GetProfileDNSSECSettings", ctx, profileId)} -} - -func (_c *Cache_GetProfileDNSSECSettings_Call) Run(run func(ctx context.Context, profileId string)) *Cache_GetProfileDNSSECSettings_Call { - _c.Call.Run(func(args mock.Arguments) { - var arg0 context.Context - if args[0] != nil { - arg0 = args[0].(context.Context) - } - var arg1 string - if args[1] != nil { - arg1 = args[1].(string) - } - run( - arg0, - arg1, - ) - }) - return _c -} - -func (_c *Cache_GetProfileDNSSECSettings_Call) Return(stringToString map[string]string, err error) *Cache_GetProfileDNSSECSettings_Call { - _c.Call.Return(stringToString, err) - return _c -} - -func (_c *Cache_GetProfileDNSSECSettings_Call) RunAndReturn(run func(ctx context.Context, profileId string) (map[string]string, error)) *Cache_GetProfileDNSSECSettings_Call { - _c.Call.Return(run) - return _c -} - -// GetProfileLogsSettings provides a mock function for the type Cache -func (_mock *Cache) GetProfileLogsSettings(ctx context.Context, profileId string) (map[string]string, error) { - ret := _mock.Called(ctx, profileId) - - if len(ret) == 0 { - panic("no return value specified for GetProfileLogsSettings") - } - - var r0 map[string]string - var r1 error - if returnFunc, ok := ret.Get(0).(func(context.Context, string) (map[string]string, error)); ok { - return returnFunc(ctx, profileId) - } - if returnFunc, ok := ret.Get(0).(func(context.Context, string) map[string]string); ok { - r0 = returnFunc(ctx, profileId) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(map[string]string) - } - } - if returnFunc, ok := ret.Get(1).(func(context.Context, string) error); ok { - r1 = returnFunc(ctx, profileId) - } else { - r1 = ret.Error(1) - } - return r0, r1 -} - -// Cache_GetProfileLogsSettings_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetProfileLogsSettings' -type Cache_GetProfileLogsSettings_Call struct { - *mock.Call -} - -// GetProfileLogsSettings is a helper method to define mock.On call -// - ctx context.Context -// - profileId string -func (_e *Cache_Expecter) GetProfileLogsSettings(ctx interface{}, profileId interface{}) *Cache_GetProfileLogsSettings_Call { - return &Cache_GetProfileLogsSettings_Call{Call: _e.mock.On("GetProfileLogsSettings", ctx, profileId)} -} - -func (_c *Cache_GetProfileLogsSettings_Call) Run(run func(ctx context.Context, profileId string)) *Cache_GetProfileLogsSettings_Call { - _c.Call.Run(func(args mock.Arguments) { - var arg0 context.Context - if args[0] != nil { - arg0 = args[0].(context.Context) - } - var arg1 string - if args[1] != nil { - arg1 = args[1].(string) - } - run( - arg0, - arg1, - ) - }) - return _c -} - -func (_c *Cache_GetProfileLogsSettings_Call) Return(stringToString map[string]string, err error) *Cache_GetProfileLogsSettings_Call { - _c.Call.Return(stringToString, err) - return _c -} - -func (_c *Cache_GetProfileLogsSettings_Call) RunAndReturn(run func(ctx context.Context, profileId string) (map[string]string, error)) *Cache_GetProfileLogsSettings_Call { - _c.Call.Return(run) - return _c -} - -// GetProfilePrivacySettings provides a mock function for the type Cache -func (_mock *Cache) GetProfilePrivacySettings(ctx context.Context, profileId string) (map[string]string, error) { - ret := _mock.Called(ctx, profileId) +// GetBlocklistExceptionEntry provides a mock function for the type Cache +func (_mock *Cache) GetBlocklistExceptionEntry(ctx context.Context, blocklistId string, domain string) (bool, error) { + ret := _mock.Called(ctx, blocklistId, domain) if len(ret) == 0 { - panic("no return value specified for GetProfilePrivacySettings") + panic("no return value specified for GetBlocklistExceptionEntry") } - var r0 map[string]string + var r0 bool var r1 error - if returnFunc, ok := ret.Get(0).(func(context.Context, string) (map[string]string, error)); ok { - return returnFunc(ctx, profileId) + if returnFunc, ok := ret.Get(0).(func(context.Context, string, string) (bool, error)); ok { + return returnFunc(ctx, blocklistId, domain) } - if returnFunc, ok := ret.Get(0).(func(context.Context, string) map[string]string); ok { - r0 = returnFunc(ctx, profileId) + if returnFunc, ok := ret.Get(0).(func(context.Context, string, string) bool); ok { + r0 = returnFunc(ctx, blocklistId, domain) } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(map[string]string) - } + r0 = ret.Get(0).(bool) } - if returnFunc, ok := ret.Get(1).(func(context.Context, string) error); ok { - r1 = returnFunc(ctx, profileId) + if returnFunc, ok := ret.Get(1).(func(context.Context, string, string) error); ok { + r1 = returnFunc(ctx, blocklistId, domain) } else { r1 = ret.Error(1) } return r0, r1 } -// Cache_GetProfilePrivacySettings_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetProfilePrivacySettings' -type Cache_GetProfilePrivacySettings_Call struct { +// Cache_GetBlocklistExceptionEntry_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetBlocklistExceptionEntry' +type Cache_GetBlocklistExceptionEntry_Call struct { *mock.Call } -// GetProfilePrivacySettings is a helper method to define mock.On call +// GetBlocklistExceptionEntry is a helper method to define mock.On call // - ctx context.Context -// - profileId string -func (_e *Cache_Expecter) GetProfilePrivacySettings(ctx interface{}, profileId interface{}) *Cache_GetProfilePrivacySettings_Call { - return &Cache_GetProfilePrivacySettings_Call{Call: _e.mock.On("GetProfilePrivacySettings", ctx, profileId)} +// - blocklistId string +// - domain string +func (_e *Cache_Expecter) GetBlocklistExceptionEntry(ctx interface{}, blocklistId interface{}, domain interface{}) *Cache_GetBlocklistExceptionEntry_Call { + return &Cache_GetBlocklistExceptionEntry_Call{Call: _e.mock.On("GetBlocklistExceptionEntry", ctx, blocklistId, domain)} } -func (_c *Cache_GetProfilePrivacySettings_Call) Run(run func(ctx context.Context, profileId string)) *Cache_GetProfilePrivacySettings_Call { +func (_c *Cache_GetBlocklistExceptionEntry_Call) Run(run func(ctx context.Context, blocklistId string, domain string)) *Cache_GetBlocklistExceptionEntry_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { @@ -601,88 +192,25 @@ func (_c *Cache_GetProfilePrivacySettings_Call) Run(run func(ctx context.Context if args[1] != nil { arg1 = args[1].(string) } - run( - arg0, - arg1, - ) - }) - return _c -} - -func (_c *Cache_GetProfilePrivacySettings_Call) Return(stringToString map[string]string, err error) *Cache_GetProfilePrivacySettings_Call { - _c.Call.Return(stringToString, err) - return _c -} - -func (_c *Cache_GetProfilePrivacySettings_Call) RunAndReturn(run func(ctx context.Context, profileId string) (map[string]string, error)) *Cache_GetProfilePrivacySettings_Call { - _c.Call.Return(run) - return _c -} - -// GetProfileServicesBlocked provides a mock function for the type Cache -func (_mock *Cache) GetProfileServicesBlocked(ctx context.Context, profileId string) ([]string, error) { - ret := _mock.Called(ctx, profileId) - - if len(ret) == 0 { - panic("no return value specified for GetProfileServicesBlocked") - } - - var r0 []string - var r1 error - if returnFunc, ok := ret.Get(0).(func(context.Context, string) ([]string, error)); ok { - return returnFunc(ctx, profileId) - } - if returnFunc, ok := ret.Get(0).(func(context.Context, string) []string); ok { - r0 = returnFunc(ctx, profileId) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]string) - } - } - if returnFunc, ok := ret.Get(1).(func(context.Context, string) error); ok { - r1 = returnFunc(ctx, profileId) - } else { - r1 = ret.Error(1) - } - return r0, r1 -} - -// Cache_GetProfileServicesBlocked_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetProfileServicesBlocked' -type Cache_GetProfileServicesBlocked_Call struct { - *mock.Call -} - -// GetProfileServicesBlocked is a helper method to define mock.On call -// - ctx context.Context -// - profileId string -func (_e *Cache_Expecter) GetProfileServicesBlocked(ctx interface{}, profileId interface{}) *Cache_GetProfileServicesBlocked_Call { - return &Cache_GetProfileServicesBlocked_Call{Call: _e.mock.On("GetProfileServicesBlocked", ctx, profileId)} -} - -func (_c *Cache_GetProfileServicesBlocked_Call) Run(run func(ctx context.Context, profileId string)) *Cache_GetProfileServicesBlocked_Call { - _c.Call.Run(func(args mock.Arguments) { - var arg0 context.Context - if args[0] != nil { - arg0 = args[0].(context.Context) - } - var arg1 string - if args[1] != nil { - arg1 = args[1].(string) + var arg2 string + if args[2] != nil { + arg2 = args[2].(string) } run( arg0, arg1, + arg2, ) }) return _c } -func (_c *Cache_GetProfileServicesBlocked_Call) Return(strings []string, err error) *Cache_GetProfileServicesBlocked_Call { - _c.Call.Return(strings, err) +func (_c *Cache_GetBlocklistExceptionEntry_Call) Return(b bool, err error) *Cache_GetBlocklistExceptionEntry_Call { + _c.Call.Return(b, err) return _c } -func (_c *Cache_GetProfileServicesBlocked_Call) RunAndReturn(run func(ctx context.Context, profileId string) ([]string, error)) *Cache_GetProfileServicesBlocked_Call { +func (_c *Cache_GetBlocklistExceptionEntry_Call) RunAndReturn(run func(ctx context.Context, blocklistId string, domain string) (bool, error)) *Cache_GetBlocklistExceptionEntry_Call { _c.Call.Return(run) return _c } @@ -754,71 +282,3 @@ func (_c *Cache_GetProfileSettingsBatch_Call) RunAndReturn(run func(ctx context. _c.Call.Return(run) return _c } - -// GetProfileStatisticsSettings provides a mock function for the type Cache -func (_mock *Cache) GetProfileStatisticsSettings(ctx context.Context, profileId string) (map[string]string, error) { - ret := _mock.Called(ctx, profileId) - - if len(ret) == 0 { - panic("no return value specified for GetProfileStatisticsSettings") - } - - var r0 map[string]string - var r1 error - if returnFunc, ok := ret.Get(0).(func(context.Context, string) (map[string]string, error)); ok { - return returnFunc(ctx, profileId) - } - if returnFunc, ok := ret.Get(0).(func(context.Context, string) map[string]string); ok { - r0 = returnFunc(ctx, profileId) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(map[string]string) - } - } - if returnFunc, ok := ret.Get(1).(func(context.Context, string) error); ok { - r1 = returnFunc(ctx, profileId) - } else { - r1 = ret.Error(1) - } - return r0, r1 -} - -// Cache_GetProfileStatisticsSettings_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetProfileStatisticsSettings' -type Cache_GetProfileStatisticsSettings_Call struct { - *mock.Call -} - -// GetProfileStatisticsSettings is a helper method to define mock.On call -// - ctx context.Context -// - profileId string -func (_e *Cache_Expecter) GetProfileStatisticsSettings(ctx interface{}, profileId interface{}) *Cache_GetProfileStatisticsSettings_Call { - return &Cache_GetProfileStatisticsSettings_Call{Call: _e.mock.On("GetProfileStatisticsSettings", ctx, profileId)} -} - -func (_c *Cache_GetProfileStatisticsSettings_Call) Run(run func(ctx context.Context, profileId string)) *Cache_GetProfileStatisticsSettings_Call { - _c.Call.Run(func(args mock.Arguments) { - var arg0 context.Context - if args[0] != nil { - arg0 = args[0].(context.Context) - } - var arg1 string - if args[1] != nil { - arg1 = args[1].(string) - } - run( - arg0, - arg1, - ) - }) - return _c -} - -func (_c *Cache_GetProfileStatisticsSettings_Call) Return(stringToString map[string]string, err error) *Cache_GetProfileStatisticsSettings_Call { - _c.Call.Return(stringToString, err) - return _c -} - -func (_c *Cache_GetProfileStatisticsSettings_Call) RunAndReturn(run func(ctx context.Context, profileId string) (map[string]string, error)) *Cache_GetProfileStatisticsSettings_Call { - _c.Call.Return(run) - return _c -} diff --git a/proxy/mocks/emitter.go b/proxy/mocks/emitter.go index 90a7521a..01b79ce3 100644 --- a/proxy/mocks/emitter.go +++ b/proxy/mocks/emitter.go @@ -51,16 +51,16 @@ func (_m *Emitter) EmitQueryLogs(ctx context.Context, data []model.EventQueryLog return r0 } -// EmitStatistics provides a mock function with given fields: ctx, data -func (_m *Emitter) EmitStatistics(ctx context.Context, data []model.EventStatistics) error { +// EmitServiceStatistics provides a mock function with given fields: ctx, data +func (_m *Emitter) EmitServiceStatistics(ctx context.Context, data []model.ServiceStatistics) error { ret := _m.Called(ctx, data) if len(ret) == 0 { - panic("no return value specified for EmitStatistics") + panic("no return value specified for EmitServiceStatistics") } var r0 error - if rf, ok := ret.Get(0).(func(context.Context, []model.EventStatistics) error); ok { + if rf, ok := ret.Get(0).(func(context.Context, []model.ServiceStatistics) error); ok { r0 = rf(ctx, data) } else { r0 = ret.Error(0) diff --git a/proxy/mocks/filter.go b/proxy/mocks/filter.go index b1caa86b..29b35191 100644 --- a/proxy/mocks/filter.go +++ b/proxy/mocks/filter.go @@ -3,6 +3,8 @@ package mocks import ( + context "context" + proxy "github.com/AdguardTeam/dnsproxy/proxy" mock "github.com/stretchr/testify/mock" @@ -14,17 +16,17 @@ type Filter struct { mock.Mock } -// Execute provides a mock function with given fields: reqCtx, dctx -func (_m *Filter) Execute(reqCtx *requestcontext.RequestContext, dctx *proxy.DNSContext) error { - ret := _m.Called(reqCtx, dctx) +// Execute provides a mock function with given fields: ctx, reqCtx, dctx +func (_m *Filter) Execute(ctx context.Context, reqCtx *requestcontext.RequestContext, dctx *proxy.DNSContext) error { + ret := _m.Called(ctx, reqCtx, dctx) if len(ret) == 0 { panic("no return value specified for Execute") } var r0 error - if rf, ok := ret.Get(0).(func(*requestcontext.RequestContext, *proxy.DNSContext) error); ok { - r0 = rf(reqCtx, dctx) + if rf, ok := ret.Get(0).(func(context.Context, *requestcontext.RequestContext, *proxy.DNSContext) error); ok { + r0 = rf(ctx, reqCtx, dctx) } else { r0 = ret.Error(0) } diff --git a/proxy/model/event.go b/proxy/model/event.go index 373c2903..ba8025a8 100644 --- a/proxy/model/event.go +++ b/proxy/model/event.go @@ -6,9 +6,10 @@ type EventQueryLog struct { Metadata Metadata } -// EventStatistics holds a statistics data +// EventStatistics is one query's counter increments (total 1, blocked and +// dnssec 0 or 1). It carries no identifier and no time on purpose. type EventStatistics struct { - Statistics *Statistics + Queries Queries } type Metadata struct { diff --git a/proxy/model/filter_result.go b/proxy/model/filter_result.go index 15e96deb..5e865a1d 100644 --- a/proxy/model/filter_result.go +++ b/proxy/model/filter_result.go @@ -3,6 +3,9 @@ package model const ( StatusBlocked Status = "blocked" StatusProcessed Status = "processed" + // StatusUnavailable means a settings-store read failed before filtering could + // complete. The query is never resolved upstream and is answered SERVFAIL. + StatusUnavailable Status = "unavailable" ) type Status string diff --git a/proxy/model/profile_settings.go b/proxy/model/profile_settings.go index 06bc84b4..ab8f39d4 100644 --- a/proxy/model/profile_settings.go +++ b/proxy/model/profile_settings.go @@ -1,18 +1,56 @@ package model -// ProfileSettings holds all profile settings fetched in a single batch. +import "errors" + +// ErrSettingsNotFound marks a settings hash that the store answered for but +// that is empty: the profile, or that settings group, does not exist. Every +// other store error is an infrastructure failure. Defined here so both the +// cache and the server can use it without an import cycle. +var ErrSettingsNotFound = errors.New("settings not found") + +// ProfileSettings holds everything the proxy needs to know about one profile, +// fetched in a single batch and cached in process. The filter stages read +// their per-profile inputs from here; only blocklist membership is looked up +// live. type ProfileSettings struct { Privacy map[string]string Logs map[string]string DNSSEC map[string]string RebindingProtection map[string]string Advanced map[string]string + Statistics map[string]string + + // Blocklists is the subscribed blocklist IDs in subscription order. + Blocklists []string + // Services is the blocked service IDs. + Services []string + // CustomRules is every custom rule hash of the profile; each map is the + // rule's Redis hash (value, action, syntax, ...). + CustomRules []map[string]string - // Per-key errors (nil means success). A missing key in Redis returns - // an empty map (not an error), so these only fire on real Redis failures. + // Per-key errors (nil means success). An empty hash wraps + // cache.ErrSettingsNotFound; anything else is a store failure. Lists and + // sets are never "not found": a missing key reads as empty. PrivacyErr error LogsErr error DNSSECErr error RebindingProtectionErr error AdvancedErr error + StatisticsErr error + BlocklistsErr error + ServicesErr error + CustomRulesErr error +} + +// StoreError returns the first per-key error that is a store failure rather +// than an absent hash, or nil when every group was read (present or not). +// Privacy is excluded: its absence is the profile-existence signal and is +// handled by the caller. +func (s *ProfileSettings) StoreError() error { + for _, err := range []error{s.LogsErr, s.DNSSECErr, s.RebindingProtectionErr, s.AdvancedErr, s.StatisticsErr, s.BlocklistsErr, s.ServicesErr, s.CustomRulesErr} { + if err != nil && !errors.Is(err, ErrSettingsNotFound) { + return err + } + } + return nil } diff --git a/proxy/model/service_statistics.go b/proxy/model/service_statistics.go new file mode 100644 index 00000000..ee988b91 --- /dev/null +++ b/proxy/model/service_statistics.go @@ -0,0 +1,54 @@ +package model + +import "time" + +// ServiceStatisticsBucket is the width of one service_statistics document. +const ServiceStatisticsBucket = time.Hour + +// ServiceStatistics is one PoP's query counters for one hour. The type has no +// profile or device field on purpose: counters are summed across every +// profile in memory before anything is written, and every proxy instance in +// a PoP adds into the same document. +type ServiceStatistics struct { + // ID is ":", e.g. "ams1:2026-09-17T13", so concurrent writers + // converge on one document per PoP and hour. + ID string `json:"-" bson:"_id"` + // Timestamp is the start of the hour the counters belong to. + Timestamp time.Time `json:"timestamp" bson:"timestamp"` + Pop string `json:"pop" bson:"pop"` + Queries Queries `json:"queries" bson:"queries"` +} + +// NewServiceStatistics opens an empty document for pop and the bucket +// containing t. +func NewServiceStatistics(pop string, t time.Time) *ServiceStatistics { + bucket := BucketStart(t) + return &ServiceStatistics{ + ID: pop + ":" + bucket.Format("2006-01-02T15"), + Timestamp: bucket, + Pop: pop, + } +} + +// Aggregate adds one event's counters; the bucket is untouched. +func (s *ServiceStatistics) Aggregate(event EventStatistics) { + s.Queries.Add(event.Queries) +} + +// BucketStart returns the UTC start of the bucket containing t. +func BucketStart(t time.Time) time.Time { + return t.UTC().Truncate(ServiceStatisticsBucket) +} + +type Queries struct { + Total int `json:"total" bson:"total"` + Blocked int `json:"blocked" bson:"blocked"` + DNSSEC int `json:"dnssec" bson:"dnssec"` +} + +// Add sums other into q. +func (q *Queries) Add(other Queries) { + q.Total += other.Total + q.Blocked += other.Blocked + q.DNSSEC += other.DNSSEC +} diff --git a/proxy/model/service_statistics_test.go b/proxy/model/service_statistics_test.go new file mode 100644 index 00000000..2c0fc0b7 --- /dev/null +++ b/proxy/model/service_statistics_test.go @@ -0,0 +1,64 @@ +package model + +import ( + "reflect" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +// specRef: proxy-statistics-behaviour.md #Y5 +func TestBucketStart(t *testing.T) { + loc := time.FixedZone("CEST", 2*3600) + tests := []struct { + name string + in time.Time + want time.Time + }{ + {name: "mid-hour is truncated to the hour", in: time.Date(2026, 9, 17, 13, 47, 59, 123456789, time.UTC), want: time.Date(2026, 9, 17, 13, 0, 0, 0, time.UTC)}, + {name: "exact hour is unchanged", in: time.Date(2026, 9, 17, 13, 0, 0, 0, time.UTC), want: time.Date(2026, 9, 17, 13, 0, 0, 0, time.UTC)}, + {name: "local time is normalised to UTC first", in: time.Date(2026, 9, 17, 1, 30, 0, 0, loc), want: time.Date(2026, 9, 16, 23, 0, 0, 0, time.UTC)}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := BucketStart(tt.in) + assert.True(t, got.Equal(tt.want), "got %s want %s", got, tt.want) + assert.Equal(t, time.UTC, got.Location()) + }) + } +} + +// specRef: proxy-statistics-behaviour.md #Y5 #Y6 +func TestNewServiceStatistics_KeyedByPopAndHour(t *testing.T) { + doc := NewServiceStatistics("ams1", time.Date(2026, 9, 17, 13, 47, 1, 0, time.UTC)) + + assert.Equal(t, "ams1:2026-09-17T13", doc.ID) + assert.True(t, doc.Timestamp.Equal(time.Date(2026, 9, 17, 13, 0, 0, 0, time.UTC))) + assert.Equal(t, "ams1", doc.Pop) + assert.Equal(t, Queries{}, doc.Queries) + + doc.Aggregate(EventStatistics{Queries: Queries{Total: 1, Blocked: 1}}) + doc.Aggregate(EventStatistics{Queries: Queries{Total: 1, DNSSEC: 1}}) + assert.Equal(t, Queries{Total: 2, Blocked: 1, DNSSEC: 1}, doc.Queries) + assert.Equal(t, "ams1:2026-09-17T13", doc.ID, "aggregation never moves the document") +} + +// specRef: proxy-statistics-behaviour.md #Y1 #Y6 +func TestServiceStatistics_SchemaCarriesNoIdentifier(t *testing.T) { + forbidden := map[string]bool{"profile_id": true, "device_id": true, "client_ip": true} + allowed := map[string]bool{"_id": true, "timestamp": true, "pop": true, "queries": true} + + typ := reflect.TypeOf(ServiceStatistics{}) + for i := 0; i < typ.NumField(); i++ { + tag := strings.Split(typ.Field(i).Tag.Get("bson"), ",")[0] + assert.False(t, forbidden[tag], "field %s must not be stored", tag) + assert.True(t, allowed[tag], "unexpected stored field %q; extend the spec first", tag) + } + assert.Equal(t, len(allowed), typ.NumField()) + + evt := reflect.TypeOf(EventStatistics{}) + assert.Equal(t, 1, evt.NumField(), "the per-query event carries counters only") + assert.Equal(t, "Queries", evt.Field(0).Name) +} diff --git a/proxy/model/statistics.go b/proxy/model/statistics.go deleted file mode 100644 index b529685d..00000000 --- a/proxy/model/statistics.go +++ /dev/null @@ -1,28 +0,0 @@ -package model - -import ( - "time" - - "go.mongodb.org/mongo-driver/bson/primitive" -) - -type Statistics struct { - ID primitive.ObjectID `json:"-" bson:"_id"` - Timestamp time.Time `json:"timestamp" bson:"timestamp"` - ProfileID string `json:"profile_id" bson:"profile_id"` - DeviceId string `json:"device_id" bson:"device_id"` - Queries Queries `json:"queries" bson:"queries"` -} - -func (s *Statistics) Aggregate(other *Statistics) { - s.Timestamp = other.Timestamp - s.Queries.Total += other.Queries.Total - s.Queries.Blocked += other.Queries.Blocked - s.Queries.DNSSEC += other.Queries.DNSSEC -} - -type Queries struct { - Total int `json:"total" bson:"total"` - Blocked int `json:"blocked" bson:"blocked"` - DNSSEC int `json:"dnssec" bson:"dnssec"` -} diff --git a/proxy/requestcontext/logging_gating_test.go b/proxy/requestcontext/logging_gating_test.go index 13168e6a..973865a2 100644 --- a/proxy/requestcontext/logging_gating_test.go +++ b/proxy/requestcontext/logging_gating_test.go @@ -10,6 +10,7 @@ import ( "github.com/AdguardTeam/dnsproxy/proxy" "github.com/ivpn/dns/libs/logging" + "github.com/ivpn/dns/proxy/model" "github.com/rs/zerolog" "github.com/rs/zerolog/log" ) @@ -34,11 +35,7 @@ func testLogger(enabled bool, logDomains bool) (logging.LoggerInterface, *bytes. func TestAddDomain_DomainLoggingEnabled(t *testing.T) { logger, buf := testLogger(true, true) rc := NewRequestContext(context.Background(), &proxy.Proxy{}, "pid", "did", - map[string]string{}, - map[string]string{"log_domains": "true", "enabled": "true"}, - map[string]string{}, - map[string]string{}, - map[string]string{}, + &model.ProfileSettings{Logs: map[string]string{"log_domains": "true", "enabled": "true"}}, logger, ) ev := rc.Logger.Info() @@ -82,11 +79,7 @@ func TestAddClientIP_ClientIPLoggingDisabled(t *testing.T) { func TestAddDomain_DomainLoggingDisabled(t *testing.T) { logger, buf := testLogger(true, false) rc := NewRequestContext(context.Background(), &proxy.Proxy{}, "pid", "did", - map[string]string{}, - map[string]string{"log_domains": "false", "enabled": "true"}, - map[string]string{}, - map[string]string{}, - map[string]string{}, + &model.ProfileSettings{Logs: map[string]string{"log_domains": "false", "enabled": "true"}}, logger, ) ev := rc.Logger.Info() @@ -100,11 +93,7 @@ func TestAddDomain_DomainLoggingDisabled(t *testing.T) { func TestMaybeDomain_DomainLoggingEnabled(t *testing.T) { logger, buf := testLogger(true, true) rc := NewRequestContext(context.Background(), &proxy.Proxy{}, "pid", "did", - map[string]string{}, - map[string]string{"log_domains": "true", "enabled": "true"}, - map[string]string{}, - map[string]string{}, - map[string]string{}, + &model.ProfileSettings{Logs: map[string]string{"log_domains": "true", "enabled": "true"}}, logger, ) ev := rc.Logger.Info() diff --git a/proxy/requestcontext/request_context.go b/proxy/requestcontext/request_context.go index 1d9a9b6e..81dbc114 100644 --- a/proxy/requestcontext/request_context.go +++ b/proxy/requestcontext/request_context.go @@ -12,34 +12,50 @@ import ( type RequestContext struct { // Ctx context.Context - ProfileId string `json:"profile_id"` - DeviceId string `json:"device_id"` - PrivacySettings map[string]string `json:"privacy_settings"` - LogsSettings map[string]string `json:"logs_settings"` - AdvancedSettings map[string]string `json:"advanced_settings"` - DNSSECSettings map[string]string `json:"dnssec_settings"` - RebindingProtectionSettings map[string]string `json:"rebinding_protection_settings"` - PartialFilteringResults []model.StageResult `json:"partial_filtering_results"` - FilterResult model.FilterResult `json:"filter_result"` - Logger logging.LoggerInterface `json:"-"` - LoggerConfig logging.LoggingConfig `json:"logger_config"` - StartTime time.Time `json:"-"` - UpstreamName string `json:"upstream_name"` + ProfileId string `json:"profile_id"` + DeviceId string `json:"device_id"` + PrivacySettings map[string]string `json:"privacy_settings"` + LogsSettings map[string]string `json:"logs_settings"` + AdvancedSettings map[string]string `json:"advanced_settings"` + DNSSECSettings map[string]string `json:"dnssec_settings"` + RebindingProtectionSettings map[string]string `json:"rebinding_protection_settings"` + StatisticsSettings map[string]string `json:"statistics_settings"` + // Per-profile filter inputs from the settings batch; the filter stages + // read these instead of the store. + Blocklists []string `json:"blocklists"` + BlockedServices []string `json:"blocked_services"` + CustomRules []map[string]string `json:"custom_rules"` + PartialFilteringResults []model.StageResult `json:"partial_filtering_results"` + FilterResult model.FilterResult `json:"filter_result"` + Logger logging.LoggerInterface `json:"-"` + LoggerConfig logging.LoggingConfig `json:"logger_config"` + StartTime time.Time `json:"-"` + UpstreamName string `json:"upstream_name"` // UpstreamErr is the resolve error captured from the vendor proxy (nil on // success). Consumed by query-log outcome classification; never serialized. UpstreamErr error `json:"-"` } -func NewRequestContext(ctx context.Context, p *proxy.Proxy, profileId string, deviceId string, privacySettings, logsSettings, dnssecSettings, rebindingProtectionSettings, advancedSettings map[string]string, logger logging.LoggerInterface) *RequestContext { +// NewRequestContext builds the per-request state from the profile's settings +// batch. Settings groups whose read failed are nil maps; callers have already +// applied their defaults. +func NewRequestContext(ctx context.Context, p *proxy.Proxy, profileId string, deviceId string, settings *model.ProfileSettings, logger logging.LoggerInterface) *RequestContext { + if settings == nil { + settings = &model.ProfileSettings{} + } return &RequestContext{ // Ctx: ctx, ProfileId: profileId, DeviceId: deviceId, - PrivacySettings: privacySettings, - LogsSettings: logsSettings, - DNSSECSettings: dnssecSettings, - RebindingProtectionSettings: rebindingProtectionSettings, - AdvancedSettings: advancedSettings, + PrivacySettings: settings.Privacy, + LogsSettings: settings.Logs, + DNSSECSettings: settings.DNSSEC, + RebindingProtectionSettings: settings.RebindingProtection, + AdvancedSettings: settings.Advanced, + StatisticsSettings: settings.Statistics, + Blocklists: settings.Blocklists, + BlockedServices: settings.Services, + CustomRules: settings.CustomRules, Logger: logger, LoggerConfig: logger.Config(), } diff --git a/proxy/requestcontext/settings_carry_test.go b/proxy/requestcontext/settings_carry_test.go new file mode 100644 index 00000000..1fadf8de --- /dev/null +++ b/proxy/requestcontext/settings_carry_test.go @@ -0,0 +1,58 @@ +package requestcontext + +import ( + "context" + "testing" + + "github.com/AdguardTeam/dnsproxy/proxy" + "github.com/ivpn/dns/libs/logging" + "github.com/ivpn/dns/proxy/model" + "github.com/stretchr/testify/assert" +) + +// NewRequestContext carries every per-profile filter input from the settings +// batch so the filter stages need no store reads of their own. +func TestNewRequestContext_CarriesBatchInputs(t *testing.T) { + logger := logging.NewDefaultFactory().ForRequest(logging.LoggingConfig{Enabled: true}) + rules := []map[string]string{{"value": "ads.example", "action": "block"}} + settings := &model.ProfileSettings{ + Privacy: map[string]string{"default_rule": "block"}, + Logs: map[string]string{"enabled": "true"}, + DNSSEC: map[string]string{"enabled": "true"}, + RebindingProtection: map[string]string{"enabled": "1"}, + Advanced: map[string]string{"recursor": "knot"}, + Statistics: map[string]string{"enabled": "false"}, + Blocklists: []string{"bl1", "bl2"}, + Services: []string{"google"}, + CustomRules: rules, + } + + rc := NewRequestContext(context.Background(), &proxy.Proxy{}, "pid", "did", settings, logger) + + assert.Equal(t, "pid", rc.ProfileId) + assert.Equal(t, "did", rc.DeviceId) + assert.Equal(t, settings.Privacy, rc.PrivacySettings) + assert.Equal(t, settings.Logs, rc.LogsSettings) + assert.Equal(t, settings.DNSSEC, rc.DNSSECSettings) + assert.Equal(t, settings.RebindingProtection, rc.RebindingProtectionSettings) + assert.Equal(t, settings.Advanced, rc.AdvancedSettings) + assert.Equal(t, settings.Statistics, rc.StatisticsSettings) + assert.Equal(t, []string{"bl1", "bl2"}, rc.Blocklists) + assert.Equal(t, []string{"google"}, rc.BlockedServices) + assert.Equal(t, rules, rc.CustomRules) + assert.Equal(t, logger.Config(), rc.LoggerConfig) +} + +func TestNewRequestContext_NilSettings(t *testing.T) { + logger := logging.NewDefaultFactory().ForRequest(logging.LoggingConfig{Enabled: true}) + + var rc *RequestContext + assert.NotPanics(t, func() { + rc = NewRequestContext(context.Background(), nil, "pid", "", nil, logger) + }) + assert.Nil(t, rc.PrivacySettings) + assert.Nil(t, rc.Blocklists) + assert.Nil(t, rc.BlockedServices) + assert.Nil(t, rc.CustomRules) + assert.Nil(t, rc.StatisticsSettings) +} diff --git a/proxy/server/clientid.go b/proxy/server/clientid.go index 08ce02b4..2e996652 100644 --- a/proxy/server/clientid.go +++ b/proxy/server/clientid.go @@ -44,7 +44,7 @@ func isValidProfileID(s string) bool { return false } for _, r := range s { - if !((r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9')) { + if (r < 'a' || r > 'z') && (r < 'A' || r > 'Z') && (r < '0' || r > '9') { return false } } @@ -88,7 +88,7 @@ func clientIDFromClientServerName( } // Find profile ID (should be the last part that's alphanumeric) - var profileIDIndex int = -1 + profileIDIndex := -1 for i := len(parts) - 1; i >= 0; i-- { if isValidProfileID(parts[i]) { profileIDIndex = i diff --git a/proxy/server/device_identification_test.go b/proxy/server/device_identification_test.go index 7a16018e..781cef3a 100644 --- a/proxy/server/device_identification_test.go +++ b/proxy/server/device_identification_test.go @@ -262,3 +262,99 @@ func TestMultiServerNameIteration(t *testing.T) { }) } } + +// specRef: #Q11 — profile IDs are strictly alphanumeric with a minimum +// length, checked per rune. This predicate is the selection gate for +// untrusted SNI input, so both accept and reject sides are pinned here. +func TestIsValidProfileIDCharacterClasses(t *testing.T) { + // Default minimum length (10) applies; every reject case below that is + // long enough fails on characters, not length. + tests := []struct { + name string + id string + want bool + }{ + {"lowercase accepted", "abcdefghij", true}, + {"uppercase accepted", "ABCDEFGHIJ", true}, + {"digits accepted", "0123456789", true}, + {"mixed accepted", "3mdq3851b9", true}, + {"below min length rejected", "abcdefghi", false}, + {"empty rejected", "", false}, + {"underscore rejected", "abcdefghi_", false}, + {"dot rejected", "abcdefghi.", false}, + {"colon rejected", "abcdefghi:", false}, + {"space rejected", "abcdefghi ", false}, + {"hyphen rejected", "abcde-fghi", false}, + {"control char rejected", "abcdefghi\x00", false}, + {"multi-byte rune rejected", "abcdefghiä", false}, + {"emoji rejected", "abcdefghi🌐", false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := isValidProfileID(tt.id); got != tt.want { + t.Errorf("isValidProfileID(%q) = %v, want %v", tt.id, got, tt.want) + } + }) + } +} + +// specRef: #Q11 — an invalid character in an SNI part must change profile-ID +// selection: the failing part is never chosen, an earlier valid part wins, +// and a subdomain with no valid part is an error. IsImmediateSubdomain does +// not DNS-validate label content, so these bytes genuinely reach the gate. +func TestClientIDFromClientServerNameProfileIDSelection(t *testing.T) { + const host = "example.com" + + tests := []struct { + name string + cliSrvName string + wantClientID string + wantDeviceID string + wantErr bool + }{ + { + name: "valid last part selected", + cliSrvName: "mydevice-3mdq3851b9.example.com", + wantClientID: "3mdq3851b9", + wantDeviceID: "mydevice", + }, + { + name: "invalid char in last part falls back to earlier valid part", + cliSrvName: "3mdq3851b9xy-bad_part.example.com", + wantClientID: "3mdq3851b9xy", + wantDeviceID: "", + }, + { + name: "invalid char in only long-enough part is an error", + cliSrvName: "mydevice-3mdq3851b_9x.example.com", + wantErr: true, + }, + { + name: "colon in single part is an error", + cliSrvName: "3mdq3851:b9x.example.com", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + clientID, deviceId, err := clientIDFromClientServerName(host, tt.cliSrvName, false, proxy.ProtoTLS) + if tt.wantErr { + if err == nil { + t.Fatalf("expected error, got clientID=%q deviceId=%q", clientID, deviceId) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if clientID != tt.wantClientID { + t.Errorf("clientID = %q, want %q", clientID, tt.wantClientID) + } + if deviceId != tt.wantDeviceID { + t.Errorf("deviceId = %q, want %q", deviceId, tt.wantDeviceID) + } + }) + } +} diff --git a/proxy/server/dnscheck_handler_test.go b/proxy/server/dnscheck_handler_test.go index 6a0ac3de..1097fbf8 100644 --- a/proxy/server/dnscheck_handler_test.go +++ b/proxy/server/dnscheck_handler_test.go @@ -4,7 +4,6 @@ import ( "testing" "github.com/miekg/dns" - gocache "github.com/patrickmn/go-cache" "github.com/stretchr/testify/require" ) @@ -101,9 +100,7 @@ func TestBuildDNSCheckResponse(t *testing.T) { req.Extra = append(req.Extra, opt) upstream := c.setupUpstream(req) - server := &Server{ - ProfileSettingsCache: gocache.New(gocache.NoExpiration, 0), - } + server := &Server{} resp := server.buildDNSCheckResponse(req, upstream) c.assert(t, req, upstream, resp) }) diff --git a/proxy/server/metrics.go b/proxy/server/metrics.go index 850229ad..c4a626a6 100644 --- a/proxy/server/metrics.go +++ b/proxy/server/metrics.go @@ -7,20 +7,26 @@ import "time" // telemetry library. type Metrics interface { RecordQuery(proto string) - RecordProfileCacheLookup(hit bool) + // RecordProfileCacheLookup counts one settings lookup by outcome: + // hit, miss (fetched), stale (last-known-good served) or unavailable. + RecordProfileCacheLookup(status string) RecordQueryDuration(proto string, d time.Duration) RecordDomainFilterDuration(proto string, d time.Duration) RecordIPFilterDuration(proto string, d time.Duration) RecordUpstreamDuration(upstream string, d time.Duration) RecordBlocked(phase string) + // RecordFilterStageError counts one failed settings-store read. phase is + // "admission", "domain" or "ip"; stage names the reader that failed. + RecordFilterStageError(phase, stage string) } type noopMetrics struct{} func (noopMetrics) RecordQuery(string) {} -func (noopMetrics) RecordProfileCacheLookup(bool) {} +func (noopMetrics) RecordProfileCacheLookup(string) {} func (noopMetrics) RecordQueryDuration(string, time.Duration) {} func (noopMetrics) RecordDomainFilterDuration(string, time.Duration) {} func (noopMetrics) RecordIPFilterDuration(string, time.Duration) {} func (noopMetrics) RecordUpstreamDuration(string, time.Duration) {} func (noopMetrics) RecordBlocked(string) {} +func (noopMetrics) RecordFilterStageError(string, string) {} diff --git a/proxy/server/post_resolve_test.go b/proxy/server/post_resolve_test.go index bc2cad4a..41ac6b35 100644 --- a/proxy/server/post_resolve_test.go +++ b/proxy/server/post_resolve_test.go @@ -1,6 +1,8 @@ package server import ( + "context" + "errors" "net" "net/netip" "sync" @@ -74,11 +76,10 @@ func awaitWG(wg *sync.WaitGroup, timeout time.Duration) bool { } } -// setupStatsBackground mocks the async EmitStatistics path (cache lookup + Send) -// and wires wg.Done into the Send call so callers can synchronise. -func setupStatsBackground(cacheMock *mocks.Cache, statsCh *mocks.CollectorChannel, wg *sync.WaitGroup) { - cacheMock.On("GetProfileStatisticsSettings", mock.Anything, testPostResolveProfileID). - Return(map[string]string{"enabled": "false"}, nil).Maybe() +// setupStatsBackground mocks the async EmitServiceStatistics path (Send) and wires +// wg.Done into the Send call so callers can synchronise. Statistics settings +// travel on the request context, so no cache expectation is needed. +func setupStatsBackground(_ *mocks.Cache, statsCh *mocks.CollectorChannel, wg *sync.WaitGroup) { wg.Add(1) statsCh.On("Send", mock.Anything).Run(func(_ mock.Arguments) { wg.Done() }).Return(nil).Once() } @@ -133,14 +134,14 @@ func TestPostResolve_IPFilterDispatch(t *testing.T) { dctx := newPostResolveDNSContext("example.com", dns.TypeA) if tt.expectIPFilter { - ipFilter.On("Execute", reqCtx, dctx).Return(nil) + ipFilter.On("Execute", mock.Anything, reqCtx, dctx).Return(nil) } - s.postResolve(reqCtx, dctx) + s.postResolve(context.Background(), reqCtx, dctx) require.True(t, awaitWG(&wg, time.Second), "background goroutines did not finish") if tt.expectIPFilter { - ipFilter.AssertCalled(t, "Execute", reqCtx, dctx) + ipFilter.AssertCalled(t, "Execute", mock.Anything, reqCtx, dctx) } else { ipFilter.AssertNotCalled(t, "Execute") } @@ -205,16 +206,16 @@ func TestPostResolve_ResponseContent(t *testing.T) { dctx.Res.Answer = []dns.RR{rr} if tt.ipFilterBlocks { - ipFilter.On("Execute", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { - rCtx := args.Get(0).(*requestcontext.RequestContext) + ipFilter.On("Execute", mock.Anything, mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + rCtx := args.Get(1).(*requestcontext.RequestContext) rCtx.FilterResult.Status = model.StatusBlocked rCtx.FilterResult.Reasons = []string{"ip_blocked"} }).Return(nil) } else { - ipFilter.On("Execute", mock.Anything, mock.Anything).Return(nil) + ipFilter.On("Execute", mock.Anything, mock.Anything, mock.Anything).Return(nil) } - s.postResolve(reqCtx, dctx) + s.postResolve(context.Background(), reqCtx, dctx) require.True(t, awaitWG(&wg, time.Second)) if tt.wantIP == "NODATA" { @@ -245,9 +246,7 @@ func TestPostResolve_CacheHit_EmitsStats(t *testing.T) { reqCtx := newPostResolveReqCtx(model.StatusProcessed, nil) dctx := newPostResolveDNSContext("example.com", dns.TypeA) - ipFilter.On("Execute", mock.Anything, mock.Anything).Return(nil) - cacheMock.On("GetProfileStatisticsSettings", mock.Anything, testPostResolveProfileID). - Return(map[string]string{"enabled": "false"}, nil) + ipFilter.On("Execute", mock.Anything, mock.Anything, mock.Anything).Return(nil) received := make(chan model.EventStatistics, 1) statsCh.On("Send", mock.MatchedBy(func(data any) bool { @@ -258,14 +257,11 @@ func TestPostResolve_CacheHit_EmitsStats(t *testing.T) { return false })).Return(nil).Once() - s.postResolve(reqCtx, dctx) + s.postResolve(context.Background(), reqCtx, dctx) select { case evt := <-received: - assert.Equal(t, testPostResolveProfileID, evt.Statistics.ProfileID) - assert.Equal(t, testPostResolveDeviceID, evt.Statistics.DeviceId) - assert.Equal(t, 1, evt.Statistics.Queries.Total) - assert.Equal(t, 0, evt.Statistics.Queries.Blocked) + assert.Equal(t, model.Queries{Total: 1}, evt.Queries, "one processed query: total only") case <-time.After(time.Second): t.Fatal("timed out waiting for statistics event") } @@ -291,9 +287,7 @@ func TestPostResolve_CacheHit_EmitsQueryLog(t *testing.T) { reqCtx := newPostResolveReqCtx(model.StatusProcessed, logsSettings) dctx := newPostResolveDNSContext("logged.example.com", dns.TypeA) - ipFilter.On("Execute", mock.Anything, mock.Anything).Return(nil) - cacheMock.On("GetProfileStatisticsSettings", mock.Anything, testPostResolveProfileID). - Return(map[string]string{"enabled": "false"}, nil).Maybe() + ipFilter.On("Execute", mock.Anything, mock.Anything, mock.Anything).Return(nil) statsCh.On("Send", mock.Anything).Return(nil).Maybe() received := make(chan model.EventQueryLog, 1) @@ -305,7 +299,7 @@ func TestPostResolve_CacheHit_EmitsQueryLog(t *testing.T) { return false })).Return(nil).Once() - s.postResolve(reqCtx, dctx) + s.postResolve(context.Background(), reqCtx, dctx) select { case evt := <-received: @@ -320,3 +314,97 @@ func TestPostResolve_CacheHit_EmitsQueryLog(t *testing.T) { t.Fatal("timed out waiting for query log event") } } + +// specRef: proxy-filtering-behaviour.md #I4 +// specRef: proxy-filtering-behaviour.md #I1 +func TestPostResolve_Unavailable_SkipsIPFilterAnswersServfail(t *testing.T) { + ipFilter := mocks.NewFilter(t) + cacheMock := mocks.NewCache(t) + statsCh := mocks.NewCollectorChannel(t) + var wg sync.WaitGroup + setupStatsBackground(cacheMock, statsCh, &wg) + + s := newPostResolveServer(t, ipFilter, cacheMock, map[string]channel.CollectorChannel{ + model.TYPE_STATISTICS: statsCh, + }) + + // Domain phase ended in Unavailable: no upstream answer exists (Res nil), + // and the IP filter has no expectation — a call fails the test. + reqCtx := newPostResolveReqCtx(model.StatusUnavailable, nil) + dctx := newPostResolveDNSContext("example.com", dns.TypeA) + dctx.Res = nil + + s.postResolve(context.Background(), reqCtx, dctx) + require.True(t, awaitWG(&wg, time.Second), "background goroutines did not finish") + + ipFilter.AssertNotCalled(t, "Execute") + require.NotNil(t, dctx.Res, "the client must receive an answer, not silence") + assert.Equal(t, dns.RcodeServerFailure, dctx.Res.Rcode) + assert.True(t, dctx.Res.Response) + assert.Equal(t, dctx.Req.Id, dctx.Res.Id) + assert.Empty(t, dctx.Res.Answer) +} + +// specRef: proxy-filtering-behaviour.md #I4 +func TestPostResolve_IPFilterUnavailable_DiscardsAnswer(t *testing.T) { + ipFilter := mocks.NewFilter(t) + cacheMock := mocks.NewCache(t) + statsCh := mocks.NewCollectorChannel(t) + var wg sync.WaitGroup + setupStatsBackground(cacheMock, statsCh, &wg) + + s := newPostResolveServer(t, ipFilter, cacheMock, map[string]channel.CollectorChannel{ + model.TYPE_STATISTICS: statsCh, + }) + + reqCtx := newPostResolveReqCtx(model.StatusProcessed, nil) + dctx := newPostResolveDNSContext("example.com", dns.TypeA) + rr, err := dns.NewRR("example.com. 300 IN A 93.184.216.34") + require.NoError(t, err) + dctx.Res.Answer = []dns.RR{rr} + + ipFilter.On("Execute", mock.Anything, mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + rCtx := args.Get(1).(*requestcontext.RequestContext) + rCtx.FilterResult.Status = model.StatusUnavailable + }).Return(errors.New("dial tcp: i/o timeout")) + + s.postResolve(context.Background(), reqCtx, dctx) + require.True(t, awaitWG(&wg, time.Second)) + + require.NotNil(t, dctx.Res) + assert.Equal(t, dns.RcodeServerFailure, dctx.Res.Rcode, "an unverifiable answer is never handed to the client") + assert.Empty(t, dctx.Res.Answer) +} + +// specRef: query-log-outcomes-behaviour.md #O11 +func TestPostResolve_Unavailable_StatsCountTotalNotBlocked(t *testing.T) { + ipFilter := mocks.NewFilter(t) + cacheMock := mocks.NewCache(t) + statsCh := mocks.NewCollectorChannel(t) + + s := newPostResolveServer(t, ipFilter, cacheMock, map[string]channel.CollectorChannel{ + model.TYPE_STATISTICS: statsCh, + }) + + reqCtx := newPostResolveReqCtx(model.StatusUnavailable, nil) + dctx := newPostResolveDNSContext("example.com", dns.TypeA) + dctx.Res = nil + + received := make(chan model.EventStatistics, 1) + statsCh.On("Send", mock.MatchedBy(func(data any) bool { + if evt, ok := data.(model.EventStatistics); ok { + received <- evt + return true + } + return false + })).Return(nil).Once() + + s.postResolve(context.Background(), reqCtx, dctx) + + select { + case evt := <-received: + assert.Equal(t, model.Queries{Total: 1}, evt.Queries, "unavailable is not a block") + case <-time.After(time.Second): + t.Fatal("timed out waiting for statistics event") + } +} diff --git a/proxy/server/query_log_outcome_test.go b/proxy/server/query_log_outcome_test.go index c1aaa1c4..031f6385 100644 --- a/proxy/server/query_log_outcome_test.go +++ b/proxy/server/query_log_outcome_test.go @@ -51,6 +51,9 @@ func TestClassifyOutcome(t *testing.T) { {"empty NOERROR answer", "O2", model.StatusProcessed, nil, outcomeDctx(dns.RcodeSuccess, false), false, "nodata"}, {"nxdomain", "O3", model.StatusProcessed, nil, outcomeDctx(dns.RcodeNameError, false), false, "nxdomain"}, {"blocked wins over everything", "O4 OE1", model.StatusBlocked, errors.New("x"), outcomeDctx(dns.RcodeSuccess, true), false, "blocked"}, + {"filter unavailable", "O11", model.StatusUnavailable, nil, outcomeDctx(dns.RcodeServerFailure, false), false, OutcomeUnavailable}, + {"filter unavailable ignores upstream answer", "O11 OE5", model.StatusUnavailable, nil, outcomeDctx(dns.RcodeSuccess, true), false, OutcomeUnavailable}, + {"filter unavailable with nil Res", "O11 OE5", model.StatusUnavailable, nil, &proxy.DNSContext{Req: new(dns.Msg)}, false, OutcomeUnavailable}, {"dnssec servfail", "O5", model.StatusProcessed, nil, outcomeDctx(dns.RcodeServerFailure, false), true, "servfail_dnssec"}, {"upstream servfail", "O6", model.StatusProcessed, nil, outcomeDctx(dns.RcodeServerFailure, false), false, "servfail_upstream"}, {"deadline exceeded", "O7", model.StatusProcessed, context.DeadlineExceeded, outcomeDctx(dns.RcodeServerFailure, false), false, "timeout"}, diff --git a/proxy/server/query_logs.go b/proxy/server/query_logs.go index 5875716b..83401407 100644 --- a/proxy/server/query_logs.go +++ b/proxy/server/query_logs.go @@ -18,19 +18,20 @@ import ( // Resolution-outcome tokens stored in QueryLog.Outcome. // Decision table: docs/specs/query-log-outcomes-behaviour.md (rows O1-O10). const ( - OutcomeResolved = "resolved" // O1: NOERROR with answer records - OutcomeNoData = "nodata" // O2: NOERROR, empty answer - OutcomeNXDomain = "nxdomain" // O3 - OutcomeBlocked = "blocked" // O4 - OutcomeServfailDNSSEC = "servfail_dnssec" // O5 - OutcomeServfailUpstrm = "servfail_upstream" // O6 - OutcomeTimeout = "timeout" // O7 - OutcomeNetworkError = "network_error" // O8 - OutcomeRefused = "refused" // O9 + OutcomeResolved = "resolved" // O1: NOERROR with answer records + OutcomeNoData = "nodata" // O2: NOERROR, empty answer + OutcomeNXDomain = "nxdomain" // O3 + OutcomeBlocked = "blocked" // O4 + OutcomeServfailDNSSEC = "servfail_dnssec" // O5 + OutcomeServfailUpstrm = "servfail_upstream" // O6 + OutcomeTimeout = "timeout" // O7 + OutcomeNetworkError = "network_error" // O8 + OutcomeRefused = "refused" // O9 + OutcomeUnavailable = "filter_unavailable" // O11: settings store failed, SERVFAIL synthesized ) // classifyOutcome maps a completed request to a resolution-outcome token. -// Precedence (spec rows O1-O10): blocked first, then transport errors captured +// Precedence (spec rows O1-O11): blocked, then unavailable, then transport errors captured // from the vendor resolve call, then rcode-based outcomes, then answer content. // Returns "" (unknown) only for the defensive nil-response-without-error case. func classifyOutcome(reqCtx *requestcontext.RequestContext, dctx *proxy.DNSContext, dnssecFailed bool) string { @@ -38,6 +39,11 @@ func classifyOutcome(reqCtx *requestcontext.RequestContext, dctx *proxy.DNSConte if reqCtx.FilterResult.Status == model.StatusBlocked { return OutcomeBlocked } + // O11 — a settings-store failure withheld the answer; the SERVFAIL is ours, + // not the recursor's, so it must not read as an upstream failure. + if reqCtx.FilterResult.Status == model.StatusUnavailable { + return OutcomeUnavailable + } // O7 / O8 — the vendor resolve call failed; the client-visible SERVFAIL was // synthesized locally, so the transport error is the truthful outcome. diff --git a/proxy/server/ratelimit_response_test.go b/proxy/server/ratelimit_response_test.go index ef3214b7..2d23a42d 100644 --- a/proxy/server/ratelimit_response_test.go +++ b/proxy/server/ratelimit_response_test.go @@ -2,7 +2,7 @@ package server import ( "context" - "errors" + "fmt" "net/http" "net/netip" "net/url" @@ -11,12 +11,13 @@ import ( "github.com/AdguardTeam/dnsproxy/proxy" "github.com/ivpn/dns/libs/logging" + "github.com/ivpn/dns/proxy/cache" "github.com/ivpn/dns/proxy/config" "github.com/ivpn/dns/proxy/internal/ratelimit" + "github.com/ivpn/dns/proxy/internal/settingscache" "github.com/ivpn/dns/proxy/mocks" "github.com/ivpn/dns/proxy/model" "github.com/miekg/dns" - gocache "github.com/patrickmn/go-cache" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" @@ -140,7 +141,7 @@ func newProfileRateLimitServer(c *mocks.Cache, profileResponse string) *Server { }, }, Cache: c, - ProfileSettingsCache: gocache.New(time.Minute, time.Minute), + ProfileSettingsCache: mustSettingsCache(time.Minute), LoggerFactory: logging.NewDefaultFactory(), RateLimiter: ratelimit.New(ratelimit.Config{ PerProfileEnabled: true, @@ -151,6 +152,15 @@ func newProfileRateLimitServer(c *mocks.Cache, profileResponse string) *Server { } } +// mustSettingsCache builds a small settings cache for server fixtures. +func mustSettingsCache(ttl time.Duration) *settingscache.Cache { + c, err := settingscache.New(ttl, 64) + if err != nil { + panic(err) + } + return c +} + // newDoHDNSContext carries profileID via the DoH path, the simplest route // through clientIDFromDNSContext in tests. func newDoHDNSContext(profileID string) *proxy.DNSContext { @@ -168,7 +178,7 @@ func newDoHDNSContext(profileID string) *proxy.DNSContext { func TestPrepareRequest_UnknownProfileNeverProfileRateLimited(t *testing.T) { c := mocks.NewCache(t) c.EXPECT().GetProfileSettingsBatch(mock.Anything, "unknownprofile1"). - Return(&model.ProfileSettings{PrivacyErr: errors.New("no [privacy] settings found for profile")}, nil) + Return(&model.ProfileSettings{PrivacyErr: fmt.Errorf("%w: [privacy]", cache.ErrSettingsNotFound)}, nil) s := newProfileRateLimitServer(c, config.RateLimitResponseRefuse) // Far past the burst of 1: every call must fail on existence, and the @@ -184,14 +194,15 @@ func TestPrepareRequest_UnknownProfileNeverProfileRateLimited(t *testing.T) { // seedCachedProfile puts a minimal existing profile into the settings cache so // prepareRequest reaches the per-profile rate-limit layer without Redis. func seedCachedProfile(s *Server, profileID string) { - fetchErr := errors.New("settings unavailable") - s.ProfileSettingsCache.Set(profileID, &model.ProfileSettings{ + // Absent settings groups (defaults apply), not store failures. + absent := fmt.Errorf("%w: [seed]", cache.ErrSettingsNotFound) + s.ProfileSettingsCache.Put(profileID, &model.ProfileSettings{ Privacy: map[string]string{}, - LogsErr: fetchErr, - DNSSECErr: fetchErr, - RebindingProtectionErr: fetchErr, - AdvancedErr: fetchErr, - }, gocache.DefaultExpiration) + LogsErr: absent, + DNSSECErr: absent, + RebindingProtectionErr: absent, + AdvancedErr: absent, + }) } // specRef: proxy-request-admission-behaviour.md #Q7 diff --git a/proxy/server/server.go b/proxy/server/server.go index 563f0464..0a643039 100644 --- a/proxy/server/server.go +++ b/proxy/server/server.go @@ -21,10 +21,10 @@ import ( "github.com/ivpn/dns/proxy/internal/dnssec" "github.com/ivpn/dns/proxy/internal/metrics" "github.com/ivpn/dns/proxy/internal/ratelimit" + "github.com/ivpn/dns/proxy/internal/settingscache" "github.com/ivpn/dns/proxy/model" "github.com/ivpn/dns/proxy/requestcontext" "github.com/miekg/dns" - gocache "github.com/patrickmn/go-cache" "github.com/prometheus/client_golang/prometheus" "github.com/rs/zerolog/log" ) @@ -43,7 +43,7 @@ type Server struct { DomainFilter filter.Filter IPFilter filter.Filter Cache cache.Cache - ProfileSettingsCache *gocache.Cache + ProfileSettingsCache *settingscache.Cache CollectorChannels map[string]channel.CollectorChannel LoggerFactory logging.FactoryInterface RateLimiter *ratelimit.RateLimiter @@ -57,6 +57,7 @@ var ( errProfileIdNotFound = errors.New("profile_id not found") errRateLimitedIP = errors.New("rate limited by IP") errRateLimitedProfile = errors.New("rate limited by profile") + errStoreProbePending = errors.New("settings store marked unavailable, probe not due") ) func NewServer(serverConfig *config.Config, collectorChannels map[string]channel.CollectorChannel) (*Server, error) { @@ -68,8 +69,18 @@ func NewServer(serverConfig *config.Config, collectorChannels map[string]channel // Initialize logging factory loggerFactory := logging.NewDefaultFactory() - // In-memory profile settings cache to avoid Redis round-trips for warm profiles. - profileSettingsCache := gocache.New(serverConfig.Server.ProfileSettingsCacheTTL, 2*serverConfig.Server.ProfileSettingsCacheTTL) + // In-process profile settings: fresh entries skip Redis, stale entries are + // last-known-good for when Redis is unreachable. + cacheMetrics := metrics.NewSettingsCacheMetrics(prometheus.DefaultRegisterer) + profileSettingsCache, err := settingscache.New( + serverConfig.Server.ProfileSettingsCacheTTL, + serverConfig.Server.ProfileSettingsCacheSize, + settingscache.WithEvictionHook(cacheMetrics.RecordEviction), + ) + if err != nil { + return nil, fmt.Errorf("profile settings cache: %w", err) + } + metrics.ObserveSettingsCache(prometheus.DefaultRegisterer, profileSettingsCache) rl := ratelimit.New(ratelimit.Config{ PerIPEnabled: serverConfig.RateLimit.PerIPEnabled, @@ -114,8 +125,12 @@ func NewServer(serverConfig *config.Config, collectorChannels map[string]channel } log.Info().Str("catalog", serverConfig.Services.CatalogPath).Str("geodb", serverConfig.Services.GeoIPASNDBPath).Msg("Services blocking enabled") - server.DomainFilter = filter.NewDomainFilter(dnsProxy, cache, servicesCatalog) - server.IPFilter = filter.NewIPFilter(dnsProxy, cache, servicesCatalog, lookup, serverConfig.Rebinding, serverConfig.Filtering) + domainFilter := filter.NewDomainFilter(dnsProxy, cache, servicesCatalog) + domainFilter.Metrics = server.Metrics + ipFilter := filter.NewIPFilter(dnsProxy, cache, servicesCatalog, lookup, serverConfig.Rebinding, serverConfig.Filtering) + ipFilter.Metrics = server.Metrics + server.DomainFilter = domainFilter + server.IPFilter = ipFilter server.Proxy = dnsProxy profileIDMinLength = serverConfig.ProfileIDMinLength @@ -143,11 +158,13 @@ func (s *Server) ServeDNS(ctx context.Context, p *proxy.Proxy, dctx *proxy.DNSCo } // postResolve runs IP filtering, emits query logs/statistics, and responds. -func (s *Server) postResolve(reqCtx *requestcontext.RequestContext, dctx *proxy.DNSContext) { - if reqCtx.FilterResult.Status != model.StatusBlocked { +func (s *Server) postResolve(ctx context.Context, reqCtx *requestcontext.RequestContext, dctx *proxy.DNSContext) { + // Only a Processed domain phase has an answer to inspect; Blocked and + // Unavailable results are final. + if reqCtx.FilterResult.Status == model.StatusProcessed { ipStart := time.Now() - if err := s.IPFilter.Execute(reqCtx, dctx); err != nil { - reqCtx.Logger.Err(err).Msg("IP Filtering error") + if err := s.IPFilter.Execute(ctx, reqCtx, dctx); err != nil { + reqCtx.Logger.Debug().Err(err).Msg("IP filtering unavailable") } s.Metrics.RecordIPFilterDuration(string(dctx.Proto), time.Since(ipStart)) if reqCtx.FilterResult.Status == model.StatusBlocked { @@ -159,7 +176,7 @@ func (s *Server) postResolve(reqCtx *requestcontext.RequestContext, dctx *proxy. s.Metrics.RecordQueryDuration(string(dctx.Proto), time.Since(reqCtx.StartTime)) } go s.EmitQueryLog(reqCtx, dctx) - go s.EmitStatistics(reqCtx, dctx) + go s.EmitServiceStatistics(reqCtx, dctx) } // prepareRequest runs everything that must precede filtering: rate limits, @@ -202,30 +219,9 @@ func (s *Server) prepareRequest(ctx context.Context, p *proxy.Proxy, dctx *proxy systemLogger.Warn().Err(errProfileIdNotProvided).Msg(errProfileIdNotProvided.Error()) return nil, nil, errProfileIdNotProvided } else { - // Try in-memory profile settings cache first. - var settings *model.ProfileSettings - if cached, ok := s.ProfileSettingsCache.Get(profileId); ok { - s.Metrics.RecordProfileCacheLookup(true) - settings = cached.(*model.ProfileSettings) - } else { - s.Metrics.RecordProfileCacheLookup(false) - // Cache miss — fetch from Redis pipeline. - var fetchErr error - settings, fetchErr = s.Cache.GetProfileSettingsBatch(ctx, profileId) - if fetchErr != nil { - systemLogger.Err(fetchErr).Msg("Failed to fetch profile settings batch") - return nil, nil, errProfileIdNotFound - } - // Cache only successful fetches (profile exists). - if settings.PrivacyErr == nil { - s.ProfileSettingsCache.Set(profileId, settings, gocache.DefaultExpiration) - } - } - - // Privacy settings are required — missing means profile doesn't exist. - if settings.PrivacyErr != nil { - systemLogger.Debug().Err(settings.PrivacyErr).Msg(errProfileIdNotFound.Error()) - return nil, nil, errProfileIdNotFound + settings, errResp, err := s.loadProfileSettings(ctx, dctx.Req, profileId, systemLogger) + if err != nil || errResp != nil { + return nil, errResp, err } // Layer 2: per-profile rate limit. Runs after the existence check so @@ -236,7 +232,6 @@ func (s *Server) prepareRequest(ctx context.Context, p *proxy.Proxy, dctx *proxy } return nil, nil, errRateLimitedProfile } - prvSettings := settings.Privacy // Logs settings: default to enabled if unavailable. logsSettings := settings.Logs @@ -272,28 +267,24 @@ func (s *Server) prepareRequest(ctx context.Context, p *proxy.Proxy, dctx *proxy LogClientIPs: logClientIPs, }) - // DNSSEC settings: default to enabled if unavailable. + // DNSSEC settings: default to enabled if absent. A hash that exists but + // does not parse is our data being wrong, not the client's: SERVFAIL (Q12). dnssecSettings := settings.DNSSEC var dnssecEnabled, sendDoBit = true, true if settings.DNSSECErr != nil { reqLogger.Debug().Msg("DNSSEC settings not found, using default values") } else { dnssecEnabled, err = strconv.ParseBool(dnssecSettings["enabled"]) - if err != nil { - reqLogger.Err(err).Msg(errProfileIdNotFound.Error()) - return nil, nil, errProfileIdNotFound + if err == nil { + sendDoBit, err = strconv.ParseBool(dnssecSettings["send_do_bit"]) } - sendDoBit, err = strconv.ParseBool(dnssecSettings["send_do_bit"]) if err != nil { - reqLogger.Err(err).Msg(errProfileIdNotFound.Error()) - return nil, nil, errProfileIdNotFound + s.Metrics.RecordFilterStageError(metrics.PhaseAdmission, metrics.StageProfileSettings) + reqLogger.Err(err).Msg("Malformed DNSSEC settings, answering SERVFAIL") + return nil, s.servFailResponse(dctx.Req), nil } } - // Rebinding protection (security): missing hash = empty map = opt-in OFF. - // Raw map is threaded through; the IP-phase filter reads the "enabled" key. - rebindingProtectionSettings := settings.RebindingProtection - // Advanced settings: default upstream if unavailable. advancedSettings := settings.Advanced upstreamName := s.Config.Upstream.Default @@ -317,7 +308,7 @@ func (s *Server) prepareRequest(ctx context.Context, p *proxy.Proxy, dctx *proxy dctx.CustomUpstreamConfig = upstreamConfig reqLogger.Trace().Str("upstream", upstreamName).Msg("Upstream set") - reqCtx = requestcontext.NewRequestContext(ctx, p, profileId, deviceId, prvSettings, logsSettings, dnssecSettings, rebindingProtectionSettings, advancedSettings, reqLogger) + reqCtx = requestcontext.NewRequestContext(ctx, p, profileId, deviceId, settings, reqLogger) reqCtx.StartTime = time.Now() reqCtx.UpstreamName = upstreamName @@ -327,6 +318,75 @@ func (s *Server) prepareRequest(ctx context.Context, p *proxy.Proxy, dctx *proxy return reqCtx, nil, nil } +// loadProfileSettings returns the settings to serve the query with, applying +// spec rows Q6 and Q12–Q14 of proxy-request-admission-behaviour.md. Exactly one +// of the results is set: settings to continue with, a SERVFAIL response, or +// errProfileIdNotFound meaning drop. +func (s *Server) loadProfileSettings(ctx context.Context, req *dns.Msg, profileId string, logger logging.LoggerInterface) (*model.ProfileSettings, *dns.Msg, error) { + cached, state := s.ProfileSettingsCache.Get(profileId) + if state == settingscache.Fresh { + s.Metrics.RecordProfileCacheLookup(metrics.CacheLookupHit) + return cached, nil, nil + } + + // While the store is known to be failing, only one probe per interval + // reaches it; everyone else is served from the cache or refused at once. + if !s.ProfileSettingsCache.FetchAllowed() { + return s.settingsUnavailable(req, cached, state, logger, errStoreProbePending) + } + + fetchCtx, cancel := context.WithTimeout(ctx, filter.StoreDeadline) + defer cancel() + fetched, fetchErr := s.Cache.GetProfileSettingsBatch(fetchCtx, profileId) + if fetchErr != nil { + // The outage is logged on its transitions only; per-query effects are + // visible through the cache and stage-error metrics. + if s.ProfileSettingsCache.StoreFailed() { + logger.Error().Err(fetchErr).Msg("Settings store unreachable, serving last-known-good settings where cached") + } + return s.settingsUnavailable(req, cached, state, logger, fetchErr) + } + if s.ProfileSettingsCache.StoreRecovered() { + logger.Info().Msg("Settings store reachable again") + } + + // Privacy settings are required. Only an empty hash means the profile does + // not exist; any other read error is a store failure. + if fetched.PrivacyErr != nil { + if errors.Is(fetched.PrivacyErr, cache.ErrSettingsNotFound) { + // The store answered: the profile is gone, and so is any stale copy. + s.ProfileSettingsCache.Evict(profileId) + s.Metrics.RecordProfileCacheLookup(metrics.CacheLookupMiss) + logger.Debug().Err(fetched.PrivacyErr).Msg(errProfileIdNotFound.Error()) + return nil, nil, errProfileIdNotFound + } + return s.settingsUnavailable(req, cached, state, logger, fetched.PrivacyErr) + } + // The remaining groups may legitimately be absent (defaults apply), but a + // read failure on any of them means the filter inputs are incomplete. + if err := fetched.StoreError(); err != nil { + return s.settingsUnavailable(req, cached, state, logger, err) + } + + s.ProfileSettingsCache.Put(profileId, fetched) + s.Metrics.RecordProfileCacheLookup(metrics.CacheLookupMiss) + return fetched, nil, nil +} + +// settingsUnavailable resolves a failed fetch: last-known-good settings when +// a stale entry exists (Q13), otherwise SERVFAIL (Q12). +func (s *Server) settingsUnavailable(req *dns.Msg, cached *model.ProfileSettings, state settingscache.State, logger logging.LoggerInterface, cause error) (*model.ProfileSettings, *dns.Msg, error) { + if state == settingscache.Stale { + s.Metrics.RecordProfileCacheLookup(metrics.CacheLookupStale) + logger.Debug().Err(cause).Msg("Serving last-known-good profile settings") + return cached, nil, nil + } + s.Metrics.RecordProfileCacheLookup(metrics.CacheLookupUnavailable) + s.Metrics.RecordFilterStageError(metrics.PhaseAdmission, metrics.StageProfileSettings) + logger.Debug().Err(cause).Msg("Profile settings unavailable, answering SERVFAIL") + return nil, s.servFailResponse(req), nil +} + // handleRequest runs domain filtering, resolves via the profile's upstream // when the query is not blocked, and finishes with postResolve for both cache // hits and misses. @@ -341,14 +401,16 @@ func (s *Server) handleRequest(ctx context.Context, dctx *proxy.DNSContext, reqC // perform filtering actions domainStart := time.Now() - if err := s.DomainFilter.Execute(reqCtx, dctx); err != nil { - reqLogger.Err(err).Msg("Filtering error") + if err := s.DomainFilter.Execute(ctx, reqCtx, dctx); err != nil { + // Per-stage failures are counted in proxy_dns_filter_stage_errors_total. + reqLogger.Debug().Err(err).Msg("Domain filtering unavailable") } s.Metrics.RecordDomainFilterDuration(string(dctx.Proto), time.Since(domainStart)) if reqCtx.FilterResult.Status == model.StatusBlocked { s.Metrics.RecordBlocked("domain") } + // Blocked and Unavailable both skip resolution; respond() synthesizes the answer. if reqCtx.FilterResult.Status == model.StatusProcessed { reqLogger.Trace().Msg("Triggering default resolver") upstreamStart := time.Now() @@ -359,10 +421,15 @@ func (s *Server) handleRequest(ctx context.Context, dctx *proxy.DNSContext, reqC s.Metrics.RecordUpstreamDuration(reqCtx.UpstreamName, time.Since(upstreamStart)) } - s.postResolve(reqCtx, dctx) + s.postResolve(ctx, reqCtx, dctx) } func (s *Server) respond(reqCtx *requestcontext.RequestContext, dctx *proxy.DNSContext) { + if reqCtx.FilterResult.Status == model.StatusUnavailable { + // An answer that could not be checked against the profile is withheld. + dctx.Res = s.servFailResponse(dctx.Req) + return + } if reqCtx.FilterResult.Status != model.StatusBlocked { return } @@ -500,6 +567,13 @@ func (s *Server) refusedResponse(req *dns.Msg) *dns.Msg { return resp } +// servFailResponse builds a minimal DNS SERVFAIL response for the given request. +func (s *Server) servFailResponse(req *dns.Msg) *dns.Msg { + resp := new(dns.Msg) + resp.SetRcode(req, dns.RcodeServerFailure) + return resp +} + // formErrResponse builds a minimal DNS FORMERR response for the given request. func (s *Server) formErrResponse(req *dns.Msg) *dns.Msg { resp := new(dns.Msg) diff --git a/proxy/server/service_statistics.go b/proxy/server/service_statistics.go new file mode 100644 index 00000000..689f4d44 --- /dev/null +++ b/proxy/server/service_statistics.go @@ -0,0 +1,27 @@ +package server + +import ( + "github.com/AdguardTeam/dnsproxy/proxy" + "github.com/getsentry/sentry-go" + "github.com/ivpn/dns/proxy/model" + "github.com/ivpn/dns/proxy/requestcontext" +) + +// EmitServiceStatistics hands one query's counters to the statistics collector. The +// event carries no profile, device or time: it is summed into a service-wide +// document stamped at flush, so nothing per-profile is ever stored. +func (s *Server) EmitServiceStatistics(reqCtx *requestcontext.RequestContext, dctx *proxy.DNSContext) { + defer sentry.Recover() + + event := model.EventStatistics{Queries: model.Queries{Total: 1}} + if reqCtx.FilterResult.Status == model.StatusBlocked { + event.Queries.Blocked = 1 + } + if dctx.Res != nil && dctx.Res.AuthenticatedData { + event.Queries.DNSSEC = 1 + } + + if err := s.CollectorChannels[model.TYPE_STATISTICS].Send(event); err != nil { + reqCtx.Logger.Err(err).Msg("Failed to send statistics event to channel") + } +} diff --git a/proxy/server/service_statistics_benchmark_test.go b/proxy/server/service_statistics_benchmark_test.go new file mode 100644 index 00000000..90fbff0b --- /dev/null +++ b/proxy/server/service_statistics_benchmark_test.go @@ -0,0 +1,32 @@ +package server + +import ( + "testing" + + "github.com/ivpn/dns/proxy/collector/channel" + "github.com/ivpn/dns/proxy/model" + "github.com/miekg/dns" +) + +// discardChannel accepts every event so the benchmark measures EmitServiceStatistics +// itself, not the collector. +type discardChannel struct{} + +func (discardChannel) Send(any) error { return nil } +func (discardChannel) Receive() (any, error) { return nil, nil } + +// BenchmarkEmitServiceStatistics is the per-query cost of the statistics path. +func BenchmarkEmitServiceStatistics(b *testing.B) { + s := &Server{CollectorChannels: map[string]channel.CollectorChannel{ + model.TYPE_STATISTICS: discardChannel{}, + }} + reqCtx := newPostResolveReqCtx(model.StatusBlocked, nil) + dctx := newPostResolveDNSContext("example.com", dns.TypeA) + dctx.Res.AuthenticatedData = true + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + s.EmitServiceStatistics(reqCtx, dctx) + } +} diff --git a/proxy/server/service_statistics_test.go b/proxy/server/service_statistics_test.go new file mode 100644 index 00000000..8067422a --- /dev/null +++ b/proxy/server/service_statistics_test.go @@ -0,0 +1,88 @@ +package server + +import ( + "testing" + + "github.com/ivpn/dns/proxy/collector/channel" + "github.com/ivpn/dns/proxy/mocks" + "github.com/ivpn/dns/proxy/model" + "github.com/miekg/dns" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +// specRef: proxy-statistics-behaviour.md #Y2 #Y3 #Y4 +func TestEmitServiceStatistics_CountersOnly(t *testing.T) { + tests := []struct { + name string + status model.Status + nilRes bool + adFlag bool + want model.Queries + }{ + { + name: "processed query without DNSSEC", + status: model.StatusProcessed, + want: model.Queries{Total: 1}, + }, + { + name: "blocked query counts as blocked", + status: model.StatusBlocked, + want: model.Queries{Total: 1, Blocked: 1}, + }, + { + name: "authenticated response counts as dnssec", + status: model.StatusProcessed, + adFlag: true, + want: model.Queries{Total: 1, DNSSEC: 1}, + }, + { + name: "unavailable result is neither blocked nor dnssec", + status: model.StatusUnavailable, + nilRes: true, + want: model.Queries{Total: 1}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + statsCh := mocks.NewCollectorChannel(t) + s := newPostResolveServer(t, mocks.NewFilter(t), mocks.NewCache(t), map[string]channel.CollectorChannel{ + model.TYPE_STATISTICS: statsCh, + }) + reqCtx := newPostResolveReqCtx(tt.status, nil) + dctx := newPostResolveDNSContext("example.com", dns.TypeA) + if tt.nilRes { + dctx.Res = nil + } else { + dctx.Res.AuthenticatedData = tt.adFlag + } + + var got model.EventStatistics + statsCh.On("Send", mock.MatchedBy(func(data any) bool { + evt, ok := data.(model.EventStatistics) + if ok { + got = evt + } + return ok + })).Return(nil).Once() + + s.EmitServiceStatistics(reqCtx, dctx) + + require.Equal(t, model.EventStatistics{Queries: tt.want}, got) + }) + } +} + +// specRef: proxy-statistics-behaviour.md #Y2 +func TestEmitServiceStatistics_SendErrorIsLoggedNotRaised(t *testing.T) { + statsCh := mocks.NewCollectorChannel(t) + statsCh.On("Send", mock.Anything).Return(assert.AnError).Once() + s := newPostResolveServer(t, mocks.NewFilter(t), mocks.NewCache(t), map[string]channel.CollectorChannel{ + model.TYPE_STATISTICS: statsCh, + }) + + assert.NotPanics(t, func() { + s.EmitServiceStatistics(newPostResolveReqCtx(model.StatusProcessed, nil), newPostResolveDNSContext("example.com", dns.TypeA)) + }) +} diff --git a/proxy/server/settings_stale_test.go b/proxy/server/settings_stale_test.go new file mode 100644 index 00000000..3a1e7a04 --- /dev/null +++ b/proxy/server/settings_stale_test.go @@ -0,0 +1,193 @@ +package server + +// Serve-stale behaviour of the profile settings cache (Q13, Q14) and the +// store breaker that gates fetches while the store is failing (Q12, Q13). +// Rows: docs/specs/proxy-request-admission-behaviour.md. + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + "github.com/ivpn/dns/libs/logging" + "github.com/ivpn/dns/proxy/cache" + "github.com/ivpn/dns/proxy/config" + "github.com/ivpn/dns/proxy/internal/ratelimit" + "github.com/ivpn/dns/proxy/internal/settingscache" + "github.com/ivpn/dns/proxy/mocks" + "github.com/ivpn/dns/proxy/model" + "github.com/miekg/dns" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +const staleTTL = 30 * time.Second + +// staleFixture is a Server whose settings cache runs on a controllable clock +// and whose rate limits are disabled, so the same profile can be queried +// repeatedly. +type staleFixture struct { + s *Server + cache *mocks.Cache + metrics *recordingMetrics + now time.Time +} + +func newStaleFixture(t *testing.T) *staleFixture { + t.Helper() + f := &staleFixture{cache: mocks.NewCache(t), metrics: &recordingMetrics{}, now: time.Unix(1_700_000_000, 0)} + sc, err := settingscache.New(staleTTL, 16, settingscache.WithClock(func() time.Time { return f.now })) + require.NoError(t, err) + f.s = &Server{ + Config: &config.Config{ + Server: &config.ServerConfig{}, + Upstream: &config.UpstreamConfig{Default: "default"}, + RateLimit: &config.RateLimitConfig{}, + }, + Cache: f.cache, + ProfileSettingsCache: sc, + LoggerFactory: logging.NewDefaultFactory(), + RateLimiter: ratelimit.New(ratelimit.Config{}, nil), + Metrics: f.metrics, + } + return f +} + +func (f *staleFixture) advance(d time.Duration) { f.now = f.now.Add(d) } + +func goodSettings(rule string) *model.ProfileSettings { + absent := fmt.Errorf("%w: [absent]", cache.ErrSettingsNotFound) + return &model.ProfileSettings{ + Privacy: map[string]string{"default_rule": rule}, + Blocklists: []string{"bl1"}, + LogsErr: absent, + DNSSECErr: absent, + RebindingProtectionErr: absent, + AdvancedErr: absent, + StatisticsErr: absent, + } +} + +var errStoreDown = errors.New("redis pipeline failed: dial tcp: connection refused") + +// specRef: proxy-request-admission-behaviour.md #Q13 #S3 #S9 +func TestPrepareRequest_StaleServedWhenStoreFails(t *testing.T) { + const profileID = "staleprofile001" + f := newStaleFixture(t) + f.cache.EXPECT().GetProfileSettingsBatch(mock.Anything, profileID).Return(goodSettings("block"), nil).Once() + f.cache.EXPECT().GetProfileSettingsBatch(mock.Anything, profileID).Return(nil, errStoreDown).Once() + + // Cold: fetched and cached. + reqCtx, errResp, err := f.s.prepareRequest(context.Background(), nil, newDoHDNSContext(profileID)) + require.NoError(t, err) + require.Nil(t, errResp) + require.Equal(t, "block", reqCtx.PrivacySettings["default_rule"]) + + // Past the TTL with the store down: last-known-good settings are used. + f.advance(staleTTL + time.Second) + reqCtx, errResp, err = f.s.prepareRequest(context.Background(), nil, newDoHDNSContext(profileID)) + require.NoError(t, err) + require.Nil(t, errResp, "stale settings must be served, not SERVFAIL") + require.NotNil(t, reqCtx) + require.Equal(t, "block", reqCtx.PrivacySettings["default_rule"]) + require.Equal(t, []string{"bl1"}, reqCtx.Blocklists) + require.Contains(t, f.metrics.lookups(), "stale") + require.Empty(t, f.metrics.pairs(), "serving stale is not an admission error") + + // Breaker open: no further fetch within the probe interval (the mock allows + // exactly two calls), still served stale. + _, errResp, err = f.s.prepareRequest(context.Background(), nil, newDoHDNSContext(profileID)) + require.NoError(t, err) + require.Nil(t, errResp) +} + +// specRef: proxy-request-admission-behaviour.md #Q12 #S4 #S9 +func TestPrepareRequest_BreakerOpenNoStaleEntry_Servfail(t *testing.T) { + const known, unknown = "knownprofile001", "coldprofile00001" + f := newStaleFixture(t) + f.cache.EXPECT().GetProfileSettingsBatch(mock.Anything, known).Return(nil, errStoreDown).Once() + + _, errResp, err := f.s.prepareRequest(context.Background(), nil, newDoHDNSContext(known)) + require.NoError(t, err) + require.NotNil(t, errResp) + require.Equal(t, dns.RcodeServerFailure, errResp.Rcode) + + // A second profile inside the probe interval is refused without a fetch. + _, errResp, err = f.s.prepareRequest(context.Background(), nil, newDoHDNSContext(unknown)) + require.NoError(t, err) + require.NotNil(t, errResp) + require.Equal(t, dns.RcodeServerFailure, errResp.Rcode) + require.Equal(t, [][2]string{{"admission", "profile_settings"}, {"admission", "profile_settings"}}, f.metrics.pairs()) + require.Contains(t, f.metrics.lookups(), "unavailable") +} + +// specRef: proxy-request-admission-behaviour.md #Q13 #S9 +func TestPrepareRequest_BreakerProbesOncePerInterval(t *testing.T) { + const profileID = "probeprofile0001" + f := newStaleFixture(t) + f.cache.EXPECT().GetProfileSettingsBatch(mock.Anything, profileID).Return(nil, errStoreDown).Once() + f.cache.EXPECT().GetProfileSettingsBatch(mock.Anything, profileID).Return(goodSettings("allow"), nil).Once() + + _, errResp, _ := f.s.prepareRequest(context.Background(), nil, newDoHDNSContext(profileID)) + require.Equal(t, dns.RcodeServerFailure, errResp.Rcode) + + // Within the interval: no fetch, still SERVFAIL. + f.advance(settingscache.DefaultProbeInterval / 2) + _, errResp, _ = f.s.prepareRequest(context.Background(), nil, newDoHDNSContext(profileID)) + require.Equal(t, dns.RcodeServerFailure, errResp.Rcode) + + // After the interval one probe goes through and succeeds. + f.advance(settingscache.DefaultProbeInterval) + reqCtx, errResp, err := f.s.prepareRequest(context.Background(), nil, newDoHDNSContext(profileID)) + require.NoError(t, err) + require.Nil(t, errResp) + require.Equal(t, "allow", reqCtx.PrivacySettings["default_rule"]) + + // Recovered: fresh hits need no fetch at all. + _, errResp, err = f.s.prepareRequest(context.Background(), nil, newDoHDNSContext(profileID)) + require.NoError(t, err) + require.Nil(t, errResp) +} + +// specRef: proxy-request-admission-behaviour.md #Q14 #S8 +func TestPrepareRequest_DeletedProfileEvictsStaleEntry(t *testing.T) { + const profileID = "deletedprofile01" + f := newStaleFixture(t) + f.cache.EXPECT().GetProfileSettingsBatch(mock.Anything, profileID).Return(goodSettings("allow"), nil).Once() + f.cache.EXPECT().GetProfileSettingsBatch(mock.Anything, profileID). + Return(&model.ProfileSettings{PrivacyErr: fmt.Errorf("%w: [privacy]", cache.ErrSettingsNotFound)}, nil).Twice() + + _, errResp, err := f.s.prepareRequest(context.Background(), nil, newDoHDNSContext(profileID)) + require.NoError(t, err) + require.Nil(t, errResp) + + // Profile deleted meanwhile; the next refresh reports not-found → Q6. + f.advance(staleTTL + time.Second) + _, errResp, err = f.s.prepareRequest(context.Background(), nil, newDoHDNSContext(profileID)) + require.ErrorIs(t, err, errProfileIdNotFound) + require.Nil(t, errResp) + + // The stale entry is gone: the next query is a plain miss that fetches again. + _, state := f.s.ProfileSettingsCache.Get(profileID) + require.Equal(t, settingscache.Miss, state) + _, _, err = f.s.prepareRequest(context.Background(), nil, newDoHDNSContext(profileID)) + require.ErrorIs(t, err, errProfileIdNotFound) + require.Empty(t, f.metrics.pairs()) +} + +// specRef: proxy-request-admission-behaviour.md #Q13 #S1 #S2 +func TestPrepareRequest_FreshEntryNeverFetches(t *testing.T) { + const profileID = "freshprofile0001" + f := newStaleFixture(t) + f.cache.EXPECT().GetProfileSettingsBatch(mock.Anything, profileID).Return(goodSettings("allow"), nil).Once() + + for range 3 { + _, errResp, err := f.s.prepareRequest(context.Background(), nil, newDoHDNSContext(profileID)) + require.NoError(t, err) + require.Nil(t, errResp) + f.advance(staleTTL / 4) + } + require.Equal(t, []string{"miss", "hit", "hit"}, f.metrics.lookups()) +} diff --git a/proxy/server/settings_unavailable_test.go b/proxy/server/settings_unavailable_test.go new file mode 100644 index 00000000..c028dc94 --- /dev/null +++ b/proxy/server/settings_unavailable_test.go @@ -0,0 +1,279 @@ +package server + +// Tests for the admission-time split between "profile does not exist" (drop, +// Q6) and "settings store unreachable" (SERVFAIL, Q12). +// Rows: docs/specs/proxy-request-admission-behaviour.md. + +import ( + "context" + "errors" + "fmt" + "sync" + "testing" + "time" + + "github.com/AdguardTeam/dnsproxy/proxy" + "github.com/ivpn/dns/proxy/cache" + "github.com/ivpn/dns/proxy/internal/settingscache" + "github.com/ivpn/dns/proxy/mocks" + "github.com/ivpn/dns/proxy/model" + "github.com/miekg/dns" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +// recordingMetrics implements Metrics and records stage-error pairs and +// settings-cache lookup outcomes. +type recordingMetrics struct { + mu sync.Mutex + stageErrors [][2]string + cacheLookups []string +} + +func (m *recordingMetrics) RecordQuery(string) {} +func (m *recordingMetrics) RecordProfileCacheLookup(status string) { + m.mu.Lock() + defer m.mu.Unlock() + m.cacheLookups = append(m.cacheLookups, status) +} +func (m *recordingMetrics) RecordQueryDuration(string, time.Duration) {} +func (m *recordingMetrics) RecordDomainFilterDuration(string, time.Duration) {} +func (m *recordingMetrics) RecordIPFilterDuration(string, time.Duration) {} +func (m *recordingMetrics) RecordUpstreamDuration(string, time.Duration) {} +func (m *recordingMetrics) RecordBlocked(string) {} +func (m *recordingMetrics) RecordFilterStageError(phase, stage string) { + m.mu.Lock() + defer m.mu.Unlock() + m.stageErrors = append(m.stageErrors, [2]string{phase, stage}) +} + +func (m *recordingMetrics) lookups() []string { + m.mu.Lock() + defer m.mu.Unlock() + return append([]string(nil), m.cacheLookups...) +} + +func (m *recordingMetrics) pairs() [][2]string { + m.mu.Lock() + defer m.mu.Unlock() + return append([][2]string(nil), m.stageErrors...) +} + +func newSettingsServer(c *mocks.Cache) (*Server, *recordingMetrics) { + s := newProfileRateLimitServer(c, "refuse") + m := &recordingMetrics{} + s.Metrics = m + return s, m +} + +// specRef: proxy-request-admission-behaviour.md #Q12 +func TestPrepareRequest_SettingsBatchError_Servfail(t *testing.T) { + const profileID = "storeerrprofile1" + c := mocks.NewCache(t) + c.EXPECT().GetProfileSettingsBatch(mock.Anything, profileID). + Return(nil, errors.New("redis pipeline failed: dial tcp 10.0.0.6:6379: i/o timeout")) + s, m := newSettingsServer(c) + + reqCtx, errResp, err := s.prepareRequest(context.Background(), nil, newDoHDNSContext(profileID)) + + require.NoError(t, err, "an infrastructure error must not be a drop") + require.Nil(t, reqCtx) + require.NotNil(t, errResp, "the client must receive an answer") + assert.Equal(t, dns.RcodeServerFailure, errResp.Rcode) + assert.True(t, errResp.Response, "QR bit set") + assert.Equal(t, [][2]string{{"admission", "profile_settings"}}, m.pairs()) +} + +// specRef: proxy-request-admission-behaviour.md #Q12 +func TestPrepareRequest_PrivacyInfraError_Servfail(t *testing.T) { + const profileID = "storeerrprofile2" + c := mocks.NewCache(t) + c.EXPECT().GetProfileSettingsBatch(mock.Anything, profileID). + Return(&model.ProfileSettings{PrivacyErr: errors.New("i/o timeout")}, nil) + s, m := newSettingsServer(c) + + reqCtx, errResp, err := s.prepareRequest(context.Background(), nil, newDoHDNSContext(profileID)) + + require.NoError(t, err, "a per-key infrastructure error must not be conflated with not-found") + require.Nil(t, reqCtx) + require.NotNil(t, errResp) + assert.Equal(t, dns.RcodeServerFailure, errResp.Rcode) + assert.Equal(t, [][2]string{{"admission", "profile_settings"}}, m.pairs()) +} + +// specRef: proxy-request-admission-behaviour.md #Q6 +func TestPrepareRequest_PrivacyNotFound_Drops(t *testing.T) { + const profileID = "unknownprofile2" + c := mocks.NewCache(t) + c.EXPECT().GetProfileSettingsBatch(mock.Anything, profileID). + Return(&model.ProfileSettings{PrivacyErr: fmt.Errorf("%w: [privacy]", cache.ErrSettingsNotFound)}, nil) + s, m := newSettingsServer(c) + + reqCtx, errResp, err := s.prepareRequest(context.Background(), nil, newDoHDNSContext(profileID)) + + require.ErrorIs(t, err, errProfileIdNotFound) + require.Nil(t, reqCtx) + require.Nil(t, errResp, "a nonexistent profile is dropped without a response") + assert.Empty(t, m.pairs(), "not-found is not a store error") +} + +// specRef: proxy-request-admission-behaviour.md #Q12 +func TestPrepareRequest_SettingsBatchError_NotCached(t *testing.T) { + const profileID = "storeerrprofile3" + f := newStaleFixture(t) + f.cache.EXPECT().GetProfileSettingsBatch(mock.Anything, profileID). + Return(nil, errors.New("redis pipeline failed")).Twice() + + // Each probe (one per breaker interval) fetches again: nothing from a failed + // fetch is cached, so the second call still reaches the store and still fails. + for i := range 2 { + _, errResp, err := f.s.prepareRequest(context.Background(), nil, newDoHDNSContext(profileID)) + require.NoError(t, err, "call %d", i) + require.NotNil(t, errResp, "call %d", i) + assert.Equal(t, dns.RcodeServerFailure, errResp.Rcode, "call %d", i) + f.advance(settingscache.DefaultProbeInterval) + } +} + +// specRef: proxy-request-admission-behaviour.md #Q12 +func TestServeDNS_SettingsUnavailableAnswersServfail(t *testing.T) { + const profileID = "storeerrprofile4" + c := mocks.NewCache(t) + c.EXPECT().GetProfileSettingsBatch(mock.Anything, profileID). + Return(nil, errors.New("redis pipeline failed: connection refused")) + s, _ := newSettingsServer(c) + + dctx := newDoHDNSContext(profileID) + err := s.ServeDNS(context.Background(), nil, dctx) + + require.NoError(t, err, "SERVFAIL is a response, not a drop") + require.NotNil(t, dctx.Res) + assert.Equal(t, dns.RcodeServerFailure, dctx.Res.Rcode) + assert.Equal(t, dctx.Req.Id, dctx.Res.Id) +} + +// specRef: proxy-request-admission-behaviour.md #Q12 +func TestServFailResponse_Shape(t *testing.T) { + s := &Server{} + req := new(dns.Msg) + req.SetQuestion("example.com.", dns.TypeA) + req.Id = 0xBEEF + + resp := s.servFailResponse(req) + + require.NotNil(t, resp) + assert.Equal(t, dns.RcodeServerFailure, resp.Rcode) + assert.True(t, resp.Response) + assert.Equal(t, uint16(0xBEEF), resp.Id) + assert.Empty(t, resp.Answer) +} + +// specRef: proxy-request-admission-behaviour.md #Q12 +// specRef: proxy-filtering-behaviour.md #I6 +func TestPrepareRequest_FilterInputReadError_Servfail(t *testing.T) { + tests := []struct { + name string + settings *model.ProfileSettings + }{ + { + name: "blocklists list unreadable", + settings: &model.ProfileSettings{ + Privacy: map[string]string{"default_rule": "allow"}, + BlocklistsErr: errors.New("i/o timeout"), + }, + }, + { + name: "custom rules unreadable", + settings: &model.ProfileSettings{ + Privacy: map[string]string{"default_rule": "allow"}, + CustomRulesErr: errors.New("i/o timeout"), + }, + }, + { + name: "services list unreadable", + settings: &model.ProfileSettings{ + Privacy: map[string]string{"default_rule": "allow"}, + ServicesErr: errors.New("connection reset by peer"), + }, + }, + } + for i, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + profileID := fmt.Sprintf("inputerrprofile%d", i) + c := mocks.NewCache(t) + c.EXPECT().GetProfileSettingsBatch(mock.Anything, profileID).Return(tt.settings, nil) + s, m := newSettingsServer(c) + + reqCtx, errResp, err := s.prepareRequest(context.Background(), nil, newDoHDNSContext(profileID)) + + require.NoError(t, err, "incomplete filter inputs are a store failure, not a drop") + require.Nil(t, reqCtx) + require.NotNil(t, errResp) + assert.Equal(t, dns.RcodeServerFailure, errResp.Rcode) + assert.Equal(t, [][2]string{{"admission", "profile_settings"}}, m.pairs()) + }) + } +} + +// specRef: proxy-request-admission-behaviour.md #Q12 +// Absent optional groups are not store failures: defaults apply and the +// request proceeds with the batch's filter inputs on the request context. +func TestPrepareRequest_AbsentOptionalGroups_Proceeds(t *testing.T) { + const profileID = "absentgroupsprofile1" + absent := func(name string) error { return fmt.Errorf("%w: [%s]", cache.ErrSettingsNotFound, name) } + rules := []map[string]string{{"value": "ads.example", "action": "block", "syntax": "domain"}} + settings := &model.ProfileSettings{ + Privacy: map[string]string{"default_rule": "allow"}, + LogsErr: absent("logs"), + DNSSECErr: absent("security dnssec"), + AdvancedErr: absent("advanced"), + RebindingProtectionErr: absent("security rebinding_protection"), + StatisticsErr: absent("statistics"), + Blocklists: []string{"bl1", "bl2"}, + Services: []string{"google"}, + CustomRules: rules, + } + c := mocks.NewCache(t) + c.EXPECT().GetProfileSettingsBatch(mock.Anything, profileID).Return(settings, nil) + s, m := newSettingsServer(c) + s.Upstreams = map[string]*proxy.CustomUpstreamConfig{"default": {}} + + reqCtx, errResp, err := s.prepareRequest(context.Background(), nil, newDoHDNSContext(profileID)) + + require.NoError(t, err) + require.Nil(t, errResp) + require.NotNil(t, reqCtx) + assert.Empty(t, m.pairs(), "absent groups are not store errors") + assert.Equal(t, []string{"bl1", "bl2"}, reqCtx.Blocklists) + assert.Equal(t, []string{"google"}, reqCtx.BlockedServices) + assert.Equal(t, rules, reqCtx.CustomRules) + assert.Nil(t, reqCtx.StatisticsSettings) + assert.Equal(t, "default", reqCtx.UpstreamName) +} + +// A DNSSEC hash that exists but does not parse is malformed server-side data: +// answered SERVFAIL and counted, never dropped as a missing profile. +// specRef: proxy-request-admission-behaviour.md #Q12 +func TestPrepareRequest_MalformedDNSSECSettings_Servfail(t *testing.T) { + const profileID = "baddnssecprofile" + absent := fmt.Errorf("%w: [absent]", cache.ErrSettingsNotFound) + c := mocks.NewCache(t) + c.EXPECT().GetProfileSettingsBatch(mock.Anything, profileID).Return(&model.ProfileSettings{ + Privacy: map[string]string{}, + DNSSEC: map[string]string{"enabled": "yes please", "send_do_bit": "1"}, + LogsErr: absent, + RebindingProtectionErr: absent, + AdvancedErr: absent, + StatisticsErr: absent, + }, nil) + s, m := newSettingsServer(c) + + reqCtx, errResp, err := s.prepareRequest(context.Background(), nil, newDoHDNSContext(profileID)) + + require.NoError(t, err, "malformed settings are not a drop") + require.Nil(t, reqCtx) + require.NotNil(t, errResp) + assert.Equal(t, dns.RcodeServerFailure, errResp.Rcode) + assert.Equal(t, [][2]string{{"admission", "profile_settings"}}, m.pairs()) +} diff --git a/proxy/server/statistics.go b/proxy/server/statistics.go deleted file mode 100644 index a0feff63..00000000 --- a/proxy/server/statistics.go +++ /dev/null @@ -1,58 +0,0 @@ -package server - -import ( - "context" - "strconv" - "time" - - "github.com/AdguardTeam/dnsproxy/proxy" - "github.com/getsentry/sentry-go" - "github.com/ivpn/dns/proxy/model" - "github.com/ivpn/dns/proxy/requestcontext" - "github.com/miekg/dns" -) - -func (s *Server) EmitStatistics(reqCtx *requestcontext.RequestContext, dctx *proxy.DNSContext) { - defer sentry.Recover() - - // Use the contextual logger from the request context - logger := reqCtx.Logger - - statsSettings, err := s.Cache.GetProfileStatisticsSettings(context.Background(), reqCtx.ProfileId) - if err != nil { - logger.Err(err).Msg("Error getting profile statistics settings") - } - statsEnabled, err := strconv.ParseBool(statsSettings["enabled"]) - if err != nil { - logger.Err(err).Msg("Error parsing profile logs settings") - } - if statsEnabled { - logger.Trace().Msg("Sending optional statistics") - // TODO: Emit optional statistics, not implemented yet - } - // emit query number statistics (obligatory) - logger.Trace().Str("protocol", string(dctx.Proto)).Str("qtype", dns.Type(dctx.Req.Question[0].Qtype).String()).Msg("Sending statistics event to channel") - stats := &model.Statistics{ - Timestamp: time.Now().UTC(), - ProfileID: reqCtx.ProfileId, - DeviceId: reqCtx.DeviceId, - Queries: model.Queries{ - Total: 1, - }, - } - if reqCtx.FilterResult.Status == model.StatusBlocked { - stats.Queries.Blocked = 1 - } - - if dctx.Res != nil && dctx.Res.AuthenticatedData { - stats.Queries.DNSSEC = 1 - } - - if err = s.CollectorChannels[model.TYPE_STATISTICS].Send( - model.EventStatistics{ - Statistics: stats, - }, - ); err != nil { - logger.Err(err).Msg("Failed to send statistics event to channel") - } -} diff --git a/proxy/server/strict_no_logging_test.go b/proxy/server/strict_no_logging_test.go index 5c6aebd6..12013a15 100644 --- a/proxy/server/strict_no_logging_test.go +++ b/proxy/server/strict_no_logging_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/ivpn/dns/libs/logging" + "github.com/ivpn/dns/proxy/model" "github.com/ivpn/dns/proxy/requestcontext" "github.com/rs/zerolog" "github.com/stretchr/testify/assert" @@ -68,11 +69,7 @@ func TestStrictNoLogging_RequestContext_LoggerIntegration(t *testing.T) { nil, "profile-no-logs", "", // deviceId - map[string]string{}, - map[string]string{"enabled": "false"}, - map[string]string{}, - map[string]string{}, - map[string]string{}, + &model.ProfileSettings{Logs: map[string]string{"enabled": "false"}}, disabledLogger, ) @@ -87,11 +84,7 @@ func TestStrictNoLogging_RequestContext_LoggerIntegration(t *testing.T) { nil, "profile-with-logs", "", // deviceId - map[string]string{}, - map[string]string{"enabled": "true"}, - map[string]string{}, - map[string]string{}, - map[string]string{}, + &model.ProfileSettings{Logs: map[string]string{"enabled": "true"}}, enabledLogger, ) diff --git a/tests/config/dnscheck.env b/tests/config/dnscheck.env index 8021d418..979301fb 100644 --- a/tests/config/dnscheck.env +++ b/tests/config/dnscheck.env @@ -2,7 +2,7 @@ DNS_AUTH_SERVER_DOMAIN="test.moddns.net" DNS_AUTH_SERVER_IP_ADDRESS="127.0.0.1" DNS_AUTH_SERVER_ASN="" -DNS_AUTH_SERVER_IP_RANGE="10.5." +DNS_AUTH_SERVER_IP_RANGE="10.5.0.0/16" # ## API CONFIG API_PORT=":3000" @@ -10,9 +10,8 @@ API_PORT=":3000" API_ALLOW_ORIGIN="*" # ## CACHE CONFIG -CACHE_TTL=1m +CACHE_TTL=15s CACHE_HMAC_KEY="test-hmac-secret-key" # ## GEO LOOKUP CONFIG -GEOIP_DB_FILE=/opt/dnscheck/GeoIPCity/GeoLite2-City.mmdb GEOIP_DB_ASN_FILE=/opt/dnscheck/GeoIP/GeoLite2-ASN.mmdb diff --git a/tests/config/proxy.env b/tests/config/proxy.env index 1034bc85..553ffc60 100644 --- a/tests/config/proxy.env +++ b/tests/config/proxy.env @@ -1,5 +1,6 @@ # ## SERVER CONFIG SERVER_NAME="moddns.dev" +POP_NAME="dev1" DNS_CHECK_DOMAIN="test.moddns.net" DNS_CHECK_PORT="53" diff --git a/tests/dns_tests/test_connection_status.py b/tests/dns_tests/test_connection_status.py index e6db177f..48ea9d89 100644 --- a/tests/dns_tests/test_connection_status.py +++ b/tests/dns_tests/test_connection_status.py @@ -1,572 +1,134 @@ -""" -Backend E2E tests for DNS Connection Status Check feature. +"""Backend E2E tests for the DNS connection-status check (dnscheck). + +Flow under test, end to end through public interfaces only: + +1. DoH query for ``