diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 850d855..4dd5ce6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,6 +35,17 @@ jobs: - name: Test run: go test ./... + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: "24" + + - name: Test reusable source UI + run: | + node --check ui/static/js/app.js + node --check ui/static/js/source-lists.js + node --test ui/static/js/source-lists.test.cjs + - name: Build run: | go build -trimpath \ diff --git a/README.md b/README.md index b56abc9..fc200ef 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,8 @@ It supports multiple endpoint providers in one deployment, currently: - Built-in local authentication with persistent server-side sessions - First-start admin bootstrap (interactive in UI or via Docker environment) - Cron scheduling with manual run support -- DNS preview before saving a job +- DNS preview before saving a job, including reusable sources +- Reusable source lists with nested includes and an editable RFC1918 preset - Hostname plus literal IPv4 and IPv4 CIDR inputs - External URL list inputs (HTTP/HTTPS), for example Cloudflare IP ranges - Run history and per-run details @@ -42,10 +43,12 @@ Common examples: ## How It Works 1. Add one or more endpoints in the UI. -2. Create a sync job with hostnames/IP inputs. +2. Create a sync job with reusable sources, inline hostnames/IP inputs, or both. 3. Assign a primary target list and optional additional targets. 4. Run manually or on schedule. -5. The service resolves DNS, computes diffs, updates provider lists, and stores run history. +5. Each run freshly expands nested sources, resolves DNS and external URLs, deduplicates/optimizes the combined entries, computes diffs, updates provider lists, and stores run history. + +Sources are flattened into IP/CIDR entries for every target provider; native remote list nesting is not used. ## Before You Start @@ -164,9 +167,9 @@ Example: 3. Test connection and save. 4. Create a new sync job. 5. Choose primary endpoint and primary target list. -6. Add hostnames, IPv4, CIDR, or external URL list entries (one per line). +6. Select **Include Reusable Sources** and/or add inline hostnames, IPv4, CIDR, or external URL list entries (one per line). 7. Optionally add additional endpoint/list targets. -8. Save and run the job. +8. Preview the combined inputs, then save and run the job. 9. Review logs and target list state. Example input: @@ -183,6 +186,16 @@ synthetics.grafana.net https://www.cloudflare.com/ips-v4 ``` +### Reusable Sources + +Open **Reusable Sources**, then choose **New Source**. Give the source a name and add hostnames, literal IPv4 addresses, IPv4 CIDRs, or external HTTP/HTTPS list URLs, one per line. A source can also include other sources. Sources have no target or schedule; jobs select them through **Include Reusable Sources**, with optional inline hostnames/entries. Job preview includes the selected sources and their nested includes. + +**RFC1918 Preset** adds an editable source named **RFC1918** containing `10.0.0.0/8`, `172.16.0.0/12`, and `192.168.0.0/16`. These are private IPv4 ranges only, not loopback, link-local, or IPv6 ranges. + +Cycles are prevented. A missing source reference fails closed rather than updating targets with an incomplete expansion. Deletion is blocked while a source is referenced by another source or a job. + +Source changes apply on the next manual or scheduled job run, not as an immediate push. Existing retention may keep removed entries until the configured retention expires; disable retention for immediate removal on the next run. + ## Operational Tips 1. Validate jobs with manual runs before enabling schedule. @@ -212,6 +225,18 @@ All `/api/*` endpoints require an authenticated session. | GET | /api/instances/{id}/target-lists | List provider target lists | | POST | /api/instances/test | Test endpoint connection | +### Reusable Source Lists + +| Method | Path | Description | +| -------- | ------ | ------------- | +| GET | /api/source-lists | List sources | +| POST | /api/source-lists | Create source | +| GET | /api/source-lists/{id} | Get source | +| PUT | /api/source-lists/{id} | Update source | +| DELETE | /api/source-lists/{id} | Delete unreferenced source | + +Create/update bodies use `name`, `hostnames` (newline-separated entries), and `included_list_ids` (a JSON number array, e.g. `[1, 2]`) for nested sources. Writes require an admin session; deleting a referenced source returns `409 Conflict`. + ### Jobs | Method | Path | Description | @@ -226,11 +251,13 @@ All `/api/*` endpoints require an authenticated session. | GET | /api/jobs/{id}/logs | Get job run history | | GET | /api/jobs/{id}/logs/{logID}/targets/{targetID} | Get exact target items sent during a run | +Job create/update and `POST /api/resolve` bodies accept `included_list_ids` as a JSON number array (e.g. `[1, 2]`), alongside optional inline `hostnames`. Preview expands the same source includes as a job run. + ### DNS And Health | Method | Path | Description | |--------|------|-------------| -| POST | /api/resolve | Preview DNS resolution | +| POST | /api/resolve | Preview inline inputs and reusable sources | | GET | /api/health | Health check | | GET | /api/dns-servers | List DNS servers | | POST | /api/dns-servers | Create DNS server | diff --git a/internal/store/source_errors_test.go b/internal/store/source_errors_test.go new file mode 100644 index 0000000..f37863b --- /dev/null +++ b/internal/store/source_errors_test.go @@ -0,0 +1,42 @@ +package store + +import ( + "errors" + "testing" +) + +func TestSourceValidationErrorsAreTyped(t *testing.T) { + s := sourceTestStore(t) + leaf := createTestSource(t, s, "leaf", "192.0.2.1") + parent := createTestSource(t, s, "parent", "", leaf.ID) + for _, tc := range []struct { + name string + fn func() error + }{ + {"empty content", func() error { return s.ValidateSourceEntries("", nil) }}, + {"missing reference", func() error { return s.ValidateSourceEntries("", []int64{99999}) }}, + {"missing name", func() error { + _, err := s.CreateSourceList(&SourceList{Hostnames: "192.0.2.1"}) + return err + }}, + {"wrapped cycle", func() error { + leaf.IncludedListIDs = []int64{parent.ID} + return s.UpdateSourceList(leaf) + }}, + } { + t.Run(tc.name, func(t *testing.T) { + var validationErr *SourceValidationError + if err := tc.fn(); !errors.As(err, &validationErr) { + t.Fatalf("error=%v, want SourceValidationError", err) + } + }) + } + if err := s.Close(); err != nil { + t.Fatal(err) + } + var validationErr *SourceValidationError + err := s.ValidateSourceEntries("192.0.2.1", nil) + if err == nil || errors.As(err, &validationErr) { + t.Fatalf("storage error=%v, must not be SourceValidationError", err) + } +} diff --git a/internal/store/source_lists.go b/internal/store/source_lists.go new file mode 100644 index 0000000..e639200 --- /dev/null +++ b/internal/store/source_lists.go @@ -0,0 +1,330 @@ +package store + +import ( + "database/sql" + "encoding/json" + "fmt" + "strings" + "time" +) + +// SourceList is a reusable ordered set of entries and nested source references. +type SourceList struct { + ID int64 `json:"id"` + Name string `json:"name"` + Hostnames string `json:"hostnames"` + IncludedListIDs []int64 `json:"included_list_ids"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` +} + +// SourceValidationError reports invalid source input, not a storage failure. +// It remains identifiable through wrapped nested-list errors with errors.As. +type SourceValidationError struct { + Message string +} + +func (e *SourceValidationError) Error() string { + return e.Message +} + +func (s *Store) migrateSourceLists() error { + if _, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS source_lists ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL, + hostnames TEXT NOT NULL, + included_list_ids TEXT NOT NULL DEFAULT '[]', + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + )`); err != nil { + return err + } + rows, err := s.db.Query(`PRAGMA table_info(sync_jobs)`) + if err != nil { + return err + } + found := false + for rows.Next() { + var cid, notNull, pk int + var name, kind string + var defaultValue sql.NullString + if err := rows.Scan(&cid, &name, &kind, ¬Null, &defaultValue, &pk); err != nil { + rows.Close() + return err + } + found = found || name == "included_list_ids" + } + err = rows.Err() + rows.Close() + if err != nil { + return err + } + if !found { + _, err = s.db.Exec(`ALTER TABLE sync_jobs ADD COLUMN included_list_ids TEXT NOT NULL DEFAULT '[]'`) + } + return err +} + +func normalizeSourceIDs(ids []int64) []int64 { + out := make([]int64, 0, len(ids)) + seen := make(map[int64]bool, len(ids)) + for _, id := range ids { + if !seen[id] { + seen[id] = true + out = append(out, id) + } + } + return out +} + +func decodeSourceIDs(raw string) ([]int64, error) { + var ids []int64 + if err := json.Unmarshal([]byte(raw), &ids); err != nil || strings.TrimSpace(raw) == "null" { + return nil, fmt.Errorf("invalid included_list_ids JSON array: %q", raw) + } + return normalizeSourceIDs(ids), nil +} + +type sourceQuerier interface { + Query(string, ...any) (*sql.Rows, error) +} + +func listSourceLists(q sourceQuerier) ([]SourceList, error) { + rows, err := q.Query(`SELECT id, name, hostnames, included_list_ids, created_at, updated_at + FROM source_lists ORDER BY name, id`) + if err != nil { + return nil, err + } + defer rows.Close() + out := make([]SourceList, 0) + for rows.Next() { + var source SourceList + var raw string + if err := rows.Scan(&source.ID, &source.Name, &source.Hostnames, &raw, &source.CreatedAt, &source.UpdatedAt); err != nil { + return nil, err + } + source.IncludedListIDs, err = decodeSourceIDs(raw) + if err != nil { + return nil, fmt.Errorf("source list %d: %w", source.ID, err) + } + out = append(out, source) + } + return out, rows.Err() +} + +func (s *Store) ListSourceLists() ([]SourceList, error) { + return listSourceLists(s.db) +} + +func (s *Store) GetSourceList(id int64) (*SourceList, error) { + var source SourceList + var raw string + err := s.db.QueryRow(`SELECT id, name, hostnames, included_list_ids, created_at, updated_at + FROM source_lists WHERE id = ?`, id).Scan(&source.ID, &source.Name, &source.Hostnames, + &raw, &source.CreatedAt, &source.UpdatedAt) + if err != nil { + return nil, err + } + source.IncludedListIDs, err = decodeSourceIDs(raw) + if err != nil { + return nil, fmt.Errorf("source list %d: %w", id, err) + } + return &source, nil +} + +// expandSources uses one snapshot, emitting inline entries first, then sources +// in depth-first include order. A shared descendant is emitted only once. +func expandSources(sources []SourceList, hostnames string, ids []int64) (string, error) { + graph := make(map[int64]SourceList, len(sources)) + for _, source := range sources { + graph[source.ID] = source + } + state := make(map[int64]uint8, len(sources)) + parts := make([]string, 0) + appendEntries := func(entries string) { + if entries = strings.TrimSpace(entries); entries != "" { + parts = append(parts, entries) + } + } + appendEntries(hostnames) + var visit func(int64) error + visit = func(id int64) error { + if state[id] == 1 { + return &SourceValidationError{Message: fmt.Sprintf("source list cycle detected at ID %d", id)} + } + if state[id] == 2 { + return nil + } + source, ok := graph[id] + if !ok { + return &SourceValidationError{Message: fmt.Sprintf("source list ID %d does not exist", id)} + } + state[id] = 1 + appendEntries(source.Hostnames) + for _, child := range source.IncludedListIDs { + if err := visit(child); err != nil { + return fmt.Errorf("source list %d (%s): %w", id, source.Name, err) + } + } + state[id] = 2 + return nil + } + for _, id := range ids { + if err := visit(id); err != nil { + return "", err + } + } + return strings.Join(parts, "\n"), nil +} + +func validateSourceContent(hostnames string, ids []int64) error { + if strings.TrimSpace(hostnames) == "" && len(ids) == 0 { + return &SourceValidationError{Message: "source entries require hostnames or included source lists"} + } + return nil +} + +func validateJobSourceIDs(q sourceQuerier, ids []int64) error { + // Preserve legacy callers that store empty inline entries without includes. + if len(ids) == 0 { + return nil + } + sources, err := listSourceLists(q) + if err != nil { + return err + } + _, err = expandSources(sources, "", ids) + return err +} + +func (s *Store) ExpandSourceEntries(hostnames string, ids []int64) (string, error) { + sources, err := s.ListSourceLists() + if err != nil { + return "", err + } + return expandSources(sources, hostnames, ids) +} + +func (s *Store) ValidateSourceEntries(hostnames string, ids []int64) error { + if err := validateSourceContent(hostnames, ids); err != nil { + return err + } + _, err := s.ExpandSourceEntries(hostnames, ids) + return err +} + +func (s *Store) CreateSourceList(source *SourceList) (int64, error) { + return s.writeSourceList(source, false) +} + +func (s *Store) UpdateSourceList(source *SourceList) error { + _, err := s.writeSourceList(source, true) + return err +} + +func (s *Store) writeSourceList(source *SourceList, update bool) (int64, error) { + s.sourceMu.Lock() + defer s.sourceMu.Unlock() + if source == nil || strings.TrimSpace(source.Name) == "" { + return 0, &SourceValidationError{Message: "source list name is required"} + } + value := *source + value.Name = strings.TrimSpace(value.Name) + value.IncludedListIDs = normalizeSourceIDs(value.IncludedListIDs) + if err := validateSourceContent(value.Hostnames, value.IncludedListIDs); err != nil { + return 0, err + } + tx, err := s.db.Begin() + if err != nil { + return 0, err + } + defer tx.Rollback() + value.UpdatedAt = time.Now().UTC().Format(time.RFC3339Nano) + raw, _ := json.Marshal(value.IncludedListIDs) + if update { + if err := tx.QueryRow(`SELECT created_at FROM source_lists WHERE id = ?`, value.ID).Scan(&value.CreatedAt); err != nil { + return 0, err + } + _, err = tx.Exec(`UPDATE source_lists SET name=?, hostnames=?, included_list_ids=?, updated_at=? WHERE id=?`, + value.Name, value.Hostnames, string(raw), value.UpdatedAt, value.ID) + } else { + value.CreatedAt = value.UpdatedAt + var result sql.Result + result, err = tx.Exec(`INSERT INTO source_lists (name, hostnames, included_list_ids, created_at, updated_at) + VALUES (?, ?, ?, ?, ?)`, value.Name, value.Hostnames, string(raw), value.CreatedAt, value.UpdatedAt) + if err == nil { + value.ID, err = result.LastInsertId() + } + } + if err != nil { + return 0, err + } + sources, err := listSourceLists(tx) + if err != nil { + return 0, err + } + ids := make([]int64, 0, len(sources)) + for _, item := range sources { + ids = append(ids, item.ID) + } + if _, err := expandSources(sources, "", ids); err != nil { + return 0, err + } + if err := tx.Commit(); err != nil { + return 0, err + } + *source = value + return value.ID, nil +} + +func (s *Store) DeleteSourceList(id int64) error { + s.sourceMu.Lock() + defer s.sourceMu.Unlock() + tx, err := s.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + sources, err := listSourceLists(tx) + if err != nil { + return err + } + for _, source := range sources { + for _, ref := range source.IncludedListIDs { + if ref == id { + return fmt.Errorf("source list %d is referenced by source list %d (%s)", id, source.ID, source.Name) + } + } + } + rows, err := tx.Query(`SELECT id, name, included_list_ids FROM sync_jobs ORDER BY id`) + if err != nil { + return err + } + defer rows.Close() + for rows.Next() { + var jobID int64 + var name, raw string + if err := rows.Scan(&jobID, &name, &raw); err != nil { + return err + } + ids, err := decodeSourceIDs(raw) + if err != nil { + return fmt.Errorf("job %d: %w", jobID, err) + } + for _, ref := range ids { + if ref == id { + return fmt.Errorf("source list %d is referenced by job %d (%s)", id, jobID, name) + } + } + } + if err := rows.Err(); err != nil { + return err + } + if err := rows.Close(); err != nil { + return err + } + if _, err := tx.Exec(`DELETE FROM source_lists WHERE id = ?`, id); err != nil { + return err + } + return tx.Commit() +} diff --git a/internal/store/source_lists_test.go b/internal/store/source_lists_test.go new file mode 100644 index 0000000..2e114b4 --- /dev/null +++ b/internal/store/source_lists_test.go @@ -0,0 +1,425 @@ +package store + +import ( + "database/sql" + "encoding/json" + "errors" + "fmt" + "path/filepath" + "reflect" + "strings" + "sync" + "testing" +) + +func sourceTestStore(t *testing.T) *Store { + t.Helper() + s, err := New(filepath.Join(t.TempDir(), "sources.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { s.Close() }) + return s +} + +func createTestSource(t *testing.T, s *Store, name, entries string, ids ...int64) *SourceList { + t.Helper() + source := &SourceList{Name: name, Hostnames: entries, IncludedListIDs: ids} + id, err := s.CreateSourceList(source) + if err != nil { + t.Fatal(err) + } + if id != source.ID { + t.Fatalf("returned ID %d != source ID %d", id, source.ID) + } + return source +} + +func sourceTestJob(t *testing.T, s *Store, ids ...int64) *SyncJob { + t.Helper() + controllerID, err := s.CreateController(&Controller{Name: "controller", URL: "https://example.com"}) + if err != nil { + t.Fatal(err) + } + return &SyncJob{Name: "job", ControllerID: controllerID, NetworkListID: "target", IncludedListIDs: ids} +} + +func TestSourceListsCRUDAndOrderedDAG(t *testing.T) { + s := sourceTestStore(t) + items, err := s.ListSourceLists() + if err != nil || items == nil || len(items) != 0 { + t.Fatalf("empty list = %v, %v", items, err) + } + leaf := createTestSource(t, s, " leaf ", "leaf.example\n192.0.2.0/24") + left := createTestSource(t, s, "left", "left.example", leaf.ID, leaf.ID) + right := createTestSource(t, s, "right", "right.example", leaf.ID) + root := createTestSource(t, s, "root", "", left.ID, right.ID, left.ID) + if leaf.Name != "leaf" || leaf.CreatedAt == "" || leaf.UpdatedAt == "" || leaf.IncludedListIDs == nil { + t.Fatalf("source normalization = %+v", leaf) + } + if !reflect.DeepEqual(root.IncludedListIDs, []int64{left.ID, right.ID}) { + t.Fatalf("normalized root refs = %v", root.IncludedListIDs) + } + got, err := s.ExpandSourceEntries(" inline.example \n", []int64{root.ID, leaf.ID, root.ID}) + want := "inline.example\nleft.example\nleaf.example\n192.0.2.0/24\nright.example" + if err != nil || got != want { + t.Fatalf("expansion = %q, %v; want %q", got, err, want) + } + if err := s.ValidateSourceEntries("", []int64{root.ID}); err != nil { + t.Fatal(err) + } + before := leaf.CreatedAt + leaf.Name, leaf.Hostnames = " renamed ", "new.example" + if err := s.UpdateSourceList(leaf); err != nil { + t.Fatal(err) + } + stored, err := s.GetSourceList(leaf.ID) + if err != nil || stored.Name != "renamed" || stored.CreatedAt != before || stored.UpdatedAt == before { + t.Fatalf("updated source = %+v, %v", stored, err) + } + got, err = s.ExpandSourceEntries("", []int64{root.ID}) + if err != nil || got != "left.example\nnew.example\nright.example" { + t.Fatalf("fresh expansion = %q, %v", got, err) + } + items, err = s.ListSourceLists() + if err != nil || len(items) != 4 || items[0].Name != "left" { + t.Fatalf("list = %+v, %v", items, err) + } + if err := s.DeleteSourceList(root.ID); err != nil { + t.Fatal(err) + } + if _, err := s.GetSourceList(root.ID); !errors.Is(err, sql.ErrNoRows) { + t.Fatalf("deleted source error = %v", err) + } + if err := s.UpdateSourceList(root); !errors.Is(err, sql.ErrNoRows) { + t.Fatalf("missing update error = %v", err) + } +} + +func TestSourceListsValidationAndRollback(t *testing.T) { + s := sourceTestStore(t) + for _, source := range []*SourceList{ + nil, {Name: " ", Hostnames: "example.com"}, {Name: "empty", Hostnames: " \n"}, + {Name: "missing", IncludedListIDs: []int64{99}}, + {Name: "negative", IncludedListIDs: []int64{-1}}, + {Name: "zero", IncludedListIDs: []int64{0}}, + {Name: "self", IncludedListIDs: []int64{1}}, + } { + if _, err := s.CreateSourceList(source); err == nil { + t.Fatalf("accepted invalid source %+v", source) + } + } + items, err := s.ListSourceLists() + if err != nil || len(items) != 0 { + t.Fatalf("failed creates persisted: %v, %v", items, err) + } + a := createTestSource(t, s, "A", "a.example") + b := createTestSource(t, s, "B", "", a.ID) + c := createTestSource(t, s, "C", "", b.ID) + for _, refs := range [][]int64{{a.ID}, {c.ID}, {999}} { + a.IncludedListIDs = refs + if err := s.UpdateSourceList(a); err == nil { + t.Fatalf("accepted invalid refs %v", refs) + } + stored, err := s.GetSourceList(a.ID) + if err != nil || len(stored.IncludedListIDs) != 0 { + t.Fatalf("failed update persisted: %+v, %v", stored, err) + } + } + if err := s.ValidateSourceEntries(" \n", nil); err == nil { + t.Fatal("accepted empty entries") + } + if err := s.ValidateSourceEntries("inline.example", []int64{999}); err == nil { + t.Fatal("inline content masked missing reference") + } + if err := s.ValidateSourceEntries("inline.example", nil); err != nil { + t.Fatal(err) + } + // Simulate corrupt persisted graphs: expansion must return no partial content. + for _, raw := range []string{fmt.Sprintf("[%d]", c.ID), "[999]", "invalid", "null", "{}"} { + if _, err := s.db.Exec(`UPDATE source_lists SET included_list_ids=? WHERE id=?`, raw, a.ID); err != nil { + t.Fatal(err) + } + got, err := s.ExpandSourceEntries("inline.example", []int64{c.ID}) + if err == nil || got != "" { + t.Fatalf("corrupt graph %q expanded to %q, %v", raw, got, err) + } + if err := s.ValidateSourceEntries("", []int64{c.ID}); err == nil { + t.Fatalf("validated corrupt graph %q", raw) + } + } +} + +func TestSourceListsJobReferencesAndDeleteGuards(t *testing.T) { + s := sourceTestStore(t) + leaf := createTestSource(t, s, "leaf", "leaf.example") + root := createTestSource(t, s, "root", "", leaf.ID) + if err := s.DeleteSourceList(leaf.ID); err == nil || !strings.Contains(err.Error(), "source list") || !strings.Contains(err.Error(), "root") { + t.Fatalf("source delete guard = %v", err) + } + job := sourceTestJob(t, s, root.ID, root.ID) + id, err := s.CreateJob(job) + if err != nil { + t.Fatal(err) + } + job.ID = id + stored, err := s.GetJob(id) + if err != nil || !reflect.DeepEqual(stored.IncludedListIDs, []int64{root.ID}) { + t.Fatalf("job refs = %+v, %v", stored, err) + } + jobs, err := s.ListJobs() + if err != nil || len(jobs) != 1 || !reflect.DeepEqual(jobs[0].IncludedListIDs, []int64{root.ID}) { + t.Fatalf("listed job refs = %+v, %v", jobs, err) + } + if err := s.DeleteSourceList(root.ID); err == nil || !strings.Contains(err.Error(), "job") || !strings.Contains(err.Error(), job.Name) { + t.Fatalf("job delete guard = %v", err) + } + job.IncludedListIDs = []int64{leaf.ID, leaf.ID} + if err := s.UpdateJob(job); err != nil { + t.Fatal(err) + } + stored, err = s.GetJob(id) + if err != nil || !reflect.DeepEqual(stored.IncludedListIDs, []int64{leaf.ID}) { + t.Fatalf("updated job refs = %+v, %v", stored, err) + } + if err := s.DeleteSourceList(root.ID); err != nil { + t.Fatal(err) + } + job.IncludedListIDs = nil + if err := s.UpdateJob(job); err != nil { + t.Fatalf("legacy empty-hostnames update: %v", err) + } + if err := s.DeleteSourceList(leaf.ID); err != nil { + t.Fatal(err) + } + if _, err := s.CreateJob(sourceTestJob(t, s)); err != nil { + t.Fatalf("legacy empty-hostnames create: %v", err) + } + job.IncludedListIDs = []int64{leaf.ID} + if _, err := s.CreateJob(job); err == nil { + t.Fatal("created job with missing source") + } + if err := s.UpdateJob(job); err == nil { + t.Fatal("updated job with missing source") + } + stored, err = s.GetJob(id) + if err != nil || len(stored.IncludedListIDs) != 0 { + t.Fatalf("invalid update persisted: %+v, %v", stored, err) + } + encoded, err := json.Marshal(stored) + if err != nil || !strings.Contains(string(encoded), `"included_list_ids":[]`) { + t.Fatalf("empty refs JSON = %s, %v", encoded, err) + } +} + +func TestSourceListsJobTargetsAtomic(t *testing.T) { + s := sourceTestStore(t) + a := createTestSource(t, s, "A", "a.example") + b := createTestSource(t, s, "B", "b.example") + job := sourceTestJob(t, s, a.ID) + duplicateTargets := []JobTarget{ + {ControllerID: job.ControllerID, NetworkListID: "duplicate"}, + {ControllerID: job.ControllerID, NetworkListID: "duplicate"}, + } + job.Targets = duplicateTargets + if _, err := s.CreateJob(job); err == nil { + t.Fatal("expected target uniqueness failure") + } + jobs, err := s.ListJobs() + if err != nil || len(jobs) != 0 { + t.Fatalf("failed create left job: %+v, %v", jobs, err) + } + job.Targets = nil + job.ID, err = s.CreateJob(job) + if err != nil { + t.Fatal(err) + } + job.Name, job.IncludedListIDs, job.Targets = "changed", []int64{b.ID}, duplicateTargets + if err := s.UpdateJob(job); err == nil { + t.Fatal("expected target uniqueness failure on update") + } + stored, err := s.GetJob(job.ID) + if err != nil || stored.Name != "job" || !reflect.DeepEqual(stored.IncludedListIDs, []int64{a.ID}) || + len(stored.Targets) != 1 || stored.Targets[0].NetworkListID != "target" { + t.Fatalf("failed update changed job/refs/targets: %+v, %v", stored, err) + } +} + +func TestSourceListsMigrationAndReopen(t *testing.T) { + path := filepath.Join(t.TempDir(), "legacy.db") + db, err := sql.Open("sqlite", path) + if err != nil { + t.Fatal(err) + } + _, err = db.Exec(`CREATE TABLE sync_jobs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL, + controller_id INTEGER NOT NULL, network_list_id TEXT NOT NULL, + hostnames TEXT NOT NULL, schedule TEXT NOT NULL DEFAULT '', enabled INTEGER NOT NULL DEFAULT 1, + last_run_at TEXT, last_result TEXT, created_at TEXT NOT NULL, updated_at TEXT NOT NULL + ); + INSERT INTO sync_jobs (name, controller_id, network_list_id, hostnames, created_at, updated_at) + VALUES ('legacy', 1, 'target', '', 'old', 'old')`) + db.Close() + if err != nil { + t.Fatal(err) + } + s, err := New(path) + if err != nil { + t.Fatal(err) + } + job, err := s.GetJob(1) + if err != nil || job.IncludedListIDs == nil || len(job.IncludedListIDs) != 0 { + t.Fatalf("migrated job = %+v, %v", job, err) + } + controllerID, err := s.CreateController(&Controller{Name: "legacy controller"}) + if err != nil { + t.Fatal(err) + } + leaf := createTestSource(t, s, "leaf", "leaf.example") + root := createTestSource(t, s, "root", "root.example", leaf.ID) + job.ControllerID, job.IncludedListIDs = controllerID, []int64{root.ID} + if err := s.UpdateJob(job); err != nil { + t.Fatal(err) + } + if err := s.Close(); err != nil { + t.Fatal(err) + } + for i := 0; i < 2; i++ { + s, err = New(path) + if err != nil { + t.Fatal(err) + } + got, err := s.ExpandSourceEntries("", []int64{root.ID}) + if err != nil || got != "root.example\nleaf.example" { + t.Fatalf("reopened expansion = %q, %v", got, err) + } + job, err = s.GetJob(1) + if err != nil || !reflect.DeepEqual(job.IncludedListIDs, []int64{root.ID}) { + t.Fatalf("reopened job = %+v, %v", job, err) + } + s.Close() + } +} + +func TestSourceListsConcurrentEdits(t *testing.T) { + s := sourceTestStore(t) + for i := 0; i < 10; i++ { + a := createTestSource(t, s, fmt.Sprintf("A%d", i), "a.example") + b := createTestSource(t, s, fmt.Sprintf("B%d", i), "b.example") + a.IncludedListIDs, b.IncludedListIDs = []int64{b.ID}, []int64{a.ID} + start := make(chan struct{}) + results := make(chan error, 2) + for _, source := range []*SourceList{a, b} { + go func(source *SourceList) { <-start; results <- s.UpdateSourceList(source) }(source) + } + close(start) + first, second := <-results, <-results + if (first == nil) == (second == nil) { + t.Fatalf("concurrent cycle edits = %v, %v; want exactly one success", first, second) + } + if _, err := s.ExpandSourceEntries("", []int64{a.ID, b.ID}); err != nil { + t.Fatal(err) + } + } + for i := 0; i < 10; i++ { + leaf := createTestSource(t, s, fmt.Sprintf("leaf%d", i), "leaf.example") + job := sourceTestJob(t, s, leaf.ID) + var wg sync.WaitGroup + wg.Add(2) + var jobErr, deleteErr error + go func() { defer wg.Done(); _, jobErr = s.CreateJob(job) }() + go func() { defer wg.Done(); deleteErr = s.DeleteSourceList(leaf.ID) }() + wg.Wait() + if (jobErr == nil) == (deleteErr == nil) { + t.Fatalf("concurrent job/delete = %v, %v; want exactly one success", jobErr, deleteErr) + } + } +} + +func TestSourceListsSharedDatabaseTransactions(t *testing.T) { + path := filepath.Join(t.TempDir(), "shared.db") + first, err := New(path) + if err != nil { + t.Fatal(err) + } + defer first.Close() + second, err := New(path) + if err != nil { + t.Fatal(err) + } + defer second.Close() + // Different Store mutexes cannot protect each other. SQLite transactions + // must still prevent both cyclic edits or a reference plus deletion committing. + for i := 0; i < 10; i++ { + a := createTestSource(t, first, fmt.Sprintf("shared A%d", i), "a.example") + b := createTestSource(t, second, fmt.Sprintf("shared B%d", i), "b.example") + a.IncludedListIDs, b.IncludedListIDs = []int64{b.ID}, []int64{a.ID} + start := make(chan struct{}) + results := make(chan error, 2) + go func() { <-start; results <- first.UpdateSourceList(a) }() + go func() { <-start; results <- second.UpdateSourceList(b) }() + close(start) + one, two := <-results, <-results + if one == nil && two == nil { + t.Fatal("both cyclic edits committed across Store instances") + } + if _, err := first.ExpandSourceEntries("", []int64{a.ID, b.ID}); err != nil { + t.Fatalf("persisted graph invalid: %v", err) + } + leaf := createTestSource(t, first, fmt.Sprintf("shared leaf%d", i), "leaf.example") + parent := &SourceList{Name: fmt.Sprintf("shared parent%d", i), IncludedListIDs: []int64{leaf.ID}} + start = make(chan struct{}) + go func() { <-start; _, err := first.CreateSourceList(parent); results <- err }() + go func() { <-start; results <- second.DeleteSourceList(leaf.ID) }() + close(start) + one, two = <-results, <-results + if one == nil && two == nil { + t.Fatal("reference and deletion both committed across Store instances") + } + if parent.ID != 0 { + if _, err := first.ExpandSourceEntries("", []int64{parent.ID}); err != nil { + t.Fatalf("persisted dangling source reference: %v", err) + } + } + job := sourceTestJob(t, first, b.ID) + start = make(chan struct{}) + go func() { <-start; _, err := first.CreateJob(job); results <- err }() + go func() { <-start; results <- second.DeleteSourceList(b.ID) }() + close(start) + one, two = <-results, <-results + if one == nil && two == nil { + t.Fatal("job reference and deletion both committed across Store instances") + } + jobs, err := first.ListJobs() + if err != nil { + t.Fatal(err) + } + for _, stored := range jobs { + if err := first.ValidateSourceEntries(stored.Hostnames, stored.IncludedListIDs); err != nil { + t.Fatalf("persisted dangling job reference: %v", err) + } + } + } +} + +func TestSourceListJSONContract(t *testing.T) { + s := sourceTestStore(t) + source := createTestSource(t, s, "JSON", "example.com") + raw, err := json.Marshal(source) + if err != nil { + t.Fatal(err) + } + var fields map[string]json.RawMessage + if err := json.Unmarshal(raw, &fields); err != nil { + t.Fatal(err) + } + for _, name := range []string{"id", "name", "hostnames", "included_list_ids", "created_at", "updated_at"} { + if _, ok := fields[name]; !ok { + t.Fatalf("missing JSON field %q in %s", name, raw) + } + } + if len(fields) != 6 || string(fields["included_list_ids"]) != "[]" { + t.Fatalf("source JSON contract = %s", raw) + } +} + diff --git a/internal/store/store.go b/internal/store/store.go index e7fc7e2..7e3355e 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "strings" + "sync" "time" _ "modernc.org/sqlite" @@ -33,6 +34,7 @@ type SyncJob struct { NetworkListID string `json:"target_list_id"` Targets []JobTarget `json:"targets,omitempty"` Hostnames string `json:"hostnames"` + IncludedListIDs []int64 `json:"included_list_ids"` Schedule string `json:"schedule"` ObservedIPTTLHours int `json:"observed_ip_ttl_hours"` CollapseCIDRs bool `json:"collapse_cidrs"` @@ -102,6 +104,9 @@ type AppUser struct { // Store provides SQLite-backed persistence for sync jobs and run logs. type Store struct { db *sql.DB + // Serialize graph mutations and job reference writes on this Store. Each + // operation also uses a transaction for snapshot validation and persistence. + sourceMu sync.Mutex } // New opens (or creates) the SQLite database and runs migrations. @@ -175,6 +180,9 @@ func (s *Store) migrate() error { if err != nil { return err } + if err := s.migrateSourceLists(); err != nil { + return err + } _, _ = s.db.Exec(`ALTER TABLE sync_jobs ADD COLUMN observed_ip_ttl_hours INTEGER NOT NULL DEFAULT 168`) _, _ = s.db.Exec(`ALTER TABLE sync_jobs ADD COLUMN collapse_cidrs INTEGER NOT NULL DEFAULT 0`) _, _ = s.db.Exec(`ALTER TABLE run_logs ADD COLUMN stats_json TEXT NOT NULL DEFAULT ''`) @@ -286,6 +294,9 @@ func (s *Store) migrate() error { ON CONFLICT(job_id, controller_id, network_list_id) DO NOTHING`, jobID, controllerID, networkListID, now, now) } + if err := rows.Err(); err != nil { + return err + } } return nil @@ -365,7 +376,7 @@ func (s *Store) DeleteController(id int64) error { func (s *Store) ListJobs() ([]SyncJob, error) { rows, err := s.db.Query(` SELECT j.id, j.name, j.controller_id, j.network_list_id, - j.hostnames, j.schedule, j.observed_ip_ttl_hours, j.collapse_cidrs, j.enabled, + j.hostnames, j.included_list_ids, j.schedule, j.observed_ip_ttl_hours, j.collapse_cidrs, j.enabled, j.last_run_at, j.last_result, j.created_at, j.updated_at, COALESCE(c.name, '') FROM sync_jobs j @@ -380,12 +391,17 @@ func (s *Store) ListJobs() ([]SyncJob, error) { for rows.Next() { var j SyncJob var enabled int + var includes string if err := rows.Scan(&j.ID, &j.Name, &j.ControllerID, &j.NetworkListID, - &j.Hostnames, &j.Schedule, &j.ObservedIPTTLHours, &j.CollapseCIDRs, &enabled, + &j.Hostnames, &includes, &j.Schedule, &j.ObservedIPTTLHours, &j.CollapseCIDRs, &enabled, &j.LastRunAt, &j.LastResult, &j.CreatedAt, &j.UpdatedAt, &j.ControllerName); err != nil { return nil, err } + j.IncludedListIDs, err = decodeSourceIDs(includes) + if err != nil { + return nil, fmt.Errorf("job %d: %w", j.ID, err) + } j.ObservedIPTTLHours = normalizeObservedIPTTLHours(j.ObservedIPTTLHours) j.Enabled = enabled != 0 targets, targetsErr := s.ListJobTargets(j.ID) @@ -405,21 +421,26 @@ func (s *Store) ListJobs() ([]SyncJob, error) { func (s *Store) GetJob(id int64) (*SyncJob, error) { var j SyncJob var enabled int + var includes string err := s.db.QueryRow(` SELECT j.id, j.name, j.controller_id, j.network_list_id, - j.hostnames, j.schedule, j.observed_ip_ttl_hours, j.collapse_cidrs, j.enabled, + j.hostnames, j.included_list_ids, j.schedule, j.observed_ip_ttl_hours, j.collapse_cidrs, j.enabled, j.last_run_at, j.last_result, j.created_at, j.updated_at, COALESCE(c.name, '') FROM sync_jobs j LEFT JOIN controllers c ON c.id = j.controller_id WHERE j.id = ?`, id).Scan( &j.ID, &j.Name, &j.ControllerID, &j.NetworkListID, - &j.Hostnames, &j.Schedule, &j.ObservedIPTTLHours, &j.CollapseCIDRs, &enabled, + &j.Hostnames, &includes, &j.Schedule, &j.ObservedIPTTLHours, &j.CollapseCIDRs, &enabled, &j.LastRunAt, &j.LastResult, &j.CreatedAt, &j.UpdatedAt, &j.ControllerName) if err != nil { return nil, err } + j.IncludedListIDs, err = decodeSourceIDs(includes) + if err != nil { + return nil, fmt.Errorf("job %d: %w", j.ID, err) + } j.ObservedIPTTLHours = normalizeObservedIPTTLHours(j.ObservedIPTTLHours) j.Enabled = enabled != 0 targets, targetsErr := s.ListJobTargets(j.ID) @@ -435,6 +456,8 @@ func (s *Store) GetJob(id int64) (*SyncJob, error) { } func (s *Store) CreateJob(j *SyncJob) (int64, error) { + s.sourceMu.Lock() + defer s.sourceMu.Unlock() now := time.Now().UTC().Format(time.RFC3339) j.ObservedIPTTLHours = normalizeObservedIPTTLHours(j.ObservedIPTTLHours) targets := normalizeTargets(j) @@ -442,12 +465,22 @@ func (s *Store) CreateJob(j *SyncJob) (int64, error) { return 0, fmt.Errorf("at least one target is required") } primary := targets[0] - result, err := s.db.Exec(` + tx, err := s.db.Begin() + if err != nil { + return 0, err + } + defer tx.Rollback() + ids := normalizeSourceIDs(j.IncludedListIDs) + if err := validateJobSourceIDs(tx, ids); err != nil { + return 0, err + } + includes, _ := json.Marshal(ids) + result, err := tx.Exec(` INSERT INTO sync_jobs (name, controller_id, network_list_id, - hostnames, schedule, observed_ip_ttl_hours, collapse_cidrs, enabled, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + hostnames, included_list_ids, schedule, observed_ip_ttl_hours, collapse_cidrs, enabled, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, j.Name, primary.ControllerID, primary.NetworkListID, - j.Hostnames, j.Schedule, j.ObservedIPTTLHours, boolToInt(j.CollapseCIDRs), boolToInt(j.Enabled), + j.Hostnames, string(includes), j.Schedule, j.ObservedIPTTLHours, boolToInt(j.CollapseCIDRs), boolToInt(j.Enabled), now, now) if err != nil { return 0, err @@ -456,13 +489,19 @@ func (s *Store) CreateJob(j *SyncJob) (int64, error) { if err != nil { return 0, err } - if err := s.SetJobTargets(jobID, targets); err != nil { + if err := setJobTargets(tx, jobID, targets); err != nil { + return 0, err + } + if err := tx.Commit(); err != nil { return 0, err } + j.IncludedListIDs = ids return jobID, nil } func (s *Store) UpdateJob(j *SyncJob) error { + s.sourceMu.Lock() + defer s.sourceMu.Unlock() now := time.Now().UTC().Format(time.RFC3339) j.ObservedIPTTLHours = normalizeObservedIPTTLHours(j.ObservedIPTTLHours) targets := normalizeTargets(j) @@ -470,20 +509,39 @@ func (s *Store) UpdateJob(j *SyncJob) error { return fmt.Errorf("at least one target is required") } primary := targets[0] - _, err := s.db.Exec(` + tx, err := s.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + ids := normalizeSourceIDs(j.IncludedListIDs) + if err := validateJobSourceIDs(tx, ids); err != nil { + return err + } + includes, _ := json.Marshal(ids) + _, err = tx.Exec(` UPDATE sync_jobs SET name=?, controller_id=?, network_list_id=?, - hostnames=?, schedule=?, observed_ip_ttl_hours=?, collapse_cidrs=?, enabled=?, updated_at=? + hostnames=?, included_list_ids=?, schedule=?, observed_ip_ttl_hours=?, collapse_cidrs=?, enabled=?, updated_at=? WHERE id=?`, j.Name, primary.ControllerID, primary.NetworkListID, - j.Hostnames, j.Schedule, j.ObservedIPTTLHours, boolToInt(j.CollapseCIDRs), + j.Hostnames, string(includes), j.Schedule, j.ObservedIPTTLHours, boolToInt(j.CollapseCIDRs), boolToInt(j.Enabled), now, j.ID) if err != nil { return err } - return s.SetJobTargets(j.ID, targets) + if err := setJobTargets(tx, j.ID, targets); err != nil { + return err + } + if err := tx.Commit(); err != nil { + return err + } + j.IncludedListIDs = ids + return nil } func (s *Store) DeleteJob(id int64) error { + s.sourceMu.Lock() + defer s.sourceMu.Unlock() _, err := s.db.Exec("DELETE FROM sync_jobs WHERE id = ?", id) return err } @@ -680,6 +738,8 @@ func (s *Store) ListJobTargets(jobID int64) ([]JobTarget, error) { } func (s *Store) SetJobTargets(jobID int64, targets []JobTarget) error { + s.sourceMu.Lock() + defer s.sourceMu.Unlock() if len(targets) == 0 { return fmt.Errorf("at least one target is required") } @@ -689,7 +749,13 @@ func (s *Store) SetJobTargets(jobID int64, targets []JobTarget) error { return err } defer tx.Rollback() + if err := setJobTargets(tx, jobID, targets); err != nil { + return err + } + return tx.Commit() +} +func setJobTargets(tx *sql.Tx, jobID int64, targets []JobTarget) error { if _, err := tx.Exec(`DELETE FROM sync_job_targets WHERE job_id = ?`, jobID); err != nil { return err } @@ -707,7 +773,7 @@ func (s *Store) SetJobTargets(jobID int64, targets []JobTarget) error { } } - return tx.Commit() + return nil } // ---------- DNSServer CRUD ---------- diff --git a/internal/syncer/source_lists_test.go b/internal/syncer/source_lists_test.go new file mode 100644 index 0000000..3563cd7 --- /dev/null +++ b/internal/syncer/source_lists_test.go @@ -0,0 +1,341 @@ +package syncer + +import ( + "database/sql" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "path/filepath" + "reflect" + "strings" + "sync" + "testing" + + "github.com/mstrhakr/network-list-sync/internal/clients" + "github.com/mstrhakr/network-list-sync/internal/store" +) + +type sourceSyncFixture struct { + db *store.Store + dbPath string + controllerID int64 + mu sync.Mutex + requests int + payloads []json.RawMessage + list clients.NetworkList +} + +func newSourceSyncFixture(t *testing.T) *sourceSyncFixture { + t.Helper() + f := &sourceSyncFixture{ + dbPath: filepath.Join(t.TempDir(), "sync.db"), + list: clients.NetworkList{ID: "list-1", Name: "Allowlist", Type: "IPV4_ADDRESSES"}, + } + var err error + f.db, err = store.New(f.dbPath) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = f.db.Close() }) + // Remove seeded public resolvers. All inputs are literals, so DNS is never + // needed; even an accidental lookup can only reach loopback. + servers, err := f.db.ListDNSServers() + if err != nil { + t.Fatal(err) + } + for _, server := range servers { + if err := f.db.DeleteDNSServer(server.ID); err != nil { + t.Fatal(err) + } + } + if _, err := f.db.CreateDNSServer(&store.DNSServer{ + Name: "Unused loopback resolver", Address: "127.0.0.1:1", Enabled: true, + }); err != nil { + t.Fatal(err) + } + + const siteID = "11111111-1111-1111-1111-111111111111" + const apiBase = "/proxy/network/integration/v1" + listPath := apiBase + "/sites/" + siteID + "/traffic-matching-lists/list-1" + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + f.mu.Lock() + defer f.mu.Unlock() + f.requests++ + if r.Header.Get("X-API-Key") != "test-key" { + t.Errorf("provider API key = %q", r.Header.Get("X-API-Key")) + http.Error(w, "missing key", http.StatusUnauthorized) + return + } + w.Header().Set("Content-Type", "application/json") + switch { + case r.Method == http.MethodGet && r.URL.Path == apiBase+"/sites": + switch r.URL.Query().Get("limit") { + case "1": + _, _ = fmt.Fprint(w, `{"offset":0,"limit":1,"count":0,"totalCount":0,"data":[]}`) + case "200": + _, _ = fmt.Fprintf(w, `{"offset":0,"limit":200,"count":1,"totalCount":1,"data":[{"id":%q,"internalReference":"default","name":"Default"}]}`, siteID) + default: + t.Errorf("unexpected sites query: %s", r.URL) + http.Error(w, "unexpected query", http.StatusBadRequest) + } + case r.Method == http.MethodGet && r.URL.Path == listPath: + if err := json.NewEncoder(w).Encode(f.list); err != nil { + t.Errorf("encode provider list: %v", err) + } + case r.Method == http.MethodPut && r.URL.Path == listPath: + var raw json.RawMessage + if err := json.NewDecoder(r.Body).Decode(&raw); err != nil { + t.Errorf("decode provider payload: %v", err) + http.Error(w, "invalid JSON", http.StatusBadRequest) + return + } + var update clients.NetworkList + if err := json.Unmarshal(raw, &update); err != nil { + t.Errorf("decode provider items: %v", err) + http.Error(w, "invalid list", http.StatusBadRequest) + return + } + f.payloads = append(f.payloads, raw) + f.list.Name, f.list.Type, f.list.Items = update.Name, update.Type, update.Items + w.WriteHeader(http.StatusNoContent) + default: + t.Errorf("unexpected provider request: %s %s", r.Method, r.URL) + http.Error(w, "not found", http.StatusNotFound) + } + })) + t.Cleanup(ts.Close) + f.controllerID, err = f.db.CreateController(&store.Controller{ + Provider: "unifi", Name: "Loopback UniFi", URL: ts.URL, Site: "default", APIKey: "test-key", + }) + if err != nil { + t.Fatal(err) + } + return f +} + +func (f *sourceSyncFixture) source(t *testing.T, name, entries string, ids ...int64) *store.SourceList { + t.Helper() + source := &store.SourceList{Name: name, Hostnames: entries, IncludedListIDs: ids} + if _, err := f.db.CreateSourceList(source); err != nil { + t.Fatal(err) + } + return source +} + +func (f *sourceSyncFixture) job(t *testing.T, entries string, ids ...int64) *store.SyncJob { + t.Helper() + job := &store.SyncJob{ + Name: "Source integration", ControllerID: f.controllerID, NetworkListID: f.list.ID, + Hostnames: entries, IncludedListIDs: ids, ObservedIPTTLHours: 0, Enabled: true, + } + var err error + job.ID, err = f.db.CreateJob(job) + if err != nil { + t.Fatal(err) + } + return job +} + +func (f *sourceSyncFixture) assertPayload(t *testing.T, count int, want []clients.TrafficMatchItem) { + t.Helper() + f.mu.Lock() + defer f.mu.Unlock() + if len(f.payloads) != count { + t.Fatalf("provider writes = %d, want %d", len(f.payloads), count) + } + raw := f.payloads[count-1] + var payload struct { + Type string `json:"type"` + Name string `json:"name"` + Items []clients.TrafficMatchItem `json:"items"` + } + if err := json.Unmarshal(raw, &payload); err != nil { + t.Fatal(err) + } + if payload.Type != "IPV4_ADDRESSES" || payload.Name != "Allowlist" || !reflect.DeepEqual(payload.Items, want) { + t.Fatalf("provider payload = %s, want flat items %+v", raw, want) + } + var fields map[string]json.RawMessage + if err := json.Unmarshal(raw, &fields); err != nil { + t.Fatal(err) + } + if len(fields) != 3 { + t.Fatalf("provider payload must contain only type/name/items, got %s", raw) + } + // Check the wire shape, not merely a decoder that ignores unknown fields. + var items []map[string]json.RawMessage + if err := json.Unmarshal(fields["items"], &items); err != nil { + t.Fatal(err) + } + for _, item := range items { + if len(item) != 2 || item["type"] == nil || item["value"] == nil { + t.Fatalf("non-flat provider item: %v", item) + } + } +} + +func TestSourceListsRunNestedFlatPayload(t *testing.T) { + f := newSourceSyncFixture(t) + leaf := f.source(t, "RFC1918", "10.0.0.0/8\n172.16.0.0/12\n192.168.0.0/16") + left := f.source(t, "Left", "10.1.2.3\n10.0.0.0/8", leaf.ID) + right := f.source(t, "Right", "192.168.1.0/24\n203.0.113.10", leaf.ID) + root := f.source(t, "Root", "172.16.1.1", left.ID, right.ID) + job := f.job(t, "203.0.113.10\n203.0.113.10", root.ID, leaf.ID) + + result := New().Run(f.db, job.ID) + if result.Status != "success" || result.ChangesMade != 4 { + t.Fatalf("Run = %+v", result) + } + want := []clients.TrafficMatchItem{ + {Type: "SUBNET", Value: "10.0.0.0/8"}, + {Type: "SUBNET", Value: "172.16.0.0/12"}, + {Type: "SUBNET", Value: "192.168.0.0/16"}, + {Type: "IP_ADDRESS", Value: "203.0.113.10"}, + } + f.assertPayload(t, 1, want) + // Shared descendants are expanded once; repeated entries still contribute + // to duplicate statistics before covered addresses/subnets are removed. + if result.Stats.InputEntries != 10 || result.Stats.UniqueEntries != 7 || + result.Stats.DuplicateEntries != 3 || result.Stats.CoveredEntriesRemoved != 3 || result.Stats.OutputEntries != 4 { + t.Fatalf("nested optimization stats = %+v", result.Stats) + } + if len(result.Targets) != 1 || result.Targets[0].EntryCount != len(want) { + t.Fatalf("sent target snapshots = %+v", result.Targets) + } + var snapshot []clients.TrafficMatchItem + if err := json.Unmarshal(result.Targets[0].Items, &snapshot); err != nil || !reflect.DeepEqual(snapshot, want) { + t.Fatalf("sent snapshot = %+v, error = %v", snapshot, err) + } + // The mock retains writes, so the same job must not send a second PUT. + if again := New().Run(f.db, job.ID); again.Status != "success" || again.ChangesMade != 0 { + t.Fatalf("unchanged Run = %+v", again) + } + f.assertPayload(t, 1, want) +} + +func TestSourceListsRunIncludesOnlyUsesSourceEdits(t *testing.T) { + f := newSourceSyncFixture(t) + leaf := f.source(t, "Editable", "10.0.0.0/8\n203.0.113.10") + root := f.source(t, "Includes only", "", leaf.ID) + job := f.job(t, "", root.ID) + syn := New() + if result := syn.Run(f.db, job.ID); result.Status != "success" { + t.Fatalf("includes-only Run = %+v", result) + } + f.assertPayload(t, 1, []clients.TrafficMatchItem{ + {Type: "SUBNET", Value: "10.0.0.0/8"}, {Type: "IP_ADDRESS", Value: "203.0.113.10"}, + }) + leaf.Hostnames = "192.168.0.0/16\n203.0.113.20" + if err := f.db.UpdateSourceList(leaf); err != nil { + t.Fatal(err) + } + if result := syn.Run(f.db, job.ID); result.Status != "success" || result.ChangesMade != 4 { + t.Fatalf("Run after source edit = %+v", result) + } + f.assertPayload(t, 2, []clients.TrafficMatchItem{ + {Type: "SUBNET", Value: "192.168.0.0/16"}, {Type: "IP_ADDRESS", Value: "203.0.113.20"}, + }) + saved, err := f.db.GetJob(job.ID) + if err != nil { + t.Fatal(err) + } + if saved.Hostnames != "" || saved.ObservedIPTTLHours != 0 || !reflect.DeepEqual(saved.IncludedListIDs, []int64{root.ID}) { + t.Fatalf("source edit changed job configuration: %+v", saved) + } +} + +func (f *sourceSyncFixture) assertExpansionFailure(t *testing.T, result SyncResult, message string) { + t.Helper() + if result.Status != "error" || !strings.HasPrefix(result.Message, "expand source lists: ") || + !strings.Contains(result.Message, message) || result.ChangesMade != 0 || len(result.Targets) != 0 { + t.Fatalf("expected expansion failure containing %q, got %+v", message, result) + } + f.mu.Lock() + defer f.mu.Unlock() + if f.requests != 0 || len(f.payloads) != 0 { + t.Fatalf("invalid sources reached provider: requests=%d writes=%d", f.requests, len(f.payloads)) + } +} + +func TestSourceListsExecuteMissingReferenceBeforeDNSOrProvider(t *testing.T) { + f := newSourceSyncFixture(t) + // A synthetic job bypasses CreateJob's missing-reference safeguards. + job := &store.SyncJob{ + ControllerID: f.controllerID, NetworkListID: f.list.ID, + Hostnames: "203.0.113.10", IncludedListIDs: []int64{99999}, ObservedIPTTLHours: 0, + } + servers, err := f.db.ListDNSServers() + if err != nil { + t.Fatal(err) + } + for _, server := range servers { + if err := f.db.DeleteDNSServer(server.ID); err != nil { + t.Fatal(err) + } + } + f.assertExpansionFailure(t, New().execute(f.db, job), "source list ID 99999 does not exist") +} + +func TestSourceListsRunCorruptReferencesBeforeProvider(t *testing.T) { + for _, tc := range []struct { + name string + raw string + message string + }{ + {name: "malformed JSON", raw: "invalid", message: "invalid included_list_ids JSON array"}, + {name: "null refs", raw: "null", message: "invalid included_list_ids JSON array"}, + {name: "object refs", raw: "{}", message: "invalid included_list_ids JSON array"}, + {name: "missing nested ref", raw: "[99999]", message: "source list ID 99999 does not exist"}, + {name: "nested cycle", message: "source list cycle detected"}, + } { + t.Run(tc.name, func(t *testing.T) { + f := newSourceSyncFixture(t) + leaf := f.source(t, "Leaf", "10.0.0.0/8") + root := f.source(t, "Root", "", leaf.ID) + job := f.job(t, "203.0.113.10", root.ID) + if tc.name == "nested cycle" { + tc.raw = fmt.Sprintf("[%d]", root.ID) + } + // Normal store writes reject bad references and cycles. Deliberately + // corrupt persisted data to exercise the syncer's defensive path. + rawDB, err := sql.Open("sqlite", f.dbPath) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = rawDB.Close() }) + if _, err := rawDB.Exec("UPDATE source_lists SET included_list_ids = ? WHERE id = ?", tc.raw, leaf.ID); err != nil { + t.Fatal(err) + } + result := New().Run(f.db, job.ID) + f.assertExpansionFailure(t, result, tc.message) + logs, err := f.db.GetRunLogs(job.ID, 10) + if err != nil || len(logs) != 1 || logs[0].Status != "error" || logs[0].Message != result.Message { + t.Fatalf("failed run logs = %+v, error = %v", logs, err) + } + }) + } +} + +func TestSourceListsCycleSafeguardPreservesRunnableGraph(t *testing.T) { + f := newSourceSyncFixture(t) + leaf := f.source(t, "Leaf", "10.0.0.0/8") + root := f.source(t, "Root", "", leaf.ID) + job := f.job(t, "", root.ID) + for _, id := range []int64{leaf.ID, root.ID} { + leaf.IncludedListIDs = []int64{id} + if err := f.db.UpdateSourceList(leaf); err == nil || !strings.Contains(err.Error(), "cycle") { + t.Fatalf("accepted cycle referencing %d: %v", id, err) + } + } + f.mu.Lock() + requests, writes := f.requests, len(f.payloads) + f.mu.Unlock() + if requests != 0 || writes != 0 { + t.Fatalf("cycle validation reached provider: requests=%d writes=%d", requests, writes) + } + if result := New().Run(f.db, job.ID); result.Status != "success" { + t.Fatalf("Run after rejected cycles = %+v", result) + } + f.assertPayload(t, 1, []clients.TrafficMatchItem{{Type: "SUBNET", Value: "10.0.0.0/8"}}) +} diff --git a/internal/syncer/syncer.go b/internal/syncer/syncer.go index 9f46d52..8e84e75 100644 --- a/internal/syncer/syncer.go +++ b/internal/syncer/syncer.go @@ -76,6 +76,10 @@ func (s *Syncer) Run(db *store.Store, jobID int64) SyncResult { } func (s *Syncer) execute(db *store.Store, job *store.SyncJob) SyncResult { + hostnames, err := db.ExpandSourceEntries(job.Hostnames, job.IncludedListIDs) + if err != nil { + return SyncResult{Status: "error", Message: fmt.Sprintf("expand source lists: %v", err)} + } servers, err := db.ListEnabledDNSServerAddresses() if err != nil { return SyncResult{Status: "error", Message: fmt.Sprintf("load DNS servers: %v", err)} @@ -83,7 +87,7 @@ func (s *Syncer) execute(db *store.Store, job *store.SyncJob) SyncResult { if len(servers) == 0 { return SyncResult{Status: "error", Message: "no DNS servers configured: add at least one enabled DNS server"} } - hostIPs, resolutionStats, err := ResolveHostnamesWithStats(job.Hostnames, servers) + hostIPs, resolutionStats, err := ResolveHostnamesWithStats(hostnames, servers) if err != nil { return SyncResult{Status: "error", Message: fmt.Sprintf("DNS resolution: %v", err)} } diff --git a/internal/web/handler.go b/internal/web/handler.go index 190caee..10688c5 100644 --- a/internal/web/handler.go +++ b/internal/web/handler.go @@ -66,6 +66,12 @@ func NewHandler(s *store.Store, syn *syncer.Syncer, sched *scheduler.Scheduler, mux.HandleFunc("GET /api/instances/{id}/target-lists", h.listNetworkLists) mux.HandleFunc("POST /api/instances/test", h.testController) + mux.HandleFunc("GET /api/source-lists", h.listSourceLists) + mux.HandleFunc("POST /api/source-lists", h.createSourceList) + mux.HandleFunc("GET /api/source-lists/{id}", h.getSourceList) + mux.HandleFunc("PUT /api/source-lists/{id}", h.updateSourceList) + mux.HandleFunc("DELETE /api/source-lists/{id}", h.deleteSourceList) + mux.HandleFunc("GET /api/jobs", h.listJobs) mux.HandleFunc("POST /api/jobs", h.createJob) mux.HandleFunc("GET /api/jobs/{id}", h.getJob) @@ -451,10 +457,14 @@ func (h *Handler) createJob(w http.ResponseWriter, r *http.Request) { return } - if job.Name == "" || job.Hostnames == "" { + if strings.TrimSpace(job.Name) == "" || (strings.TrimSpace(job.Hostnames) == "" && len(job.IncludedListIDs) == 0) { writeError(w, http.StatusBadRequest, "missing required fields") return } + if err := h.store.ValidateSourceEntries(job.Hostnames, job.IncludedListIDs); err != nil { + writeSourceEntriesError(w, err) + return + } if len(job.Targets) == 0 { if job.ControllerID == 0 || job.NetworkListID == "" { writeError(w, http.StatusBadRequest, "missing required target fields") @@ -468,11 +478,13 @@ func (h *Handler) createJob(w http.ResponseWriter, r *http.Request) { id, err := h.store.CreateJob(&job) if err != nil { - writeError(w, http.StatusInternalServerError, err.Error()) + writeSourceEntriesError(w, err) return } - h.scheduler.Reload(id) + if h.scheduler != nil { + h.scheduler.Reload(id) + } job.ID = id writeJSON(w, http.StatusCreated, job) @@ -509,6 +521,14 @@ func (h *Handler) updateJob(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusBadRequest, "invalid JSON") return } + if strings.TrimSpace(job.Name) == "" || (strings.TrimSpace(job.Hostnames) == "" && len(job.IncludedListIDs) == 0) { + writeError(w, http.StatusBadRequest, "missing required fields") + return + } + if err := h.store.ValidateSourceEntries(job.Hostnames, job.IncludedListIDs); err != nil { + writeSourceEntriesError(w, err) + return + } if len(job.Targets) == 0 { if job.ControllerID == 0 || job.NetworkListID == "" { writeError(w, http.StatusBadRequest, "missing required target fields") @@ -522,11 +542,13 @@ func (h *Handler) updateJob(w http.ResponseWriter, r *http.Request) { job.ID = id if err := h.store.UpdateJob(&job); err != nil { - writeError(w, http.StatusInternalServerError, err.Error()) + writeSourceEntriesError(w, err) return } - h.scheduler.Reload(id) + if h.scheduler != nil { + h.scheduler.Reload(id) + } writeJSON(w, http.StatusOK, job) } @@ -541,7 +563,9 @@ func (h *Handler) deleteJob(w http.ResponseWriter, r *http.Request) { return } - h.scheduler.Remove(id) + if h.scheduler != nil { + h.scheduler.Remove(id) + } if err := h.store.DeleteJob(id); err != nil { writeError(w, http.StatusInternalServerError, err.Error()) @@ -669,19 +693,29 @@ func (h *Handler) resolveHostnames(w http.ResponseWriter, r *http.Request) { return } var input struct { - Hostnames string `json:"hostnames"` + Hostnames string `json:"hostnames"` + IncludedListIDs []int64 `json:"included_list_ids"` } if err := json.NewDecoder(r.Body).Decode(&input); err != nil { writeError(w, http.StatusBadRequest, "invalid JSON") return } - extraServers, _ := h.store.ListEnabledDNSServerAddresses() + hostnames, err := h.store.ExpandSourceEntries(input.Hostnames, input.IncludedListIDs) + if err != nil { + writeSourceEntriesError(w, err) + return + } + extraServers, err := h.store.ListEnabledDNSServerAddresses() + if err != nil { + writeError(w, http.StatusInternalServerError, err.Error()) + return + } if len(extraServers) == 0 { writeError(w, http.StatusUnprocessableEntity, "no DNS servers configured: add at least one enabled DNS server") return } - hostIPs, err := syncer.ResolveHostnames(input.Hostnames, extraServers) + hostIPs, err := syncer.ResolveHostnames(hostnames, extraServers) if err != nil { writeError(w, http.StatusUnprocessableEntity, err.Error()) return diff --git a/internal/web/handler_source_lists.go b/internal/web/handler_source_lists.go new file mode 100644 index 0000000..d2e8d0a --- /dev/null +++ b/internal/web/handler_source_lists.go @@ -0,0 +1,156 @@ +package web + +import ( + "database/sql" + "encoding/json" + "errors" + "net/http" + "strings" + + "github.com/mstrhakr/network-list-sync/internal/store" +) + +func (h *Handler) listSourceLists(w http.ResponseWriter, r *http.Request) { + lists, err := h.store.ListSourceLists() + if err != nil { + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + if lists == nil { + lists = []store.SourceList{} + } + writeJSON(w, http.StatusOK, lists) +} + +func (h *Handler) getSourceList(w http.ResponseWriter, r *http.Request) { + id, err := parseID(r) + if err != nil || id <= 0 { + writeError(w, http.StatusBadRequest, "invalid source list ID") + return + } + list, err := h.store.GetSourceList(id) + if err != nil { + writeSourceListError(w, err) + return + } + writeJSON(w, http.StatusOK, list) +} + +func (h *Handler) createSourceList(w http.ResponseWriter, r *http.Request) { + if !h.requireAdmin(w, r) { + return + } + var list store.SourceList + if err := json.NewDecoder(r.Body).Decode(&list); err != nil { + writeError(w, http.StatusBadRequest, "invalid JSON") + return + } + list.Name = strings.TrimSpace(list.Name) + if list.Name == "" { + writeError(w, http.StatusBadRequest, "name is required") + return + } + id, err := h.store.CreateSourceList(&list) + if err != nil { + writeSourceListError(w, err) + return + } + // Return persisted values, including normalized references and timestamps. + saved, err := h.store.GetSourceList(id) + if err != nil { + writeSourceListError(w, err) + return + } + writeJSON(w, http.StatusCreated, saved) +} + +func (h *Handler) updateSourceList(w http.ResponseWriter, r *http.Request) { + if !h.requireAdmin(w, r) { + return + } + id, err := parseID(r) + if err != nil || id <= 0 { + writeError(w, http.StatusBadRequest, "invalid source list ID") + return + } + if _, err := h.store.GetSourceList(id); err != nil { + writeSourceListError(w, err) + return + } + var list store.SourceList + if err := json.NewDecoder(r.Body).Decode(&list); err != nil { + writeError(w, http.StatusBadRequest, "invalid JSON") + return + } + list.ID = id + list.Name = strings.TrimSpace(list.Name) + if list.Name == "" { + writeError(w, http.StatusBadRequest, "name is required") + return + } + if err := h.store.UpdateSourceList(&list); err != nil { + writeSourceListError(w, err) + return + } + saved, err := h.store.GetSourceList(id) + if err != nil { + writeSourceListError(w, err) + return + } + writeJSON(w, http.StatusOK, saved) +} + +func (h *Handler) deleteSourceList(w http.ResponseWriter, r *http.Request) { + if !h.requireAdmin(w, r) { + return + } + id, err := parseID(r) + if err != nil || id <= 0 { + writeError(w, http.StatusBadRequest, "invalid source list ID") + return + } + if _, err := h.store.GetSourceList(id); err != nil { + writeSourceListError(w, err) + return + } + if err := h.store.DeleteSourceList(id); err != nil { + writeSourceListError(w, err) + return + } + w.WriteHeader(http.StatusNoContent) +} + +func writeSourceEntriesError(w http.ResponseWriter, err error) { + status := http.StatusInternalServerError + var validationErr *store.SourceValidationError + if errors.As(err, &validationErr) { + status = http.StatusBadRequest + } + writeError(w, status, err.Error()) +} + +func writeSourceListError(w http.ResponseWriter, err error) { + if errors.Is(err, sql.ErrNoRows) { + writeError(w, http.StatusNotFound, "source list not found") + return + } + var validationErr *store.SourceValidationError + if errors.As(err, &validationErr) { + writeSourceEntriesError(w, err) + return + } + // Reference conflicts still use descriptive errors. Inspect the root cause + // so nested-list names cannot affect classification. + cause := err + for errors.Unwrap(cause) != nil { + cause = errors.Unwrap(cause) + } + message := cause.Error() + status := http.StatusInternalServerError + switch { + case strings.HasPrefix(message, "source list ") && + (strings.Contains(message, " is referenced by source list ") || strings.Contains(message, " is referenced by job ")): + status = http.StatusConflict + } + writeError(w, status, err.Error()) +} diff --git a/internal/web/handler_source_lists_test.go b/internal/web/handler_source_lists_test.go new file mode 100644 index 0000000..91fa6fa --- /dev/null +++ b/internal/web/handler_source_lists_test.go @@ -0,0 +1,420 @@ +package web + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "path/filepath" + "reflect" + "strings" + "testing" + "testing/fstest" + + "github.com/mstrhakr/network-list-sync/internal/auth" + "github.com/mstrhakr/network-list-sync/internal/scheduler" + "github.com/mstrhakr/network-list-sync/internal/store" + "github.com/mstrhakr/network-list-sync/internal/syncer" +) + +type sourceListWebFixture struct { + store *store.Store + h http.Handler + admin *http.Cookie + reader *http.Cookie +} + +func newSourceListWebFixture(t *testing.T) *sourceListWebFixture { + t.Helper() + s, err := store.New(filepath.Join(t.TempDir(), "sync.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = s.Close() }) + syn := syncer.New() + sched := scheduler.New(s, syn) + t.Cleanup(sched.Stop) + uiFS := fstest.MapFS{ + "templates/index.gohtml": {Data: []byte(`{{define "index"}}index{{end}}`)}, + "templates/login.gohtml": {Data: []byte(`{{define "login"}}{{.Error}}{{end}}`)}, + "templates/partials/shell.gohtml": {Data: []byte(`{{define "shell"}}shell{{end}}`)}, + "static/css/app.css": {Data: []byte("body{}")}, + } + f := &sourceListWebFixture{store: s, h: NewHandler(s, syn, sched, uiFS)} + f.admin = f.login(t, "admin", true) + f.request(t, http.MethodPost, "/api/users", map[string]any{ + "username": "reader", "password": "source-lists-secure-pass", "is_admin": false, + }, f.admin, http.StatusCreated) + f.reader = f.login(t, "reader", false) + return f +} + +func (f *sourceListWebFixture) login(t *testing.T, username string, setup bool) *http.Cookie { + t.Helper() + form := url.Values{"username": {username}, "password": {"source-lists-secure-pass"}, "provider": {"local"}} + if setup { + form.Set("confirm_password", "source-lists-secure-pass") + } + req := httptest.NewRequest(http.MethodPost, "/login", strings.NewReader(form.Encode())) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + rr := httptest.NewRecorder() + f.h.ServeHTTP(rr, req) + if rr.Code != http.StatusSeeOther || len(rr.Result().Cookies()) == 0 { + t.Fatalf("login %s: status=%d body=%s", username, rr.Code, rr.Body.String()) + } + return rr.Result().Cookies()[0] +} + +func (f *sourceListWebFixture) request(t *testing.T, method, path string, payload any, cookie *http.Cookie, status int) *httptest.ResponseRecorder { + t.Helper() + var body []byte + if raw, ok := payload.(string); ok { + body = []byte(raw) + } else if payload != nil { + var err error + body, err = json.Marshal(payload) + if err != nil { + t.Fatal(err) + } + } + req := httptest.NewRequest(method, path, bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + if cookie != nil { + req.AddCookie(cookie) + } + rr := httptest.NewRecorder() + f.h.ServeHTTP(rr, req) + if rr.Code != status { + t.Fatalf("%s %s: status=%d want=%d body=%s", method, path, rr.Code, status, rr.Body.String()) + } + if status >= 400 && !strings.Contains(rr.Body.String(), `"error":`) { + t.Fatalf("%s %s: missing JSON error: %s", method, path, rr.Body.String()) + } + return rr +} + +func decodeSourceList(t *testing.T, rr *httptest.ResponseRecorder) store.SourceList { + t.Helper() + var list store.SourceList + if err := json.Unmarshal(rr.Body.Bytes(), &list); err != nil { + t.Fatal(err) + } + return list +} + +func (f *sourceListWebFixture) createList(t *testing.T, name, hostnames string, ids ...int64) store.SourceList { + t.Helper() + return decodeSourceList(t, f.request(t, http.MethodPost, "/api/source-lists", store.SourceList{ + Name: name, Hostnames: hostnames, IncludedListIDs: ids, + }, f.admin, http.StatusCreated)) +} + +func sourceListPath(id int64) string { + return fmt.Sprintf("/api/source-lists/%d", id) +} + +func TestSourceListHTTPCRUDAndNesting(t *testing.T) { + f := newSourceListWebFixture(t) + rr := f.request(t, http.MethodGet, "/api/source-lists", nil, f.reader, http.StatusOK) + if strings.TrimSpace(rr.Body.String()) != "[]" { + t.Fatalf("empty lists = %s, want []", rr.Body.String()) + } + leaf := f.createList(t, " leaf ", "192.0.2.1") + if leaf.ID <= 0 || leaf.Name != "leaf" || leaf.CreatedAt == "" || leaf.UpdatedAt == "" { + t.Fatalf("created list = %+v", leaf) + } + parent := f.createList(t, "parent", "", leaf.ID) + root := f.createList(t, "root", "192.0.2.2", parent.ID) + got := decodeSourceList(t, f.request(t, http.MethodGet, sourceListPath(root.ID), nil, f.reader, http.StatusOK)) + if got.ID != root.ID || !reflect.DeepEqual(got.IncludedListIDs, []int64{parent.ID}) { + t.Fatalf("GET root = %+v", got) + } + rr = f.request(t, http.MethodGet, "/api/source-lists", nil, f.reader, http.StatusOK) + var lists []store.SourceList + if err := json.Unmarshal(rr.Body.Bytes(), &lists); err != nil || len(lists) != 3 { + t.Fatalf("list response=%s error=%v", rr.Body.String(), err) + } + updated := decodeSourceList(t, f.request(t, http.MethodPut, sourceListPath(root.ID), map[string]any{ + "id": leaf.ID, "name": "renamed", "hostnames": "192.0.2.3", "included_list_ids": []int64{parent.ID}, + }, f.admin, http.StatusOK)) + if updated.ID != root.ID || updated.Name != "renamed" || updated.CreatedAt != root.CreatedAt { + t.Fatalf("updated list = %+v", updated) + } + expanded, err := f.store.ExpandSourceEntries("", []int64{root.ID}) + if err != nil || !strings.Contains(expanded, "192.0.2.1") || !strings.Contains(expanded, "192.0.2.3") { + t.Fatalf("nested expansion=%q error=%v", expanded, err) + } + f.request(t, http.MethodDelete, sourceListPath(leaf.ID), nil, f.admin, http.StatusConflict) + f.request(t, http.MethodDelete, sourceListPath(root.ID), nil, f.admin, http.StatusNoContent) + f.request(t, http.MethodGet, sourceListPath(root.ID), nil, f.reader, http.StatusNotFound) + f.request(t, http.MethodDelete, sourceListPath(parent.ID), nil, f.admin, http.StatusNoContent) + f.request(t, http.MethodDelete, sourceListPath(leaf.ID), nil, f.admin, http.StatusNoContent) +} + +func TestSourceListHTTPAuthentication(t *testing.T) { + f := newSourceListWebFixture(t) + list := f.createList(t, "list", "192.0.2.1") + for _, tc := range []struct{ method, path string }{ + {http.MethodGet, "/api/source-lists"}, + {http.MethodGet, sourceListPath(list.ID)}, + {http.MethodPost, "/api/source-lists"}, + {http.MethodPut, sourceListPath(list.ID)}, + {http.MethodDelete, sourceListPath(list.ID)}, + } { + t.Run("unauthenticated_"+tc.method+tc.path, func(t *testing.T) { + f.request(t, tc.method, tc.path, nil, nil, http.StatusUnauthorized) + }) + if tc.method != http.MethodGet { + t.Run("reader_"+tc.method, func(t *testing.T) { + // Authorization must run before JSON decoding or any mutation. + f.request(t, tc.method, tc.path, "{", f.reader, http.StatusForbidden) + }) + } + } + f.request(t, http.MethodPost, "/api/resolve", `{"included_list_ids":[1]}`, f.reader, http.StatusForbidden) + f.request(t, http.MethodPost, "/api/jobs", `{"included_list_ids":[1]}`, f.reader, http.StatusForbidden) + f.request(t, http.MethodPut, "/api/jobs/1", `{"included_list_ids":[1]}`, f.reader, http.StatusForbidden) + got, err := f.store.GetSourceList(list.ID) + if err != nil || got.Name != "list" { + t.Fatalf("unauthorized mutation: list=%+v err=%v", got, err) + } +} + +func TestSourceListHTTPValidationAndMissing(t *testing.T) { + f := newSourceListWebFixture(t) + leaf := f.createList(t, "leaf", "192.0.2.1") + parent := f.createList(t, "parent", "", leaf.ID) + for _, tc := range []struct { + name, method, path string + payload any + status int + }{ + {"create JSON", http.MethodPost, "/api/source-lists", "{", http.StatusBadRequest}, + {"create name", http.MethodPost, "/api/source-lists", `{"name":" "}`, http.StatusBadRequest}, + {"create empty content", http.MethodPost, "/api/source-lists", `{"name":"empty","hostnames":" "}`, http.StatusBadRequest}, + {"create reference", http.MethodPost, "/api/source-lists", `{"name":"invalid","included_list_ids":[99999]}`, http.StatusBadRequest}, + {"create negative reference", http.MethodPost, "/api/source-lists", `{"name":"invalid","included_list_ids":[-1]}`, http.StatusBadRequest}, + {"update JSON", http.MethodPut, sourceListPath(leaf.ID), "{", http.StatusBadRequest}, + {"update name", http.MethodPut, sourceListPath(leaf.ID), `{"name":" "}`, http.StatusBadRequest}, + {"update empty content", http.MethodPut, sourceListPath(leaf.ID), `{"name":"empty"}`, http.StatusBadRequest}, + {"update reference", http.MethodPut, sourceListPath(leaf.ID), `{"name":"invalid","included_list_ids":[99999]}`, http.StatusBadRequest}, + {"self cycle", http.MethodPut, sourceListPath(leaf.ID), store.SourceList{Name: "cycle", IncludedListIDs: []int64{leaf.ID}}, http.StatusBadRequest}, + {"nested cycle", http.MethodPut, sourceListPath(leaf.ID), store.SourceList{Name: "cycle", IncludedListIDs: []int64{parent.ID}}, http.StatusBadRequest}, + {"missing get", http.MethodGet, sourceListPath(99999), nil, http.StatusNotFound}, + {"missing update", http.MethodPut, sourceListPath(99999), store.SourceList{Name: "missing", Hostnames: "192.0.2.1"}, http.StatusNotFound}, + {"missing delete", http.MethodDelete, sourceListPath(99999), nil, http.StatusNotFound}, + } { + t.Run(tc.name, func(t *testing.T) { + f.request(t, tc.method, tc.path, tc.payload, f.admin, tc.status) + }) + } + for _, method := range []string{http.MethodGet, http.MethodPut, http.MethodDelete} { + for _, id := range []string{"x", "0", "-1"} { + f.request(t, method, "/api/source-lists/"+id, `{"name":"valid"}`, f.admin, http.StatusBadRequest) + } + } + got, err := f.store.GetSourceList(leaf.ID) + if err != nil || got.Name != "leaf" || got.Hostnames != "192.0.2.1" || len(got.IncludedListIDs) != 0 { + t.Fatalf("invalid updates changed leaf: %+v err=%v", got, err) + } +} + +func TestSourceListHTTPPreview(t *testing.T) { + f := newSourceListWebFixture(t) + leaf := f.createList(t, "leaf", "192.0.2.1") + parent := f.createList(t, "parent", "192.0.2.2", leaf.ID) + rr := f.request(t, http.MethodPost, "/api/resolve", map[string]any{ + "hostnames": "192.0.2.3", "included_list_ids": []int64{parent.ID}, + }, f.admin, http.StatusOK) + var result []struct { + IP string `json:"ip"` + } + if err := json.Unmarshal(rr.Body.Bytes(), &result); err != nil { + t.Fatal(err) + } + if len(result) != 3 || result[0].IP != "192.0.2.1" || result[1].IP != "192.0.2.2" || result[2].IP != "192.0.2.3" { + t.Fatalf("nested preview=%s", rr.Body.String()) + } + rr = f.request(t, http.MethodPost, "/api/resolve", map[string]any{"included_list_ids": []int64{leaf.ID}}, f.admin, http.StatusOK) + if !strings.Contains(rr.Body.String(), "192.0.2.1") { + t.Fatalf("includes-only preview=%s", rr.Body.String()) + } + // Graph validation precedes DNS configuration checks. + servers, err := f.store.ListDNSServers() + if err != nil { + t.Fatal(err) + } + for _, server := range servers { + server.Enabled = false + if err := f.store.UpdateDNSServer(&server); err != nil { + t.Fatal(err) + } + } + for _, payload := range []string{`{"included_list_ids":[99999]}`, `{"hostnames":"192.0.2.4","included_list_ids":[-1]}`, "{"} { + f.request(t, http.MethodPost, "/api/resolve", payload, f.admin, http.StatusBadRequest) + } + f.request(t, http.MethodPost, "/api/resolve", map[string]any{"included_list_ids": []int64{leaf.ID}}, f.admin, http.StatusUnprocessableEntity) +} + +func TestSourceListHTTPIncludesOnlyJobs(t *testing.T) { + f := newSourceListWebFixture(t) + leaf := f.createList(t, "leaf", "192.0.2.1") + parent := f.createList(t, "parent", "", leaf.ID) + controllerID, err := f.store.CreateController(&store.Controller{Name: "test", URL: "https://example.test", APIKey: "key"}) + if err != nil { + t.Fatal(err) + } + job := store.SyncJob{Name: "includes-only", ControllerID: controllerID, NetworkListID: "target", IncludedListIDs: []int64{parent.ID}} + rr := f.request(t, http.MethodPost, "/api/jobs", job, f.admin, http.StatusCreated) + var saved store.SyncJob + if err := json.Unmarshal(rr.Body.Bytes(), &saved); err != nil { + t.Fatal(err) + } + path := fmt.Sprintf("/api/jobs/%d", saved.ID) + got, err := f.store.GetJob(saved.ID) + if err != nil || got.Hostnames != "" || !reflect.DeepEqual(got.IncludedListIDs, job.IncludedListIDs) { + t.Fatalf("saved job=%+v err=%v", got, err) + } + f.request(t, http.MethodDelete, sourceListPath(parent.ID), nil, f.admin, http.StatusConflict) + job.IncludedListIDs = []int64{leaf.ID} + f.request(t, http.MethodPut, path, job, f.admin, http.StatusOK) + f.request(t, http.MethodGet, path, nil, f.reader, http.StatusOK) + f.request(t, http.MethodGet, "/api/jobs", nil, f.reader, http.StatusOK) + for _, ids := range [][]int64{{99999}, {-1}, nil} { + job.IncludedListIDs = ids + f.request(t, http.MethodPost, "/api/jobs", job, f.admin, http.StatusBadRequest) + f.request(t, http.MethodPut, path, job, f.admin, http.StatusBadRequest) + } + got, err = f.store.GetJob(saved.ID) + if err != nil || !reflect.DeepEqual(got.IncludedListIDs, []int64{leaf.ID}) { + t.Fatalf("invalid update changed job=%+v err=%v", got, err) + } + // Existing inline-only jobs remain valid, and omitting includes clears them. + job.Hostnames = "192.0.2.2" + job.IncludedListIDs = nil + f.request(t, http.MethodPut, path, job, f.admin, http.StatusOK) + got, err = f.store.GetJob(saved.ID) + if err != nil || got.Hostnames != job.Hostnames || len(got.IncludedListIDs) != 0 { + t.Fatalf("inline-only update=%+v err=%v", got, err) + } + job.Hostnames = "" + job.IncludedListIDs = []int64{leaf.ID} + f.request(t, http.MethodPut, path, job, f.admin, http.StatusOK) + f.request(t, http.MethodDelete, sourceListPath(leaf.ID), nil, f.admin, http.StatusConflict) + f.request(t, http.MethodDelete, path, nil, f.admin, http.StatusNoContent) + f.request(t, http.MethodDelete, sourceListPath(parent.ID), nil, f.admin, http.StatusNoContent) + f.request(t, http.MethodDelete, sourceListPath(leaf.ID), nil, f.admin, http.StatusNoContent) +} + +func TestSourceListJobsWithoutScheduler(t *testing.T) { + f := newSourceListWebFixture(t) + list := f.createList(t, "list", "192.0.2.1") + controllerID, err := f.store.CreateController(&store.Controller{Name: "test", URL: "https://example.test", APIKey: "key"}) + if err != nil { + t.Fatal(err) + } + h := &Handler{store: f.store} + job := store.SyncJob{Name: "job", ControllerID: controllerID, NetworkListID: "target", IncludedListIDs: []int64{list.ID}} + body, err := json.Marshal(job) + if err != nil { + t.Fatal(err) + } + request := func(fn http.HandlerFunc, id string, status int) *httptest.ResponseRecorder { + req := httptest.NewRequest(http.MethodPost, "/api/jobs", bytes.NewReader(body)) + req.SetPathValue("id", id) + req = req.WithContext(context.WithValue(req.Context(), principalContextKey, &auth.Principal{IsAdmin: true})) + rr := httptest.NewRecorder() + fn(rr, req) + if rr.Code != status { + t.Fatalf("status=%d want=%d body=%s", rr.Code, status, rr.Body.String()) + } + return rr + } + rr := request(h.createJob, "", http.StatusCreated) + if err := json.Unmarshal(rr.Body.Bytes(), &job); err != nil { + t.Fatal(err) + } + id := fmt.Sprint(job.ID) + request(h.updateJob, id, http.StatusOK) + request(h.deleteJob, id, http.StatusNoContent) +} + +func TestSourceListHTTPDatabaseErrors(t *testing.T) { + s, err := store.New(filepath.Join(t.TempDir(), "sync.db")) + if err != nil { + t.Fatal(err) + } + if err := s.Close(); err != nil { + t.Fatal(err) + } + h := &Handler{store: s} + for _, fn := range []http.HandlerFunc{h.listSourceLists, h.getSourceList, h.createSourceList, h.updateSourceList, h.deleteSourceList} { + req := httptest.NewRequest(http.MethodPost, "/api/source-lists/1", strings.NewReader(`{"name":"list","hostnames":"192.0.2.1"}`)) + req.SetPathValue("id", "1") + req = req.WithContext(context.WithValue(req.Context(), principalContextKey, &auth.Principal{IsAdmin: true})) + rr := httptest.NewRecorder() + fn(rr, req) + if rr.Code != http.StatusInternalServerError { + t.Fatalf("closed DB status=%d body=%s", rr.Code, rr.Body.String()) + } + } +} + +func TestJobSourceHTTPDatabaseErrors(t *testing.T) { + s, err := store.New(filepath.Join(t.TempDir(), "sync.db")) + if err != nil { + t.Fatal(err) + } + if err := s.Close(); err != nil { + t.Fatal(err) + } + h := &Handler{store: s} + for _, tc := range []struct { + name string + fn http.HandlerFunc + }{ + {"create", h.createJob}, + {"update", h.updateJob}, + {"preview", h.resolveHostnames}, + } { + for _, entries := range []string{`"hostnames":"192.0.2.1"`, `"included_list_ids":[1]`} { + t.Run(tc.name+"/"+entries, func(t *testing.T) { + body := `{"name":"job","instance_id":1,"target_list_id":"target",` + entries + `}` + req := httptest.NewRequest(http.MethodPost, "/api/jobs/1", strings.NewReader(body)) + req.SetPathValue("id", "1") + req = req.WithContext(context.WithValue(req.Context(), principalContextKey, &auth.Principal{IsAdmin: true})) + rr := httptest.NewRecorder() + tc.fn(rr, req) + if rr.Code != http.StatusInternalServerError { + t.Fatalf("closed DB status=%d want=500 body=%s", rr.Code, rr.Body.String()) + } + }) + } + } +} + +func TestSourceErrorClassificationUsesType(t *testing.T) { + for _, tc := range []struct { + name string + err error + status int + }{ + {"wrapped validation", fmt.Errorf("nested source: %w", &store.SourceValidationError{Message: "invalid reference"}), http.StatusBadRequest}, + {"storage", errors.New("database is closed"), http.StatusInternalServerError}, + {"validation-like storage text", errors.New("source list ID 1 does not exist"), http.StatusInternalServerError}, + } { + t.Run(tc.name, func(t *testing.T) { + for _, write := range []func(http.ResponseWriter, error){writeSourceEntriesError, writeSourceListError} { + rr := httptest.NewRecorder() + write(rr, tc.err) + if rr.Code != tc.status { + t.Fatalf("status=%d want=%d body=%s", rr.Code, tc.status, rr.Body.String()) + } + } + }) + } +} diff --git a/internal/web/handler_ui_test.go b/internal/web/handler_ui_test.go index 25e42f9..6f51c93 100644 --- a/internal/web/handler_ui_test.go +++ b/internal/web/handler_ui_test.go @@ -1,13 +1,38 @@ package web import ( + "bytes" + "html/template" "net/http" "net/http/httptest" + "os" "strings" "testing" "testing/fstest" + + "github.com/mstrhakr/network-list-sync/internal/auth" ) +func TestRealTemplatesRenderReusableSources(t *testing.T) { + tmpl := template.Must(template.New("index").ParseFS(os.DirFS("../../ui"), "templates/*.gohtml", "templates/partials/*.gohtml")) + var rendered bytes.Buffer + principal := auth.Principal{IsAdmin: true, Username: "admin", AuthProvider: auth.ProviderLocal} + if err := tmpl.ExecuteTemplate(&rendered, "index", map[string]any{"Principal": principal}); err != nil { + t.Fatalf("execute real templates: %v", err) + } + for _, marker := range []string{ + `id="sourceListsModal"`, `id="jobSourceLists"`, `id="nestedSourceLists"`, + `/static/js/source-lists.js`, `RFC1918 Preset`, + } { + if !strings.Contains(rendered.String(), marker) { + t.Errorf("rendered UI missing %q", marker) + } + } + if strings.Contains(rendered.String(), `id="hostnames" rows="8" required`) { + t.Error("inline inputs must be optional for includes-only jobs") + } +} + func TestNewHandlerRendersTemplateIndex(t *testing.T) { t.Parallel() diff --git a/ui/static/css/app.css b/ui/static/css/app.css index 485f835..bcbe69f 100644 --- a/ui/static/css/app.css +++ b/ui/static/css/app.css @@ -24,6 +24,37 @@ * { margin: 0; padding: 0; box-sizing: border-box; } +.source-picker { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(180px, 1fr)); + gap: 0.5rem; +} + +.source-choice, +.form-group .source-choice { + display: flex; + align-items: flex-start; + gap: 0.6rem; + padding: 0.7rem; + border: 1px solid var(--border); + border-radius: var(--radius-sm); + background: var(--panel-bg); + cursor: pointer; + overflow-wrap: anywhere; +} + +.source-choice input { width: auto; margin-top: 0.25rem; accent-color: var(--primary); } +.source-choice small { display: block; color: var(--text-secondary); font-weight: normal; } +.source-choice:has(input:checked) { border-color: var(--primary); } +.source-manager-actions { display: flex; flex-wrap: wrap; gap: 0.5rem; margin: 0.75rem 0; } +.source-list-cards { display: grid; gap: 0.6rem; margin: 1rem 0; } +.source-list-card { padding: 0.9rem; background: var(--panel-bg); border: 1px solid var(--border); border-radius: var(--radius-sm); } +.source-entry-summary { white-space: pre-wrap; overflow-wrap: anywhere; max-height: 8rem; overflow: auto; margin-top: 0.4rem; color: var(--text-secondary); font-size: 0.8rem; } +#sourceListsModal { z-index: 110; } +#sourceListsModal > .modal-content > .hint, +#sourceListsModal > .modal-content > .source-manager-actions, +#sourceListsContent { margin-left: 1.5rem; margin-right: 1.5rem; } + body { font-family: 'Manrope', 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif; background: diff --git a/ui/static/js/app.js b/ui/static/js/app.js index 2dfb2ad..d6fb2a5 100644 --- a/ui/static/js/app.js +++ b/ui/static/js/app.js @@ -967,6 +967,7 @@ async function saveJob(event) { target_list_id: primaryNetworkListID, targets: targets, hostnames: document.getElementById('hostnames').value, + included_list_ids: selectedSourceIDs('jobSourceLists'), schedule: getScheduleValueFromForm(), observed_ip_ttl_hours: observedTTLHours, collapse_cidrs: document.getElementById('collapseCIDRs').checked, @@ -977,6 +978,10 @@ async function saveJob(event) { showToast('Add at least one endpoint target', 'error'); return; } + if (!data.hostnames.trim() && !data.included_list_ids.length) { + showToast('Add entries or select a reusable source', 'error'); + return; + } if (selectedSchedulePreset === 'custom' && !data.schedule) { showToast('Custom cron schedule cannot be blank', 'error'); return; @@ -1116,8 +1121,9 @@ function renderNetworkList(networkList) { async function previewResolve() { if (!isAdminUser) return; const hostnames = document.getElementById('hostnames').value; + const includedListIDs = selectedSourceIDs('jobSourceLists'); const preview = document.getElementById('resolvePreview'); - if (!hostnames.trim()) { + if (!hostnames.trim() && !includedListIDs.length) { preview.classList.add('hidden'); return; } @@ -1129,7 +1135,7 @@ async function previewResolve() { const resp = await fetch(API + '/resolve', { method: 'POST', headers: {'Content-Type': 'application/json'}, - body: JSON.stringify({ hostnames: hostnames }), + body: JSON.stringify({ hostnames: hostnames, included_list_ids: includedListIDs }), }); const data = await resp.json(); if (resp.ok) { @@ -1430,7 +1436,7 @@ function showJobModal(job) { const modal = document.getElementById('jobModal'); const title = document.getElementById('jobModalTitle'); - loadControllers().then(function() { + Promise.all([loadControllers(), loadSourceLists()]).then(function() { populateControllerDropdown(job ? job.instance_id : ''); document.getElementById('additionalTargetsRows').innerHTML = ''; @@ -1471,10 +1477,13 @@ function showJobModal(job) { document.getElementById('networkListId').innerHTML = ''; } + renderSourcePicker('jobSourceLists', job ? job.included_list_ids : []); onSchedulePresetChange(); onObservedIpRetentionToggle(); document.getElementById('resolvePreview').classList.add('hidden'); modal.classList.remove('hidden'); + }).catch(function(err) { + showToast('Cannot load job sources: ' + err.message, 'error'); }); } diff --git a/ui/static/js/source-lists.js b/ui/static/js/source-lists.js new file mode 100644 index 0000000..a7eb026 --- /dev/null +++ b/ui/static/js/source-lists.js @@ -0,0 +1,159 @@ +let sourceLists = []; + +async function loadSourceLists() { + const response = await fetch(API + '/source-lists'); + if (!response.ok) throw new Error('Failed to load reusable sources'); + sourceLists = await response.json(); +} + +function selectedSourceIDs(containerID) { + return Array.from(document.getElementById(containerID).querySelectorAll('input:checked')) + .map(function(input) { return Number(input.value); }); +} + +function sourceReaches(sourceID, targetID, visited) { + if (sourceID === targetID) return true; + visited = visited || new Set(); + if (visited.has(sourceID)) return false; + visited.add(sourceID); + const source = sourceLists.find(function(item) { return item.id === sourceID; }); + return !!source && (source.included_list_ids || []).some(function(id) { + return sourceReaches(id, targetID, visited); + }); +} + +function renderSourcePicker(containerID, selectedIDs, editingID) { + const selected = new Set(selectedIDs || []); + const container = document.getElementById(containerID); + const available = sourceLists.filter(function(source) { + return !editingID || !sourceReaches(source.id, editingID); + }); + container.innerHTML = available.map(function(source) { + return ''; + }).join(''); + // Never silently drop references if another admin removes a source mid-edit. + selected.forEach(function(id) { + if (!sourceLists.some(function(source) { return source.id === id; })) { + const label = document.createElement('label'); + label.className = 'source-choice text-error'; + const input = document.createElement('input'); + input.type = 'checkbox'; + input.value = id; + input.checked = true; + label.append(input, document.createTextNode(' Missing source #' + id + ' — deselect to remove')); + container.appendChild(label); + } + }); + if (!container.children.length) { + container.innerHTML = '

No sources available. Create a source or use the RFC1918 preset.

'; + } +} + +async function showSourceListsModal() { + if (!ensureAdminAction()) return; + try { + await loadSourceLists(); + renderSourceListCards(); + cancelSourceListEdit(); + document.getElementById('sourceListsModal').classList.remove('hidden'); + } catch (err) { + showToast(err.message, 'error'); + } +} + +function hideSourceListsModal() { + document.getElementById('sourceListsModal').classList.add('hidden'); +} + +function renderSourceListCards() { + document.getElementById('sourceListsContent').innerHTML = sourceLists.length ? sourceLists.map(function(source) { + const nestedNames = (source.included_list_ids || []).map(function(id) { + const child = sourceLists.find(function(item) { return item.id === id; }); + return child ? child.name : 'Missing #' + id; + }); + return '
' + escapeHtml(source.name) + '' + + (nestedNames.length ? '

Includes: ' + escapeHtml(nestedNames.join(', ')) + '

' : '') + + '
' + escapeHtml(source.hostnames || '(Nested sources only)') + '
' + + '
' + + '
'; + }).join('') : '

No reusable sources yet. Start with RFC1918 or create your own.

'; +} + +function editSourceList(id, preset) { + if (!ensureAdminAction()) return; + const source = sourceLists.find(function(item) { return item.id === id; }); + document.getElementById('sourceListId').value = source ? source.id : ''; + document.getElementById('sourceListName').value = source ? source.name : (preset ? 'RFC1918' : ''); + document.getElementById('sourceListHostnames').value = source ? source.hostnames : + (preset ? '10.0.0.0/8\n172.16.0.0/12\n192.168.0.0/16' : ''); + document.getElementById('sourceEditorTitle').textContent = source ? 'Edit Source' : 'New Source'; + renderSourcePicker('nestedSourceLists', source ? source.included_list_ids : [], source ? source.id : null); + document.getElementById('sourceListForm').classList.remove('hidden'); + document.getElementById('sourceListName').focus(); +} + +function cancelSourceListEdit() { + document.getElementById('sourceListForm').classList.add('hidden'); + document.getElementById('sourceListForm').reset(); +} + +async function refreshSourceListUI() { + const jobSelected = selectedSourceIDs('jobSourceLists'); + await loadSourceLists(); + renderSourceListCards(); + renderSourcePicker('jobSourceLists', jobSelected); + document.getElementById('resolvePreview').classList.add('hidden'); +} + +async function saveSourceList(event) { + event.preventDefault(); + if (!ensureAdminAction()) return; + const id = document.getElementById('sourceListId').value; + const data = { + name: document.getElementById('sourceListName').value.trim(), + hostnames: document.getElementById('sourceListHostnames').value, + included_list_ids: selectedSourceIDs('nestedSourceLists'), + }; + if (!data.hostnames.trim() && !data.included_list_ids.length) { + showToast('Add entries or include another source', 'error'); + return; + } + try { + const response = await fetch(API + '/source-lists' + (id ? '/' + id : ''), { + method: id ? 'PUT' : 'POST', + headers: {'Content-Type': 'application/json'}, + body: JSON.stringify(data), + }); + if (!response.ok) { + const error = await response.json(); + throw new Error(error.error || 'Save failed'); + } + cancelSourceListEdit(); + await refreshSourceListUI(); + showToast('Source saved. Dependent jobs use changes on their next run.', 'success'); + } catch (err) { + showToast(err.message, 'error'); + } +} + +async function deleteSourceList(id) { + if (!ensureAdminAction()) return; + if (!confirm('Delete this reusable source? Sources used by jobs or other sources cannot be deleted.')) return; + try { + const response = await fetch(API + '/source-lists/' + id, { method: 'DELETE' }); + if (!response.ok) { + const error = await response.json(); + throw new Error(error.error || 'Delete failed'); + } + if (Number(document.getElementById('sourceListId').value) === id) cancelSourceListEdit(); + await refreshSourceListUI(); + if (!document.getElementById('sourceListForm').classList.contains('hidden')) { + renderSourcePicker('nestedSourceLists', selectedSourceIDs('nestedSourceLists'), Number(document.getElementById('sourceListId').value)); + } + showToast('Source deleted', 'success'); + } catch (err) { + showToast(err.message, 'error'); + } +} \ No newline at end of file diff --git a/ui/static/js/source-lists.test.cjs b/ui/static/js/source-lists.test.cjs new file mode 100644 index 0000000..5f596b5 --- /dev/null +++ b/ui/static/js/source-lists.test.cjs @@ -0,0 +1,125 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const vm = require('node:vm'); +const path = require('node:path'); + +function harness() { + const elements = new Map(); + const document = { + getElementById(id) { + if (!elements.has(id)) { + const hidden = new Set(['hidden']); + elements.set(id, { + value: '', html: '', textContent: '', children: [], inputs: [], + get innerHTML() { return this.html; }, + set innerHTML(value) { + this.html = value; + this.children = Array.from(value.matchAll(/ ({})); + }, + classList: { + add: (name) => hidden.add(name), remove: (name) => hidden.delete(name), + contains: (name) => hidden.has(name), + }, + querySelectorAll() { return this.inputs.filter((input) => input.checked); }, + appendChild(child) { this.children.push(child); }, focus() {}, reset() {}, + }); + } + return elements.get(id); + }, + createElement() { return { append(...children) { this.children = children; } }; }, + createTextNode(text) { return text; }, + }; + const calls = []; + const context = vm.createContext({ + document, API: '/api', ensureAdminAction: () => true, + escapeHtml: (value) => String(value).replaceAll('&', '&').replaceAll('<', '<').replaceAll('>', '>').replaceAll('"', '"'), + showToast: (message, type) => calls.push({message, type}), + fetch: async () => ({ok: true, json: async () => []}), confirm: () => true, + }); + vm.runInContext(fs.readFileSync(path.join(__dirname, 'source-lists.js'), 'utf8'), context); + return {context, document, calls, run: (code) => vm.runInContext(code, context)}; +} + +test('picker excludes self and indirect ancestors, retains descendants', () => { + const h = harness(); + h.run(`sourceLists = [ + {id:1,name:'RFC1918',included_list_ids:[]}, + {id:2,name:'API',included_list_ids:[1]}, + {id:3,name:'Combined',included_list_ids:[2]}, + {id:4,name:'Other',included_list_ids:[]} + ]; renderSourcePicker('nestedSourceLists', [1], 2);`); + const html = h.document.getElementById('nestedSourceLists').innerHTML; + assert.match(html, /value="1" checked/); + assert.match(html, /value="4"/); + assert.doesNotMatch(html, /value="[23]"/); +}); + +test('RFC1918 preset supplies exactly three private IPv4 ranges', () => { + const h = harness(); + h.run('editSourceList(null, true)'); + assert.equal(h.document.getElementById('sourceListName').value, 'RFC1918'); + assert.equal(h.document.getElementById('sourceListHostnames').value, '10.0.0.0/8\n172.16.0.0/12\n192.168.0.0/16'); + assert.equal(h.document.getElementById('sourceListForm').classList.contains('hidden'), false); +}); + +test('source names and entries are escaped in rendered cards and picker', () => { + const h = harness(); + h.run(`sourceLists = [{id:1,name:'',hostnames:'',included_list_ids:[]}]; + renderSourceListCards(); renderSourcePicker('jobSourceLists', []);`); + assert.match(h.document.getElementById('sourceListsContent').innerHTML, /<script>/); + assert.doesNotMatch(h.document.getElementById('sourceListsContent').innerHTML, / { + const h = harness(); + h.run('renderSourcePicker("jobSourceLists", [99])'); + const missing = h.document.getElementById('jobSourceLists').children[0]; + assert.equal(missing.children[0].value, 99); + assert.equal(missing.children[0].checked, true); +}); + +test('source save supports nested-only input and sends numeric IDs', async () => { + const h = harness(); + h.document.getElementById('sourceListName').value = 'Combined'; + h.document.getElementById('nestedSourceLists').inputs = [{value: '1', checked: true}]; + let sent; + h.context.fetch = async (url, options) => { + if (options) sent = {url, ...JSON.parse(options.body)}; + return {ok: true, json: async () => [{id: 1, name: 'RFC1918', included_list_ids: []}]}; + }; + await h.run('saveSourceList({preventDefault(){}})'); + assert.deepEqual(sent, {url: '/api/source-lists', name: 'Combined', hostnames: '', included_list_ids: [1]}); + assert.equal(h.calls.at(-1).type, 'success'); +}); + +test('empty sources fail validation and API errors preserve editor input', async () => { + const h = harness(); + await h.run('saveSourceList({preventDefault(){}})'); + assert.equal(h.calls.at(-1).message, 'Add entries or include another source'); + h.document.getElementById('sourceListName').value = 'Office'; + h.document.getElementById('sourceListHostnames').value = '10.0.0.0/8'; + h.context.fetch = async () => ({ok: false, json: async () => ({error: 'source list cycle detected'})}); + await h.run('saveSourceList({preventDefault(){}})'); + assert.equal(h.calls.at(-1).message, 'source list cycle detected'); + assert.equal(h.document.getElementById('sourceListHostnames').value, '10.0.0.0/8'); +}); + +test('job preview resolves selected sources even without inline entries', async () => { + const h = harness(); + const app = fs.readFileSync(path.join(__dirname, 'app.js'), 'utf8'); + const previewFunction = app.slice(app.indexOf('async function previewResolve()'), app.indexOf('function renderJobCards(')); + h.context.isAdminUser = true; + h.context.console = console; + vm.runInContext(previewFunction, h.context); + h.document.getElementById('jobSourceLists').inputs = [{value: '7', checked: true}]; + let payload; + h.context.fetch = async (_url, options) => { + payload = JSON.parse(options.body); + return {ok: true, json: async () => [{ip: '10.0.0.0/8', hostname: '10.0.0.0/8'}]}; + }; + await h.run('previewResolve()'); + assert.deepEqual(payload, {hostnames: '', included_list_ids: [7]}); + assert.match(h.document.getElementById('resolvePreview').innerHTML, /10\.0\.0\.0\/8/); +}); \ No newline at end of file diff --git a/ui/templates/index.gohtml b/ui/templates/index.gohtml index 7e0ee9d..b9e2852 100644 --- a/ui/templates/index.gohtml +++ b/ui/templates/index.gohtml @@ -20,6 +20,7 @@ {{template "page-shell" .}} {{template "job-modal" .}} + {{template "source-lists-modal" .}} {{template "controller-modal" .}} {{template "dns-modal" .}} {{template "users-modal" .}} @@ -30,6 +31,7 @@ {{template "overlay-menus" .}} + {{end}} \ No newline at end of file diff --git a/ui/templates/partials/job_modal.gohtml b/ui/templates/partials/job_modal.gohtml index abccf62..10ecf05 100644 --- a/ui/templates/partials/job_modal.gohtml +++ b/ui/templates/partials/job_modal.gohtml @@ -32,7 +32,14 @@
- + + Optional when reusable sources are selected below. +
+
+ +
+ + Sources can include other sources. Current entries are expanded on every run, then deduplicated.
diff --git a/ui/templates/partials/page_shell.gohtml b/ui/templates/partials/page_shell.gohtml index 485f2b3..68a3897 100644 --- a/ui/templates/partials/page_shell.gohtml +++ b/ui/templates/partials/page_shell.gohtml @@ -29,6 +29,7 @@
{{if and .Principal .Principal.IsAdmin}} + {{end}}
diff --git a/ui/templates/partials/source_lists_modal.gohtml b/ui/templates/partials/source_lists_modal.gohtml new file mode 100644 index 0000000..5ce4891 --- /dev/null +++ b/ui/templates/partials/source_lists_modal.gohtml @@ -0,0 +1,44 @@ +{{define "source-lists-modal"}} + +{{end}} \ No newline at end of file