diff --git a/README.md b/README.md index 174ba0d..b56abc9 100644 --- a/README.md +++ b/README.md @@ -190,6 +190,12 @@ https://www.cloudflare.com/ips-v4 3. Use exact release tags in production. 4. Keep endpoint credentials scoped to minimum required permissions. +### List optimization and run history + +Each sync removes duplicate entries and entries already covered by another exact IP/CIDR range. Jobs can optionally collapse fully covered IPv4 ranges into CIDRs. Collapse preserves the exact address set and only creates `/24` or larger networks; the option is off by default. Existing smaller CIDRs are retained as provided. + +Run history reports input/unique counts, duplicates and covered entries removed, CIDRs created, and total entries saved. When a run updates a target, its history includes an on-demand view of the exact items sent to that target. + ## API Endpoints All `/api/*` endpoints require an authenticated session. @@ -218,6 +224,7 @@ All `/api/*` endpoints require an authenticated session. | GET | /api/jobs/{id}/target-list | Get primary or selected target list state | | POST | /api/jobs/{id}/run | Trigger immediate run | | GET | /api/jobs/{id}/logs | Get job run history | +| GET | /api/jobs/{id}/logs/{logID}/targets/{targetID} | Get exact target items sent during a run | ### DNS And Health diff --git a/internal/store/store.go b/internal/store/store.go index d769b44..e7fc7e2 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -2,6 +2,7 @@ package store import ( "database/sql" + "encoding/json" "fmt" "strings" "time" @@ -34,6 +35,7 @@ type SyncJob struct { Hostnames string `json:"hostnames"` Schedule string `json:"schedule"` ObservedIPTTLHours int `json:"observed_ip_ttl_hours"` + CollapseCIDRs bool `json:"collapse_cidrs"` Enabled bool `json:"enabled"` LastRunAt *string `json:"last_run_at"` LastResult *string `json:"last_result"` @@ -54,14 +56,26 @@ type JobTarget struct { // RunLog represents a single execution record for a sync job. type RunLog struct { - ID int64 `json:"id"` - JobID int64 `json:"job_id"` - StartedAt string `json:"started_at"` - FinishedAt *string `json:"finished_at"` - Status string `json:"status"` - Message string `json:"message"` - ChangesMade int `json:"changes_made"` - Details string `json:"details"` + ID int64 `json:"id"` + JobID int64 `json:"job_id"` + StartedAt string `json:"started_at"` + FinishedAt *string `json:"finished_at"` + Status string `json:"status"` + Message string `json:"message"` + ChangesMade int `json:"changes_made"` + Details string `json:"details"` + Stats json.RawMessage `json:"stats,omitempty"` + Targets []RunTargetSnapshot `json:"targets,omitempty"` +} + +// RunTargetSnapshot stores the exact list sent to a target for a run. +type RunTargetSnapshot struct { + ID int64 `json:"id"` + Label string `json:"label"` + Name string `json:"name,omitempty"` + Type string `json:"type,omitempty"` + EntryCount int `json:"entry_count"` + Items json.RawMessage `json:"items,omitempty"` } // DNSServer represents a custom DNS resolver endpoint. @@ -154,6 +168,7 @@ func (s *Store) migrate() error { message TEXT NOT NULL DEFAULT '', changes_made INTEGER NOT NULL DEFAULT 0, details TEXT NOT NULL DEFAULT '', + stats_json TEXT NOT NULL DEFAULT '', FOREIGN KEY (job_id) REFERENCES sync_jobs(id) ON DELETE CASCADE ); `) @@ -161,6 +176,8 @@ func (s *Store) migrate() error { return err } _, _ = s.db.Exec(`ALTER TABLE sync_jobs ADD COLUMN observed_ip_ttl_hours INTEGER NOT NULL DEFAULT 168`) + _, _ = s.db.Exec(`ALTER TABLE sync_jobs ADD COLUMN collapse_cidrs INTEGER NOT NULL DEFAULT 0`) + _, _ = s.db.Exec(`ALTER TABLE run_logs ADD COLUMN stats_json TEXT NOT NULL DEFAULT ''`) _, _ = s.db.Exec(`ALTER TABLE controllers ADD COLUMN provider TEXT NOT NULL DEFAULT 'unifi'`) // Add skip_tls_verify to existing databases that predate this column. _, _ = s.db.Exec(`ALTER TABLE controllers ADD COLUMN skip_tls_verify INTEGER NOT NULL DEFAULT 0`) @@ -194,6 +211,19 @@ func (s *Store) migrate() error { PRIMARY KEY (job_id, ip), FOREIGN KEY (job_id) REFERENCES sync_jobs(id) ON DELETE CASCADE )`) + _, _ = s.db.Exec(`CREATE TABLE IF NOT EXISTS run_log_targets ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + run_log_id INTEGER NOT NULL, + label TEXT NOT NULL, + list_name TEXT NOT NULL DEFAULT '', + list_type TEXT NOT NULL DEFAULT '', + entry_count INTEGER NOT NULL, + items_json TEXT NOT NULL, + FOREIGN KEY (run_log_id) REFERENCES run_logs(id) ON DELETE CASCADE + )`) + _, _ = s.db.Exec(`ALTER TABLE run_log_targets ADD COLUMN list_name TEXT NOT NULL DEFAULT ''`) + _, _ = s.db.Exec(`ALTER TABLE run_log_targets ADD COLUMN list_type TEXT NOT NULL DEFAULT ''`) + _, _ = s.db.Exec(`CREATE INDEX IF NOT EXISTS idx_run_log_targets_run_log_id ON run_log_targets(run_log_id)`) // App auth identities and server-side sessions. _, _ = s.db.Exec(`CREATE TABLE IF NOT EXISTS app_users ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -335,7 +365,7 @@ func (s *Store) DeleteController(id int64) error { func (s *Store) ListJobs() ([]SyncJob, error) { rows, err := s.db.Query(` SELECT j.id, j.name, j.controller_id, j.network_list_id, - j.hostnames, j.schedule, j.observed_ip_ttl_hours, j.enabled, + j.hostnames, j.schedule, j.observed_ip_ttl_hours, j.collapse_cidrs, j.enabled, j.last_run_at, j.last_result, j.created_at, j.updated_at, COALESCE(c.name, '') FROM sync_jobs j @@ -351,7 +381,7 @@ func (s *Store) ListJobs() ([]SyncJob, error) { var j SyncJob var enabled int if err := rows.Scan(&j.ID, &j.Name, &j.ControllerID, &j.NetworkListID, - &j.Hostnames, &j.Schedule, &j.ObservedIPTTLHours, &enabled, + &j.Hostnames, &j.Schedule, &j.ObservedIPTTLHours, &j.CollapseCIDRs, &enabled, &j.LastRunAt, &j.LastResult, &j.CreatedAt, &j.UpdatedAt, &j.ControllerName); err != nil { return nil, err @@ -377,14 +407,14 @@ func (s *Store) GetJob(id int64) (*SyncJob, error) { var enabled int err := s.db.QueryRow(` SELECT j.id, j.name, j.controller_id, j.network_list_id, - j.hostnames, j.schedule, j.observed_ip_ttl_hours, j.enabled, + j.hostnames, j.schedule, j.observed_ip_ttl_hours, j.collapse_cidrs, j.enabled, j.last_run_at, j.last_result, j.created_at, j.updated_at, COALESCE(c.name, '') FROM sync_jobs j LEFT JOIN controllers c ON c.id = j.controller_id WHERE j.id = ?`, id).Scan( &j.ID, &j.Name, &j.ControllerID, &j.NetworkListID, - &j.Hostnames, &j.Schedule, &j.ObservedIPTTLHours, &enabled, + &j.Hostnames, &j.Schedule, &j.ObservedIPTTLHours, &j.CollapseCIDRs, &enabled, &j.LastRunAt, &j.LastResult, &j.CreatedAt, &j.UpdatedAt, &j.ControllerName) if err != nil { @@ -414,10 +444,10 @@ func (s *Store) CreateJob(j *SyncJob) (int64, error) { primary := targets[0] result, err := s.db.Exec(` INSERT INTO sync_jobs (name, controller_id, network_list_id, - hostnames, schedule, observed_ip_ttl_hours, enabled, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, + hostnames, schedule, observed_ip_ttl_hours, collapse_cidrs, enabled, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, j.Name, primary.ControllerID, primary.NetworkListID, - j.Hostnames, j.Schedule, j.ObservedIPTTLHours, boolToInt(j.Enabled), + j.Hostnames, j.Schedule, j.ObservedIPTTLHours, boolToInt(j.CollapseCIDRs), boolToInt(j.Enabled), now, now) if err != nil { return 0, err @@ -442,10 +472,10 @@ func (s *Store) UpdateJob(j *SyncJob) error { primary := targets[0] _, err := s.db.Exec(` UPDATE sync_jobs SET name=?, controller_id=?, network_list_id=?, - hostnames=?, schedule=?, observed_ip_ttl_hours=?, enabled=?, updated_at=? + hostnames=?, schedule=?, observed_ip_ttl_hours=?, collapse_cidrs=?, enabled=?, updated_at=? WHERE id=?`, j.Name, primary.ControllerID, primary.NetworkListID, - j.Hostnames, j.Schedule, j.ObservedIPTTLHours, + j.Hostnames, j.Schedule, j.ObservedIPTTLHours, boolToInt(j.CollapseCIDRs), boolToInt(j.Enabled), now, j.ID) if err != nil { return err @@ -467,9 +497,9 @@ func (s *Store) UpdateJobLastRun(id int64, runAt string, result string) error { func (s *Store) CreateRunLog(l *RunLog) (int64, error) { result, err := s.db.Exec(` - INSERT INTO run_logs (job_id, started_at, status, message, changes_made, details) - VALUES (?, ?, ?, ?, ?, ?)`, - l.JobID, l.StartedAt, l.Status, l.Message, l.ChangesMade, l.Details) + INSERT INTO run_logs (job_id, started_at, status, message, changes_made, details, stats_json) + VALUES (?, ?, ?, ?, ?, ?, ?)`, + l.JobID, l.StartedAt, l.Status, l.Message, l.ChangesMade, l.Details, string(l.Stats)) if err != nil { return 0, err } @@ -477,16 +507,36 @@ func (s *Store) CreateRunLog(l *RunLog) (int64, error) { } func (s *Store) UpdateRunLog(l *RunLog) error { - _, err := s.db.Exec(` - UPDATE run_logs SET finished_at=?, status=?, message=?, changes_made=?, details=? + tx, err := s.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + if _, err := tx.Exec(` + UPDATE run_logs SET finished_at=?, status=?, message=?, changes_made=?, details=?, stats_json=? WHERE id=?`, - l.FinishedAt, l.Status, l.Message, l.ChangesMade, l.Details, l.ID) - return err + l.FinishedAt, l.Status, l.Message, l.ChangesMade, l.Details, string(l.Stats), l.ID); err != nil { + return err + } + if _, err := tx.Exec(`DELETE FROM run_log_targets WHERE run_log_id = ?`, l.ID); err != nil { + return err + } + for _, target := range l.Targets { + if len(target.Items) == 0 { + continue + } + if _, err := tx.Exec(` + INSERT INTO run_log_targets (run_log_id, label, list_name, list_type, entry_count, items_json) + VALUES (?, ?, ?, ?, ?, ?)`, l.ID, target.Label, target.Name, target.Type, target.EntryCount, string(target.Items)); err != nil { + return err + } + } + return tx.Commit() } func (s *Store) GetRunLogs(jobID int64, limit int) ([]RunLog, error) { rows, err := s.db.Query(` - SELECT id, job_id, started_at, finished_at, status, message, changes_made, details + SELECT id, job_id, started_at, finished_at, status, message, changes_made, details, stats_json FROM run_logs WHERE job_id = ? ORDER BY started_at DESC LIMIT ?`, jobID, limit) if err != nil { @@ -497,13 +547,65 @@ func (s *Store) GetRunLogs(jobID int64, limit int) ([]RunLog, error) { var logs []RunLog for rows.Next() { var l RunLog + var statsJSON string if err := rows.Scan(&l.ID, &l.JobID, &l.StartedAt, &l.FinishedAt, - &l.Status, &l.Message, &l.ChangesMade, &l.Details); err != nil { + &l.Status, &l.Message, &l.ChangesMade, &l.Details, &statsJSON); err != nil { return nil, err } + if statsJSON != "" { + l.Stats = json.RawMessage(statsJSON) + } logs = append(logs, l) } - return logs, rows.Err() + if err := rows.Err(); err != nil { + return nil, err + } + if err := rows.Close(); err != nil { + return nil, err + } + for i := range logs { + logs[i].Targets, err = s.ListRunTargetSnapshots(logs[i].ID) + if err != nil { + return nil, err + } + } + return logs, nil +} + +// ListRunTargetSnapshots returns target labels and item counts without loading stored payloads. +func (s *Store) ListRunTargetSnapshots(runLogID int64) ([]RunTargetSnapshot, error) { + rows, err := s.db.Query(` + SELECT id, label, list_name, list_type, entry_count FROM run_log_targets WHERE run_log_id = ? ORDER BY id`, runLogID) + if err != nil { + return nil, err + } + defer rows.Close() + var snapshots []RunTargetSnapshot + for rows.Next() { + var snapshot RunTargetSnapshot + if err := rows.Scan(&snapshot.ID, &snapshot.Label, &snapshot.Name, &snapshot.Type, &snapshot.EntryCount); err != nil { + return nil, err + } + snapshots = append(snapshots, snapshot) + } + return snapshots, rows.Err() +} + +// GetRunTargetSnapshot retrieves a stored list payload scoped to the owning job and run. +func (s *Store) GetRunTargetSnapshot(jobID, runLogID, targetID int64) (*RunTargetSnapshot, error) { + var snapshot RunTargetSnapshot + var itemsJSON string + err := s.db.QueryRow(` + SELECT t.id, t.label, t.list_name, t.list_type, t.entry_count, t.items_json + FROM run_log_targets t + JOIN run_logs r ON r.id = t.run_log_id + WHERE r.job_id = ? AND r.id = ? AND t.id = ?`, jobID, runLogID, targetID).Scan( + &snapshot.ID, &snapshot.Label, &snapshot.Name, &snapshot.Type, &snapshot.EntryCount, &itemsJSON) + if err != nil { + return nil, err + } + snapshot.Items = json.RawMessage(itemsJSON) + return &snapshot, nil } func boolToInt(b bool) int { diff --git a/internal/store/store_test.go b/internal/store/store_test.go index 0a75b39..20dc9ed 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -2,6 +2,7 @@ package store import ( "database/sql" + "encoding/json" "errors" "path/filepath" "testing" @@ -53,6 +54,7 @@ func TestStore_EndToEndCRUDAndObservedIPs(t *testing.T) { Hostnames: "example.com", Schedule: "*/5 * * * *", ObservedIPTTLHours: -1, + CollapseCIDRs: true, Enabled: true, }) if err != nil { @@ -66,10 +68,14 @@ func TestStore_EndToEndCRUDAndObservedIPs(t *testing.T) { if job.ObservedIPTTLHours != DefaultObservedIPTTLHours { t.Fatalf("ObservedIPTTLHours = %d, want %d", job.ObservedIPTTLHours, DefaultObservedIPTTLHours) } + if !job.CollapseCIDRs { + t.Fatal("CollapseCIDRs = false, want true") + } job.Schedule = "0 */6 * * *" job.Enabled = false job.ObservedIPTTLHours = 0 + job.CollapseCIDRs = false if err := s.UpdateJob(job); err != nil { t.Fatalf("UpdateJob() error = %v", err) } @@ -84,6 +90,9 @@ func TestStore_EndToEndCRUDAndObservedIPs(t *testing.T) { if jobs[0].ControllerName != "main-updated" { t.Fatalf("ControllerName = %q, want main-updated", jobs[0].ControllerName) } + if jobs[0].CollapseCIDRs { + t.Fatal("ListJobs() CollapseCIDRs = true, want false") + } if err := s.UpdateJobLastRun(jobID, "2026-01-01T00:00:00Z", "success: done"); err != nil { t.Fatalf("UpdateJobLastRun() error = %v", err) @@ -94,7 +103,22 @@ func TestStore_EndToEndCRUDAndObservedIPs(t *testing.T) { t.Fatalf("CreateRunLog() error = %v", err) } finished := "2026-01-01T00:01:00Z" - if err := s.UpdateRunLog(&RunLog{ID: logID, FinishedAt: &finished, Status: "success", Message: "ok", ChangesMade: 2, Details: "d"}); err != nil { + if err := s.UpdateRunLog(&RunLog{ + ID: logID, + FinishedAt: &finished, + Status: "success", + Message: "ok", + ChangesMade: 2, + Details: "d", + Stats: json.RawMessage(`{"entries_saved":1}`), + Targets: []RunTargetSnapshot{{ + Label: "unifi:nl-1 @ main", + Name: "blocked feeds", + Type: "IPV4_ADDRESSES", + EntryCount: 1, + Items: json.RawMessage(`[{"type":"SUBNET","value":"192.0.2.0/24"}]`), + }}, + }); err != nil { t.Fatalf("UpdateRunLog() error = %v", err) } logs, err := s.GetRunLogs(jobID, 10) @@ -104,6 +128,16 @@ func TestStore_EndToEndCRUDAndObservedIPs(t *testing.T) { if len(logs) != 1 || logs[0].Status != "success" { t.Fatalf("unexpected logs = %+v", logs) } + if !json.Valid(logs[0].Stats) || len(logs[0].Targets) != 1 || logs[0].Targets[0].EntryCount != 1 { + t.Fatalf("run log stats/targets = %+v", logs[0]) + } + snapshot, err := s.GetRunTargetSnapshot(jobID, logID, logs[0].Targets[0].ID) + if err != nil { + t.Fatalf("GetRunTargetSnapshot() error = %v", err) + } + if !json.Valid(snapshot.Items) || snapshot.Label != "unifi:nl-1 @ main" || snapshot.Name != "blocked feeds" || snapshot.Type != "IPV4_ADDRESSES" { + t.Fatalf("stored target snapshot = %+v", snapshot) + } dnsID, err := s.CreateDNSServer(&DNSServer{Name: "local", Address: "10.0.0.2:53", Enabled: false}) if err != nil { diff --git a/internal/syncer/optimize.go b/internal/syncer/optimize.go new file mode 100644 index 0000000..23c1fc1 --- /dev/null +++ b/internal/syncer/optimize.go @@ -0,0 +1,190 @@ +package syncer + +import ( + "fmt" + "net/netip" + "sort" +) + +const minCollapsedIPv4Prefix = 24 + +// OptimizationStats summarizes the effects of deduplication and CIDR collapse. +type OptimizationStats struct { + InputEntries int `json:"input_entries"` + UniqueEntries int `json:"unique_entries"` + DuplicateEntries int `json:"duplicate_entries"` + CoveredEntriesRemoved int `json:"covered_entries_removed"` + CIDRBlocks int `json:"cidr_blocks"` + CIDRsCreated int `json:"cidrs_created"` + OutputEntries int `json:"output_entries"` + EntriesSaved int `json:"entries_saved"` +} + +type ipv4Range struct { + start uint64 + end uint64 + value string +} + +// OptimizeIPv4Entries removes duplicates and entries fully covered by another +// entry. When collapseCIDRs is true, it also emits the smallest exact CIDR cover +// whose generated prefixes are /24 or larger; remaining addresses stay as IPs. +func OptimizeIPv4Entries(entries []string, collapseCIDRs bool) ([]string, OptimizationStats, error) { + stats := OptimizationStats{InputEntries: len(entries)} + unique := make(map[string]ipv4Range, len(entries)) + for _, entry := range entries { + rng, err := parseIPv4Range(entry) + if err != nil { + return nil, stats, err + } + if _, exists := unique[rng.value]; exists { + continue + } + unique[rng.value] = rng + } + stats.UniqueEntries = len(unique) + stats.DuplicateEntries = stats.InputEntries - stats.UniqueEntries + + ranges := make([]ipv4Range, 0, len(unique)) + for _, rng := range unique { + ranges = append(ranges, rng) + } + sort.Slice(ranges, func(i, j int) bool { + if ranges[i].start != ranges[j].start { + return ranges[i].start < ranges[j].start + } + if ranges[i].end != ranges[j].end { + return ranges[i].end > ranges[j].end + } + return ranges[i].value < ranges[j].value + }) + + // Remove a range only when the already-seen union fully covers it. + covered := make([]ipv4Range, 0, len(ranges)) + var coveredEnd uint64 + hasCoverage := false + for _, rng := range ranges { + if hasCoverage && rng.end <= coveredEnd { + stats.CoveredEntriesRemoved++ + continue + } + covered = append(covered, rng) + if !hasCoverage || rng.end > coveredEnd { + coveredEnd = rng.end + hasCoverage = true + } + } + + var output []string + if collapseCIDRs { + output = collapseIPv4Ranges(covered) + } else { + output = make([]string, 0, len(covered)) + for _, rng := range covered { + output = append(output, rng.value) + } + } + for _, entry := range output { + if _, err := netip.ParsePrefix(entry); err == nil { + stats.CIDRBlocks++ + if _, existed := unique[entry]; !existed { + stats.CIDRsCreated++ + } + } + } + stats.OutputEntries = len(output) + stats.EntriesSaved = stats.InputEntries - stats.OutputEntries + return output, stats, nil +} + +func parseIPv4Range(value string) (ipv4Range, error) { + if addr, err := netip.ParseAddr(value); err == nil && addr.Is4() { + n := ipv4Uint(addr) + return ipv4Range{start: n, end: n, value: addr.String()}, nil + } + prefix, err := netip.ParsePrefix(value) + if err != nil || !prefix.Addr().Is4() { + return ipv4Range{}, fmt.Errorf("invalid IPv4 address or CIDR %q", value) + } + prefix = prefix.Masked() + start := ipv4Uint(prefix.Addr()) + span := uint64(1) << uint(32-prefix.Bits()) + canonical := prefix.String() + if prefix.Bits() == 32 { + canonical = prefix.Addr().String() + } + return ipv4Range{start: start, end: start + span - 1, value: canonical}, nil +} + +func ipv4Uint(addr netip.Addr) uint64 { + b := addr.As4() + return uint64(b[0])<<24 | uint64(b[1])<<16 | uint64(b[2])<<8 | uint64(b[3]) +} + +func uintIPv4(value uint64) netip.Addr { + return netip.AddrFrom4([4]byte{ + byte(value >> 24), byte(value >> 16), byte(value >> 8), byte(value), + }) +} + +func collapseIPv4Ranges(ranges []ipv4Range) []string { + preservedSmallCIDRs := make(map[uint64][]ipv4Range) + for _, rng := range ranges { + prefix, err := netip.ParsePrefix(rng.value) + if err == nil && prefix.Bits() > minCollapsedIPv4Prefix { + preservedSmallCIDRs[rng.start] = append(preservedSmallCIDRs[rng.start], rng) + } + } + for start := range preservedSmallCIDRs { + sort.Slice(preservedSmallCIDRs[start], func(i, j int) bool { + return preservedSmallCIDRs[start][i].end > preservedSmallCIDRs[start][j].end + }) + } + + merged := make([]ipv4Range, 0, len(ranges)) + for _, rng := range ranges { + last := len(merged) - 1 + if last < 0 || rng.start > merged[last].end+1 { + merged = append(merged, ipv4Range{start: rng.start, end: rng.end}) + continue + } + if rng.end > merged[last].end { + merged[last].end = rng.end + } + } + + var output []string + for _, rng := range merged { + for current := rng.start; current <= rng.end; { + prefix := 32 + for candidate := 0; candidate <= minCollapsedIPv4Prefix; candidate++ { + blockSize := uint64(1) << uint(32-candidate) + if current%blockSize == 0 && blockSize <= rng.end-current+1 { + prefix = candidate + break + } + } + if prefix == 32 { + for _, existing := range preservedSmallCIDRs[current] { + if existing.end <= rng.end { + output = append(output, existing.value) + current = existing.end + 1 + prefix = 0 + break + } + } + if prefix == 0 { + continue + } + } + blockSize := uint64(1) << uint(32-prefix) + if prefix == 32 { + output = append(output, uintIPv4(current).String()) + } else { + output = append(output, netip.PrefixFrom(uintIPv4(current), prefix).String()) + } + current += blockSize + } + } + return output +} diff --git a/internal/syncer/optimize_test.go b/internal/syncer/optimize_test.go new file mode 100644 index 0000000..a23d46d --- /dev/null +++ b/internal/syncer/optimize_test.go @@ -0,0 +1,72 @@ +package syncer + +import ( + "reflect" + "strconv" + "testing" +) + +func TestOptimizeIPv4Entries_DeduplicatesAndRemovesCoveredEntries(t *testing.T) { + input := []string{ + "192.168.0.0/24", + "192.168.0.5", + "192.168.0.5", + "192.168.1.5", + "192.168.1.5/32", + } + + got, stats, err := OptimizeIPv4Entries(input, false) + if err != nil { + t.Fatalf("OptimizeIPv4Entries() error = %v", err) + } + want := []string{"192.168.0.0/24", "192.168.1.5"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("OptimizeIPv4Entries() = %v, want %v", got, want) + } + if stats.DuplicateEntries != 2 || stats.CoveredEntriesRemoved != 1 || stats.EntriesSaved != 3 { + t.Fatalf("unexpected stats: %+v", stats) + } +} + +func TestOptimizeIPv4Entries_CollapsesFull24ButNotPartial24(t *testing.T) { + input := make([]string, 0, 257) + for i := 0; i < 256; i++ { + input = append(input, "192.168.0."+strconv.Itoa(i)) + } + input = append(input, "192.168.1.50") + + got, stats, err := OptimizeIPv4Entries(input, true) + if err != nil { + t.Fatalf("OptimizeIPv4Entries() error = %v", err) + } + want := []string{"192.168.0.0/24", "192.168.1.50"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("OptimizeIPv4Entries() = %v, want %v", got, want) + } + if stats.CIDRBlocks != 1 || stats.OutputEntries != 2 || stats.EntriesSaved != 255 { + t.Fatalf("unexpected stats: %+v", stats) + } +} + +func TestOptimizeIPv4Entries_OnlyCollapsesExactCoverageAndMinimumPrefix(t *testing.T) { + input := []string{"10.0.0.0/23", "10.0.2.0/25", "10.0.2.128", "10.0.2.130"} + got, stats, err := OptimizeIPv4Entries(input, true) + if err != nil { + t.Fatalf("OptimizeIPv4Entries() error = %v", err) + } + want := []string{"10.0.0.0/23", "10.0.2.0/25", "10.0.2.128", "10.0.2.130"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("OptimizeIPv4Entries() = %v, want %v", got, want) + } + if stats.CIDRBlocks != 2 || stats.CoveredEntriesRemoved != 0 { + t.Fatalf("unexpected stats: %+v", stats) + } +} + +func TestOptimizeIPv4Entries_RejectsNonIPv4Entries(t *testing.T) { + for _, input := range [][]string{{"example.com"}, {"2001:db8::1"}, {"192.0.2.0/33"}} { + if _, _, err := OptimizeIPv4Entries(input, true); err == nil { + t.Errorf("OptimizeIPv4Entries(%q) expected error", input) + } + } +} \ No newline at end of file diff --git a/internal/syncer/resolver.go b/internal/syncer/resolver.go index 81989c1..cb87b61 100644 --- a/internal/syncer/resolver.go +++ b/internal/syncer/resolver.go @@ -41,6 +41,13 @@ type multiResolver struct { exchange dnsExchangeFunc } +// ResolutionStats describes source entries after DNS/list expansion but before deduplication. +type ResolutionStats struct { + ResolvedEntries int `json:"resolved_entries"` + UniqueEntries int `json:"unique_entries"` + DuplicateEntries int `json:"duplicate_entries"` +} + // newMultiResolverWithServers builds a resolver that queries exactly the provided servers. // Returns an error if the list is empty — callers must ensure at least one server is configured. func newMultiResolverWithServers(servers []string) (*multiResolver, error) { @@ -59,7 +66,18 @@ func newMultiResolverWithServers(servers []string) (*multiResolver, error) { // or IPv4 CIDR ranges into a de-duplicated set of IPv4 entries. // servers is the complete list of DNS server addresses to query; an empty list is an error. func ResolveHostnames(hostnamesText string, servers []string) (map[string]string, error) { + result, _, err := ResolveHostnamesWithStats(hostnamesText, servers) + return result, err +} + +// ResolveHostnamesWithStats resolves source entries and reports duplicate resolved entries. +func ResolveHostnamesWithStats(hostnamesText string, servers []string) (map[string]string, ResolutionStats, error) { result := make(map[string]string) + resolvedEntries := 0 + addEntry := func(entry, source string) { + resolvedEntries++ + addResolvedSource(result, entry, source) + } var resolver *multiResolver var errors []string externalCache := make(map[string][]string) @@ -101,12 +119,12 @@ func ResolveHostnames(hostnamesText string, servers []string) (map[string]string resolvedSource := sourceLabelForResolvedEntry(source, line) if ip, ok := normalizeIPv4Literal(line); ok { - addResolvedSource(result, ip, resolvedSource) + addEntry(ip, resolvedSource) return } if cidr, ok := normalizeIPv4CIDR(line); ok { - addResolvedSource(result, cidr, resolvedSource) + addEntry(cidr, resolvedSource) return } @@ -126,7 +144,7 @@ func ResolveHostnames(hostnamesText string, servers []string) (map[string]string } for _, ip := range ips { - addResolvedSource(result, ip, resolvedSource) + addEntry(ip, resolvedSource) } } @@ -134,15 +152,20 @@ func ResolveHostnames(hostnamesText string, servers []string) (map[string]string resolveLine(rawLine, "", 0) } + stats := ResolutionStats{ + ResolvedEntries: resolvedEntries, + UniqueEntries: len(result), + DuplicateEntries: resolvedEntries - len(result), + } if len(result) == 0 { errMsg := "no IPv4 addresses resolved from hostname list" if len(errors) > 0 { errMsg += ": " + strings.Join(errors, "; ") } - return nil, fmt.Errorf("%s", errMsg) + return nil, stats, fmt.Errorf("%s", errMsg) } - return result, nil + return result, stats, nil } func (r *multiResolver) ResolveIPv4(hostname string) ([]string, error) { diff --git a/internal/syncer/resolver_utils_test.go b/internal/syncer/resolver_utils_test.go index 84f114d..a0c6225 100644 --- a/internal/syncer/resolver_utils_test.go +++ b/internal/syncer/resolver_utils_test.go @@ -73,3 +73,16 @@ func TestResolverHelpers(t *testing.T) { t.Fatalf("bytesCompare equal = %d", got) } } + +func TestResolveHostnamesWithStats_CountsDuplicateResolvedEntries(t *testing.T) { + got, stats, err := ResolveHostnamesWithStats("192.0.2.1\n192.0.2.1\n192.0.2.0/24", []string{"1.1.1.1:53"}) + if err != nil { + t.Fatalf("ResolveHostnamesWithStats() error = %v", err) + } + if len(got) != 2 { + t.Fatalf("resolved entries = %v, want two unique entries", got) + } + if stats.ResolvedEntries != 3 || stats.UniqueEntries != 2 || stats.DuplicateEntries != 1 { + t.Fatalf("resolution stats = %+v", stats) + } +} diff --git a/internal/syncer/syncer.go b/internal/syncer/syncer.go index 418cc48..9f46d52 100644 --- a/internal/syncer/syncer.go +++ b/internal/syncer/syncer.go @@ -2,6 +2,7 @@ package syncer import ( "bytes" + "encoding/json" "fmt" "log" "sort" @@ -15,10 +16,12 @@ import ( // SyncResult captures the outcome of a single sync execution. type SyncResult struct { - Status string `json:"status"` - Message string `json:"message"` - ChangesMade int `json:"changes_made"` - Details string `json:"details"` + Status string `json:"status"` + Message string `json:"message"` + ChangesMade int `json:"changes_made"` + Details string `json:"details"` + Stats OptimizationStats `json:"stats"` + Targets []store.RunTargetSnapshot `json:"targets,omitempty"` } // Syncer executes DNS-to-IP-list sync operations across providers. @@ -63,6 +66,8 @@ func (s *Syncer) Run(db *store.Store, jobID int64) SyncResult { runLog.Message = result.Message runLog.ChangesMade = result.ChangesMade runLog.Details = result.Details + runLog.Stats, _ = json.Marshal(result.Stats) + runLog.Targets = result.Targets db.UpdateRunLog(runLog) db.UpdateJobLastRun(jobID, finished, result.Status+": "+result.Message) @@ -78,13 +83,14 @@ func (s *Syncer) execute(db *store.Store, job *store.SyncJob) SyncResult { if len(servers) == 0 { return SyncResult{Status: "error", Message: "no DNS servers configured: add at least one enabled DNS server"} } - hostIPs, err := ResolveHostnames(job.Hostnames, servers) + hostIPs, resolutionStats, err := ResolveHostnamesWithStats(job.Hostnames, servers) if err != nil { return SyncResult{Status: "error", Message: fmt.Sprintf("DNS resolution: %v", err)} } now := time.Now().UTC() retention := observedIPRetention(job) + inputEntries := resolutionStats.ResolvedEntries if retention > 0 { retentionCutoff := now.Add(-retention).Format(time.RFC3339) if err := db.UpsertObservedIPs(job.ID, hostIPs, now.Format(time.RFC3339)); err != nil { @@ -97,6 +103,7 @@ func (s *Syncer) execute(db *store.Store, job *store.SyncJob) SyncResult { if err != nil { return SyncResult{Status: "error", Message: fmt.Sprintf("load observed IPs: %v", err)} } + inputEntries += countObservedIPsNotCurrent(observedIPs, hostIPs) hostIPs = mergeResolvedIPs(observedIPs, hostIPs) } else { if err := db.DeleteObservedIPs(job.ID); err != nil { @@ -112,11 +119,18 @@ func (s *Syncer) execute(db *store.Store, job *store.SyncJob) SyncResult { targets = []store.JobTarget{{ControllerID: job.ControllerID, NetworkListID: job.NetworkListID}} } - newIPs := SortedIPs(hostIPs) + newIPs, optimizationStats, err := OptimizeIPv4Entries(SortedIPs(hostIPs), job.CollapseCIDRs) + if err != nil { + return SyncResult{Status: "error", Message: fmt.Sprintf("optimize IPv4 entries: %v", err)} + } + optimizationStats.InputEntries = inputEntries + optimizationStats.DuplicateEntries = inputEntries - optimizationStats.UniqueEntries + optimizationStats.EntriesSaved = inputEntries - len(newIPs) totalChanges := 0 succeeded := 0 failed := 0 var detailParts []string + var targetSnapshots []store.RunTargetSnapshot for _, target := range targets { ctrl, err := db.GetController(target.ControllerID) @@ -148,6 +162,18 @@ func (s *Syncer) execute(db *store.Store, job *store.SyncJob) SyncResult { succeeded++ totalChanges += result.changes detailParts = append(detailParts, fmt.Sprintf("[%s:%s @ %s]\n%s", providerLabel, target.NetworkListID, ctrl.Name, result.details)) + if result.sent && len(result.items) > 0 { + itemsJSON, marshalErr := json.Marshal(result.items) + if marshalErr == nil { + targetSnapshots = append(targetSnapshots, store.RunTargetSnapshot{ + Label: fmt.Sprintf("%s:%s @ %s", providerLabel, target.NetworkListID, ctrl.Name), + Name: result.listName, + Type: result.listType, + EntryCount: len(result.items), + Items: itemsJSON, + }) + } + } } status := "success" @@ -161,12 +187,28 @@ func (s *Syncer) execute(db *store.Store, job *store.SyncJob) SyncResult { Message: message, ChangesMade: totalChanges, Details: strings.Join(detailParts, "\n\n"), + Stats: optimizationStats, + Targets: targetSnapshots, } } +func countObservedIPsNotCurrent(observed, current map[string]string) int { + count := 0 + for ip := range observed { + if _, isCurrent := current[ip]; !isCurrent { + count++ + } + } + return count +} + type targetSyncResult struct { - changes int - details string + changes int + details string + listName string + listType string + sent bool + items []clients.TrafficMatchItem } func (s *Syncer) syncTarget(p clients.Provider, listID string, newIPs []string, hostIPs map[string]string) (targetSyncResult, error) { @@ -177,14 +219,34 @@ func (s *Syncer) syncTarget(p clients.Provider, listID string, newIPs []string, oldIPs := ExtractIPsFromItems(nl.Items) sort.Strings(oldIPs) added, removed, kept := DiffIPs(oldIPs, newIPs) - if len(added) == 0 && len(removed) == 0 { + oldCanonical := append([]string(nil), oldIPs...) + newCanonical := append([]string(nil), newIPs...) + sort.Strings(oldCanonical) + sort.Strings(newCanonical) + if equalStrings(oldCanonical, newCanonical) { return targetSyncResult{details: fmt.Sprintf("No changes needed (%d IPs match)", len(kept))}, nil } nl.Items = IPsToItems(newIPs) if err := p.UpdateNetworkList(nl); err != nil { return targetSyncResult{}, fmt.Errorf("update network list: %w", err) } - return targetSyncResult{changes: len(added) + len(removed), details: FormatDiff(added, removed, kept, hostIPs)}, nil + changes := len(added) + len(removed) + if changes == 0 { + changes = 1 // representation-only cleanup (duplicates or covered entries) + } + return targetSyncResult{changes: changes, details: FormatDiff(added, removed, kept, hostIPs), listName: nl.Name, listType: nl.Type, sent: true, items: nl.Items}, nil +} + +func equalStrings(left, right []string) bool { + if len(left) != len(right) { + return false + } + for i := range left { + if left[i] != right[i] { + return false + } + } + return true } // SortedIPs returns IPv4 addresses and IPv4 CIDRs from a host-IP map in a stable order. @@ -242,7 +304,11 @@ func DiffIPs(oldIPs, newIPs []string) (added, removed, kept []string) { func FormatDiff(added, removed, kept []string, hostIPs map[string]string) string { var b strings.Builder for _, ip := range added { - fmt.Fprintf(&b, "+ %s (%s)\n", ip, hostIPs[ip]) + source := hostIPs[ip] + if source == "" { + source = "aggregated entries" + } + fmt.Fprintf(&b, "+ %s (%s)\n", ip, source) } for _, ip := range removed { host := hostIPs[ip] @@ -252,7 +318,11 @@ func FormatDiff(added, removed, kept []string, hostIPs map[string]string) string fmt.Fprintf(&b, "- %s (%s)\n", ip, host) } for _, ip := range kept { - fmt.Fprintf(&b, " %s (%s)\n", ip, hostIPs[ip]) + source := hostIPs[ip] + if source == "" { + source = "aggregated entries" + } + fmt.Fprintf(&b, " %s (%s)\n", ip, source) } return b.String() } diff --git a/internal/syncer/syncer_test.go b/internal/syncer/syncer_test.go index 2c7f2eb..f382b59 100644 --- a/internal/syncer/syncer_test.go +++ b/internal/syncer/syncer_test.go @@ -5,9 +5,74 @@ import ( "testing" "time" + "github.com/mstrhakr/network-list-sync/internal/clients" "github.com/mstrhakr/network-list-sync/internal/store" ) +type testNetworkListProvider struct { + list *clients.NetworkList + updated bool +} + +func (p *testNetworkListProvider) ListNetworkLists() ([]clients.NetworkList, error) { + return []clients.NetworkList{*p.list}, nil +} + +func (p *testNetworkListProvider) GetNetworkList(string) (*clients.NetworkList, error) { + copy := *p.list + copy.Items = append([]clients.TrafficMatchItem(nil), p.list.Items...) + return ©, nil +} + +func (p *testNetworkListProvider) UpdateNetworkList(list *clients.NetworkList) error { + p.updated = true + p.list = list + return nil +} + +func TestSyncTargetWritesMixedIPAndCIDRItems(t *testing.T) { + provider := &testNetworkListProvider{list: &clients.NetworkList{ + ID: "list-1", + Name: "threat-feed", + Type: "IPV4_ADDRESSES", + Items: []clients.TrafficMatchItem{{Type: "IP_ADDRESS", Value: "192.0.2.1"}, {Type: "IP_ADDRESS", Value: "192.0.2.1"}}, + }} + + result, err := New().syncTarget(provider, "list-1", []string{"192.0.2.0/24", "198.51.100.10"}, nil) + if err != nil { + t.Fatalf("syncTarget() error = %v", err) + } + if !provider.updated { + t.Fatal("syncTarget() did not replace duplicate/old entries") + } + if len(result.items) != 2 || result.items[0].Type != "SUBNET" || result.items[1].Type != "IP_ADDRESS" { + t.Fatalf("sent items = %+v, want mixed subnet and individual IP", result.items) + } +} + +func TestSyncTargetRemovesDuplicateItemsFromExistingList(t *testing.T) { + provider := &testNetworkListProvider{list: &clients.NetworkList{ + ID: "list-1", + Name: "threat-feed", + Type: "IPV4_ADDRESSES", + Items: []clients.TrafficMatchItem{ + {Type: "IP_ADDRESS", Value: "192.0.2.1"}, + {Type: "IP_ADDRESS", Value: "192.0.2.1"}, + }, + }} + + result, err := New().syncTarget(provider, "list-1", []string{"192.0.2.1"}, map[string]string{"192.0.2.1": "feed"}) + if err != nil { + t.Fatalf("syncTarget() error = %v", err) + } + if !provider.updated || result.changes != 1 { + t.Fatalf("representation cleanup: updated=%v changes=%d", provider.updated, result.changes) + } + if len(result.items) != 1 || result.items[0].Value != "192.0.2.1" { + t.Fatalf("sent items = %+v, want one deduplicated IP", result.items) + } +} + func TestMergeResolvedIPs_PreservesObservedSuperset(t *testing.T) { observed := map[string]string{ "18.154.110.10": "integrations.ecimanufacturing.com", @@ -30,6 +95,33 @@ func TestMergeResolvedIPs_PreservesObservedSuperset(t *testing.T) { } } +func TestCountObservedIPsNotCurrent_ExcludesCurrentAnswers(t *testing.T) { + observed := map[string]string{ + "192.0.2.1": "current.example", + "192.0.2.2": "previous.example", + } + current := map[string]string{"192.0.2.1": "current.example"} + if got := countObservedIPsNotCurrent(observed, current); got != 1 { + t.Fatalf("countObservedIPsNotCurrent() = %d, want 1", got) + } +} + +func TestCurrentResolutionWithRetentionDoesNotInflateOptimizationStats(t *testing.T) { + current := map[string]string{"192.0.2.1": "feed.example"} + observed := map[string]string{"192.0.2.1": "feed.example"} + inputEntries := 1 + countObservedIPsNotCurrent(observed, current) + _, stats, err := OptimizeIPv4Entries(SortedIPs(current), false) + if err != nil { + t.Fatalf("OptimizeIPv4Entries() error = %v", err) + } + stats.InputEntries = inputEntries + stats.DuplicateEntries = inputEntries - stats.UniqueEntries + stats.EntriesSaved = inputEntries - stats.OutputEntries + if stats.InputEntries != 1 || stats.DuplicateEntries != 0 || stats.EntriesSaved != 0 { + t.Fatalf("current retained answer inflated stats: %+v", stats) + } +} + func TestObservedIPRetention_UsesJobSettingOrDefault(t *testing.T) { if got := observedIPRetention(nil); got != time.Duration(store.DefaultObservedIPTTLHours)*time.Hour { t.Fatalf("observedIPRetention(nil) = %v", got) diff --git a/internal/web/handler.go b/internal/web/handler.go index 9db1ee7..190caee 100644 --- a/internal/web/handler.go +++ b/internal/web/handler.go @@ -74,6 +74,7 @@ func NewHandler(s *store.Store, syn *syncer.Syncer, sched *scheduler.Scheduler, mux.HandleFunc("GET /api/jobs/{id}/target-list", h.getJobNetworkList) mux.HandleFunc("POST /api/jobs/{id}/run", h.runJob) mux.HandleFunc("GET /api/jobs/{id}/logs", h.getJobLogs) + mux.HandleFunc("GET /api/jobs/{id}/logs/{logID}/targets/{targetID}", h.getRunTargetSnapshot) mux.HandleFunc("POST /api/resolve", h.resolveHostnames) mux.HandleFunc("GET /api/health", h.health) mux.HandleFunc("GET /api/dns-servers", h.listDNSServers) @@ -639,6 +640,30 @@ func (h *Handler) getJobLogs(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusOK, logs) } +func (h *Handler) getRunTargetSnapshot(w http.ResponseWriter, r *http.Request) { + jobID, err := parseID(r) + if err != nil { + writeError(w, http.StatusBadRequest, "invalid job ID") + return + } + logID, err := strconv.ParseInt(r.PathValue("logID"), 10, 64) + if err != nil || logID <= 0 { + writeError(w, http.StatusBadRequest, "invalid run log ID") + return + } + targetID, err := strconv.ParseInt(r.PathValue("targetID"), 10, 64) + if err != nil || targetID <= 0 { + writeError(w, http.StatusBadRequest, "invalid target snapshot ID") + return + } + snapshot, err := h.store.GetRunTargetSnapshot(jobID, logID, targetID) + if err != nil { + writeError(w, http.StatusNotFound, "target snapshot not found") + return + } + writeJSON(w, http.StatusOK, snapshot) +} + func (h *Handler) resolveHostnames(w http.ResponseWriter, r *http.Request) { if !h.requireAdmin(w, r) { return diff --git a/internal/web/handler_helpers_test.go b/internal/web/handler_helpers_test.go index 7e9e3e0..5939360 100644 --- a/internal/web/handler_helpers_test.go +++ b/internal/web/handler_helpers_test.go @@ -135,3 +135,59 @@ func TestHealthEndpoint(t *testing.T) { t.Fatalf("expected unhealthy response, got %s", rr2.Body.String()) } } + +func TestGetRunTargetSnapshot(t *testing.T) { + s, err := store.New(filepath.Join(t.TempDir(), "sync.db")) + if err != nil { + t.Fatalf("store.New() error = %v", err) + } + defer s.Close() + + controllerID, err := s.CreateController(&store.Controller{Name: "test", URL: "http://example.test", APIKey: "key"}) + if err != nil { + t.Fatalf("CreateController() error = %v", err) + } + jobID, err := s.CreateJob(&store.SyncJob{Name: "job", ControllerID: controllerID, NetworkListID: "list", Hostnames: "192.0.2.1"}) + if err != nil { + t.Fatalf("CreateJob() error = %v", err) + } + logID, err := s.CreateRunLog(&store.RunLog{JobID: jobID, StartedAt: "2026-01-01T00:00:00Z", Status: "running"}) + if err != nil { + t.Fatalf("CreateRunLog() error = %v", err) + } + finished := "2026-01-01T00:01:00Z" + if err := s.UpdateRunLog(&store.RunLog{ + ID: logID, + FinishedAt: &finished, + Status: "success", + Targets: []store.RunTargetSnapshot{{ + Label: "unifi:list @ test", + EntryCount: 1, + Items: json.RawMessage(`[{"type":"IP_ADDRESS","value":"192.0.2.1"}]`), + }}, + }); err != nil { + t.Fatalf("UpdateRunLog() error = %v", err) + } + logs, err := s.GetRunLogs(jobID, 10) + if err != nil || len(logs) != 1 || len(logs[0].Targets) != 1 { + t.Fatalf("GetRunLogs() = %+v, error = %v", logs, err) + } + + h := &Handler{store: s} + request := httptest.NewRequest(http.MethodGet, "/api/jobs/1/logs/1/targets/1", nil) + request.SetPathValue("id", "1") + request.SetPathValue("logID", "1") + request.SetPathValue("targetID", "1") + response := httptest.NewRecorder() + h.getRunTargetSnapshot(response, request) + if response.Code != http.StatusOK || !strings.Contains(response.Body.String(), "192.0.2.1") { + t.Fatalf("snapshot response = %d %s", response.Code, response.Body.String()) + } + + request.SetPathValue("id", "999") + missingResponse := httptest.NewRecorder() + h.getRunTargetSnapshot(missingResponse, request) + if missingResponse.Code != http.StatusNotFound { + t.Fatalf("wrong-job snapshot status = %d, want 404", missingResponse.Code) + } +} diff --git a/ui/static/js/app.js b/ui/static/js/app.js index 8484310..2dfb2ad 100644 --- a/ui/static/js/app.js +++ b/ui/static/js/app.js @@ -969,6 +969,7 @@ async function saveJob(event) { hostnames: document.getElementById('hostnames').value, schedule: getScheduleValueFromForm(), observed_ip_ttl_hours: observedTTLHours, + collapse_cidrs: document.getElementById('collapseCIDRs').checked, enabled: scheduleEnabled, }; @@ -1360,23 +1361,60 @@ function renderLogs(logs) { content.innerHTML = '' + - '' + + '' + '' + logs.map(function(log) { const badgeClass = log.status === 'success' ? 'badge-success' : log.status === 'running' ? 'badge-warning' : 'badge-error'; + const stats = log.stats || {}; + const optimization = log.stats + ? 'Input entries: ' + (stats.input_entries || 0) + + '
Unique after dedupe: ' + (stats.unique_entries || 0) + + '
Duplicates removed: ' + (stats.duplicate_entries || 0) + + '
Covered entries: ' + (stats.covered_entries_removed || 0) + + '
New CIDRs: ' + (stats.cidrs_created || 0) + + '
Final entries: ' + (stats.output_entries || 0) + + '
Entries saved: ' + (stats.entries_saved || 0) + : '-'; + const targetButtons = (Array.isArray(log.targets) ? log.targets : []).map(function(target) { + return ''; + }).join('
'); return '' + '' + '' + '' + '' + + '' + ''; }).join('') + '
StartedStatusChangesMessageDetailsStartedStatusChangesMessageOptimizationDetails / Sent List
' + formatTime(log.started_at) + '' + escapeHtml(log.status) + '' + log.changes_made + '' + escapeHtml(log.message) + '' + optimization + '' + (log.details ? - '' - : '-') + + '' + : '') + (targetButtons ? (log.details ? '
' : '') + targetButtons : (!log.details ? '-' : '')) + '
'; } +async function viewRunTargetSnapshot(jobId, logId, targetId, button) { + button.disabled = true; + try { + const resp = await fetch(API + '/jobs/' + jobId + '/logs/' + logId + '/targets/' + targetId); + const snapshot = await resp.json(); + if (!resp.ok) throw new Error(snapshot.error || 'Failed to load sent list'); + const items = Array.isArray(snapshot.items) ? snapshot.items : []; + const output = document.createElement('pre'); + output.className = 'details-block'; + output.textContent = (snapshot.name ? snapshot.name + ' (' + snapshot.type + ')\n\n' : '') + items.map(function(item) { + const value = item.value || ((item.start || '') + (item.stop ? ' - ' + item.stop : '')); + return (item.type || 'UNKNOWN') + ' ' + value; + }).join('\n'); + button.parentElement.appendChild(output); + button.textContent = 'Sent list shown'; + } catch (err) { + button.disabled = false; + showToast(err.message, 'error'); + } +} + function toggleDetails(logId, btn) { const details = logDetailsMap.get(logId); if (!details) return; @@ -1405,6 +1443,7 @@ function showJobModal(job) { document.getElementById('schedulePreset').value = resolveSchedulePreset(job.schedule); document.getElementById('observedIpRetentionEnabled').checked = job.observed_ip_ttl_hours > 0; document.getElementById('observedIpTtlHours').value = job.observed_ip_ttl_hours > 0 ? job.observed_ip_ttl_hours : 168; + document.getElementById('collapseCIDRs').checked = !!job.collapse_cidrs; document.getElementById('enabled').checked = job.enabled; var targets = Array.isArray(job.targets) ? job.targets : []; var additional = []; @@ -1427,6 +1466,7 @@ function showJobModal(job) { document.getElementById('schedule').value = ''; document.getElementById('observedIpRetentionEnabled').checked = true; document.getElementById('observedIpTtlHours').value = 168; + document.getElementById('collapseCIDRs').checked = false; document.getElementById('enabled').checked = true; document.getElementById('networkListId').innerHTML = ''; } diff --git a/ui/templates/partials/job_modal.gohtml b/ui/templates/partials/job_modal.gohtml index 3a063e7..abccf62 100644 --- a/ui/templates/partials/job_modal.gohtml +++ b/ui/templates/partials/job_modal.gohtml @@ -43,6 +43,19 @@ Each row must choose an endpoint and a target list. Primary target uses the selectors above. +
+

List Optimization

+
+
+ + +
+ Always removes duplicate and fully covered entries. Optional CIDR collapse preserves the exact IP set and only creates /24 or larger networks. +
+

Schedule