diff --git a/internal/codeindex/commit.go b/internal/codeindex/commit.go new file mode 100644 index 00000000..165237ab --- /dev/null +++ b/internal/codeindex/commit.go @@ -0,0 +1,207 @@ +package codeindex + +import ( + "bufio" + "bytes" + "context" + "crypto/sha1" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "unicode/utf8" +) + +// ScanCommit reads immutable Git blobs, never worktree files or checkout filters. +// Callers own fetching and retention. A full object ID pins every document even +// when HEAD or the upstream branch advances while this scan is running. +func ScanCommit(ctx context.Context, options Options, commit string) (Result, error) { + if !fullObjectID(commit) { + return Result{}, fmt.Errorf("code scan requires a full lowercase commit object ID") + } + if options.MaxFileBytes <= 0 { + options.MaxFileBytes = DefaultMaxFileBytes + } + if options.MaxTotalBytes <= 0 { + options.MaxTotalBytes = DefaultMaxTotalBytes + } + if options.MaxFiles <= 0 { + options.MaxFiles = DefaultMaxFiles + } + if options.Path == "" { + return Result{}, fmt.Errorf("code repository path required") + } + git := func(args ...string) ([]byte, error) { + cmd := commitGit(ctx, options.Path, args...) + var out boundedGitOutput + cmd.Stdout = &out + if err := cmd.Run(); err != nil { + return nil, fmt.Errorf("read commit metadata: %w", err) + } + return out.Bytes(), nil + } + resolved, err := git("rev-parse", "--verify", commit+"^{commit}") + if err != nil { + return Result{}, err + } + if strings.TrimSpace(string(resolved)) != commit { + return Result{}, fmt.Errorf("commit identity differs") + } + dir, err := git("rev-parse", "--absolute-git-dir") + if err != nil { + return Result{}, err + } + root, err := filepath.EvalSymlinks(strings.TrimSpace(string(dir))) + if err != nil { + return Result{}, err + } + entries, err := git("ls-tree", "-r", "-l", "-z", "--full-tree", commit) + if err != nil { + return Result{}, err + } + result := Result{SourceRoot: root, GitSHA: commit, WorktreeDirty: false} + cmd := commitGit(ctx, root, "cat-file", "--batch") + stdin, err := cmd.StdinPipe() + if err != nil { + return Result{}, err + } + stdout, err := cmd.StdoutPipe() + if err != nil { + stdin.Close() + return Result{}, err + } + if err = cmd.Start(); err != nil { + stdin.Close() + return Result{}, err + } + finished := false + defer func() { + stdin.Close() + if !finished { + _ = cmd.Process.Kill() + _ = cmd.Wait() + } + }() + reader := bufio.NewReaderSize(stdout, 4096) + for _, entry := range bytes.Split(entries, []byte{0}) { + if len(entry) == 0 { + continue + } + if err := ctx.Err(); err != nil { + return Result{}, err + } + result.FilesSeen++ + metadata, rawPath, ok := bytes.Cut(entry, []byte{'\t'}) + fields := strings.Fields(string(metadata)) + if !ok || len(fields) != 4 { + return Result{}, fmt.Errorf("invalid commit tree entry") + } + if fields[1] != "blob" || (fields[0] != "100644" && fields[0] != "100755") { + result.SkippedNonRegular++ + continue + } + path := string(rawPath) + if !utf8.Valid(rawPath) || !safeRelativePath(path) { + return Result{}, fmt.Errorf("unsafe commit path") + } + size, err := strconv.ParseInt(fields[3], 10, 64) + if err != nil || size < 0 || !fullObjectID(fields[2]) { + return Result{}, fmt.Errorf("invalid commit blob metadata") + } + if size > options.MaxFileBytes { + result.SkippedLarge++ + continue + } + if _, err := fmt.Fprintln(stdin, fields[2]); err != nil { + return Result{}, err + } + header, err := reader.ReadString('\n') + if err != nil || len(header) > 256 { + return Result{}, fmt.Errorf("invalid commit blob header") + } + if strings.TrimSuffix(header, "\n") != fields[2]+" blob "+fields[3] { + return Result{}, fmt.Errorf("commit blob identity/size differs") + } + data := make([]byte, size) + if _, err := io.ReadFull(reader, data); err != nil { + return Result{}, err + } + if end, err := reader.ReadByte(); err != nil || end != '\n' { + return Result{}, fmt.Errorf("incomplete commit blob") + } + if blobObjectID(data, len(fields[2])) != fields[2] { + return Result{}, fmt.Errorf("commit blob checksum differs") + } + if bytes.IndexByte(data, 0) >= 0 || !utf8.Valid(data) { + result.SkippedBinary++ + continue + } + if result.FilesIndexed >= options.MaxFiles { + return Result{}, fmt.Errorf("tracked text file count exceeds --max-files %d", options.MaxFiles) + } + if size > options.MaxTotalBytes-result.BytesIndexed { + return Result{}, fmt.Errorf("tracked text corpus exceeds --max-total-bytes %d", options.MaxTotalBytes) + } + sum := sha256.Sum256(data) + result.Documents = append(result.Documents, Document{Path: path, Language: languageForPath(path), ContentHash: hex.EncodeToString(sum[:]), Text: string(data), ByteSize: size}) + result.FilesIndexed++ + result.BytesIndexed += size + } + if err := stdin.Close(); err != nil { + return Result{}, err + } + err = cmd.Wait() + finished = true + if err != nil { + return Result{}, fmt.Errorf("read commit blobs: %w", err) + } + return result, nil +} + +func fullObjectID(value string) bool { + if len(value) != 40 && len(value) != 64 { + return false + } + for _, c := range value { + if !strings.ContainsRune("0123456789abcdef", c) { + return false + } + } + return true +} + +func blobObjectID(data []byte, width int) string { + material := append([]byte(fmt.Sprintf("blob %d\x00", len(data))), data...) + if width == 40 { + sum := sha1.Sum(material) + return hex.EncodeToString(sum[:]) + } + sum := sha256.Sum256(material) + return hex.EncodeToString(sum[:]) +} + +type boundedGitOutput struct{ bytes.Buffer } + +func (w *boundedGitOutput) Write(p []byte) (int, error) { + if w.Len()+len(p) > 32<<20 { + return 0, fmt.Errorf("commit tree metadata exceeds32MiB") + } + return w.Buffer.Write(p) +} + +func commitGit(ctx context.Context, root string, args ...string) *exec.Cmd { + cmd := exec.CommandContext(ctx, "git", append([]string{"-c", "core.fsmonitor=false", "-C", root}, args...)...) + for _, value := range os.Environ() { + key, _, _ := strings.Cut(value, "=") + if !strings.HasPrefix(strings.ToUpper(key), "GIT_") { + cmd.Env = append(cmd.Env, value) + } + } + cmd.Env = append(cmd.Env, "GIT_CONFIG_NOSYSTEM=1", "GIT_CONFIG_GLOBAL="+os.DevNull, "GIT_TERMINAL_PROMPT=0", "GIT_OPTIONAL_LOCKS=0", "GIT_NO_REPLACE_OBJECTS=1", "GIT_NO_LAZY_FETCH=1") + return cmd +} diff --git a/internal/codeindex/commit_test.go b/internal/codeindex/commit_test.go new file mode 100644 index 00000000..de2e2b80 --- /dev/null +++ b/internal/codeindex/commit_test.go @@ -0,0 +1,124 @@ +package codeindex + +import ( + "context" + "os" + "path/filepath" + "reflect" + "strings" + "testing" +) + +func commitFixture(t *testing.T) (string, string) { + t.Helper() + root := t.TempDir() + runGit(t, root, "init", "-b", "main") + writeFile(t, filepath.Join(root, "before.txt"), "original\n") + writeFile(t, filepath.Join(root, "deleted.txt"), "delete me\n") + runGit(t, root, "add", ".") + runGit(t, root, "-c", "user.name=Test", "-c", "user.email=test@example.com", "commit", "-m", "first") + sha, err := gitText(context.Background(), root, "rev-parse", "HEAD") + if err != nil { + t.Fatal(err) + } + return root, sha +} + +func TestScanCommitPinnedDespiteDirtyWorktreeAndMovingMain(t *testing.T) { + root, first := commitFixture(t) + original, err := ScanCommit(context.Background(), Options{Path: root}, first) + if err != nil { + t.Fatal(err) + } + runGit(t, root, "mv", "before.txt", "after.txt") + runGit(t, root, "rm", "deleted.txt") + writeFile(t, filepath.Join(root, "new.go"), "package fresh\n") + runGit(t, root, "add", ".") + runGit(t, root, "-c", "user.name=Test", "-c", "user.email=test@example.com", "commit", "-m", "second") + second, err := gitText(context.Background(), root, "rev-parse", "HEAD") + if err != nil { + t.Fatal(err) + } + writeFile(t, filepath.Join(root, "after.txt"), "uncommitted developer content") + if err := os.Remove(filepath.Join(root, "new.go")); err != nil { + t.Fatal(err) + } + again, err := ScanCommit(context.Background(), Options{Path: root}, first) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(original, again) || again.WorktreeDirty { + t.Fatal("captured commit changed with worktree/HEAD") + } + next, err := ScanCommit(context.Background(), Options{Path: root}, second) + if err != nil { + t.Fatal(err) + } + got := map[string]string{} + for _, d := range next.Documents { + got[d.Path] = d.Text + } + if !reflect.DeepEqual(got, map[string]string{"after.txt": "original\n", "new.go": "package fresh\n"}) || next.WorktreeDirty || next.GitSHA != second { + t.Fatalf("wrong immutable generation: %#v", next) + } +} + +func TestScanCommitLimitsModesAndBareSource(t *testing.T) { + root, _ := commitFixture(t) + writeFile(t, filepath.Join(root, "large.txt"), strings.Repeat("x", 30)) + if err := os.WriteFile(filepath.Join(root, "binary"), []byte{0, 1, 2}, 0600); err != nil { + t.Fatal(err) + } + if err := os.Symlink("before.txt", filepath.Join(root, "link")); err != nil { + t.Fatal(err) + } + runGit(t, root, "add", ".") + runGit(t, root, "-c", "user.name=Test", "-c", "user.email=test@example.com", "commit", "-m", "limits") + sha, err := gitText(context.Background(), root, "rev-parse", "HEAD") + if err != nil { + t.Fatal(err) + } + bare := filepath.Join(t.TempDir(), "source.git") + runGit(t, root, "clone", "--bare", "--no-hardlinks", root, bare) + r, err := ScanCommit(context.Background(), Options{Path: bare, MaxFileBytes: 16}, sha) + if err != nil { + t.Fatal(err) + } + if r.FilesIndexed != 2 || r.SkippedLarge != 1 || r.SkippedBinary != 1 || r.SkippedNonRegular != 1 || r.WorktreeDirty { + t.Fatalf("unexpected scope: %#v", r) + } + for _, o := range []Options{{Path: bare, MaxFiles: 1}, {Path: bare, MaxTotalBytes: 1}} { + if _, err := ScanCommit(context.Background(), o, sha); err == nil { + t.Fatal("accepted incomplete bounded corpus") + } + } + for _, ref := range []string{"HEAD", sha[:12], "--all", strings.Repeat("f", 40)} { + if _, err := ScanCommit(context.Background(), Options{Path: bare}, ref); err == nil { + t.Fatal("accepted invalid/missing commit") + } + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := ScanCommit(ctx, Options{Path: bare}, sha); err == nil { + t.Fatal("ignored cancellation") + } +} + +func TestScanCommitIgnoresReplacementObjects(t *testing.T) { + root, first := commitFixture(t) + writeFile(t, filepath.Join(root, "before.txt"), "replacement\n") + runGit(t, root, "add", ".") + runGit(t, root, "-c", "user.name=Test", "-c", "user.email=test@example.com", "commit", "-m", "replacement") + second, err := gitText(context.Background(), root, "rev-parse", "HEAD") + if err != nil { + t.Fatal(err) + } + runGit(t, root, "replace", first, second) + r, err := ScanCommit(context.Background(), Options{Path: root}, first) + if err != nil { + t.Fatal(err) + } + if r.Documents[0].Text != "original\n" { + t.Fatal("replacement ref changed captured content") + } +} diff --git a/scripts/native-code/README.md b/scripts/native-code/README.md new file mode 100644 index 00000000..f312dbb6 --- /dev/null +++ b/scripts/native-code/README.md @@ -0,0 +1,68 @@ +# Official-main native code producer + +The code index follows `https://github.com/openclaw/openclaw.git`, branch `main`. +The previous scheduled producer scanned a development worktree without fetching, +so it could correctly report unchanged while that checkout stayed pinned. + +`upstream.py` owns only `/upstream/source.git`, a private bare cache. +It fetches main before owner settlement and retains an immutable metadata receipt +with the exact full commit ID, tree, commit timestamp and observation timestamp. +There is no checkout, reset, source-file execution, or read from a serving Gateway +or development tree. Git configuration/remote and cache ownership are checked. +Fetch failure preserves the previous publication and records failure; a cached +commit is never relabelled as a successful current upstream observation. + +The commit scanner reads Git objects directly with `ls-tree`/`cat-file`, validates +blob identities, and applies the existing text/language/size rules. Every document +comes from the one captured commit even if main advances. Worktree state is +irrelevant; committed snapshots have `worktree_dirty=false`. Limits remain +512 KiB per file (larger files skipped), 1 GiB accepted text, 100,000 documents, +45 seconds scan and 90 seconds atomic write. Missing objects, partial reads and +corpus-limit failures refuse the entire publication. The cache has a 4 GiB +admission budget and requires 2 GiB free; it never prunes source archives or +existing recovery material. Reaching that budget is an explicit failed cycle, +not a fresh result. Old owned Git objects remain while cleanup holds apply. + +`run_code_update.py` retains the prior native lifecycle: shared controller lock, +preparation while core collection runs, exact no-op without owner settlement, +fsynced handoff journal, brief settlement of the original analytics owner only, +original runner lock and schema16/inode/provider checks, and immediate restoration +of the same analytics job. Its existing journal-only recovery helper remains +compatible. The native SQLite writer opens an existing archive without migration +or creation and retains the delivered FULL/Darwin full-flush policy. + +The original `ReplaceCodeSnapshot` transaction publishes snapshot metadata and all +documents together, removing obsolete derived code rows atomically. An interrupted +write leaves the prior generation. The no-op check includes commit, source root, +dirty flag, counts and the complete path/language/content-hash/byte-size set. +There are no schema changes or observation-feed activation. Authoritative H is +the sole destination; writable portable P and all source histories stay separate. + +Existing reader fields remain unchanged: `code_snapshots.git_sha`, +`worktree_dirty`, `file_count`, `byte_count`, `indexed_at`, and `code_documents`. +`source_root` is a private cache path. Supplemental `upstream` receipt fields +identify remote, branch, full commit, tree and observation time. Each successful +cycle has its own receipt; `last-code-result.json` provides a code-specific latest +pointer, independent of embedding/cluster receipts sharing the lifecycle directory. +Native code text is not submitted for embedding by this producer. + +The isolated CLI/store files use the `native_code_overlay` build tag. Delivery +builds them against the pinned source baseline using the existing held-capture +exclusion overlay; `native_store.go` and its tests map into `internal/store`. +This does not build or install the held source-capture candidate. Install immutable +artifacts outside the checkout and preserve the existing 900-second code cadence, +without KeepAlive. Restore the previous code-job plist/artifact for rollback; +never roll the live database backward. + +Focused checks use synthetic repositories and databases: + +``` +go test ./internal/codeindex -run TestScanCommit +python3 -m unittest discover -s scripts/native-code -p 'test_*.py' +go test -overlay DELIVERY_OVERLAY -tags native_code_overlay ./internal/store -run 'TestNativeProducerCodeNoopAndRollback|TestNativeProducerExistingWriterNeverMigrates|TestNativeCodeWriterFlushAndObsoleteReplacement' +go test -overlay DELIVERY_OVERLAY -tags native_code_overlay ./scripts/native-code/cmd -run TestCodeCommitGate +``` + +Before live publication, verify the exact reviewed bundle and source receipt, +then retain a native document/count/hash comparison and deleted-path check for +the downstream reader owner. Verify ordinary core progress after restoration. diff --git a/scripts/native-code/cmd/main.go b/scripts/native-code/cmd/main.go new file mode 100644 index 00000000..7831f5b2 --- /dev/null +++ b/scripts/native-code/cmd/main.go @@ -0,0 +1,258 @@ +//go:build native_code_overlay + +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "github.com/openclaw/gitcrawl/internal/codeindex" + "github.com/openclaw/gitcrawl/internal/store" + "os" + "path/filepath" + "sort" + "strings" + "syscall" + "time" +) + +func main() { + if e := codeRun(os.Args[1:]); e != nil { + fmt.Fprintln(os.Stderr, e) + os.Exit(1) + } +} +func codeRun(args []string) error { + fs := flag.NewFlagSet("code-index", flag.ContinueOnError) + db := fs.String("db", "", "H database") + source := fs.String("source", "", "owned Git object repository") + commit := fs.String("commit", "", "full immutable commit ID") + scanOnly := fs.Bool("scan-only", false, "read immutable source without opening H") + manifest := fs.String("manifest", "", "optional new metadata manifest file") + upstream := fs.String("upstream-receipt", "", "captured official-main receipt") + gate := fs.String("commit-gate", "", "one-shot private gate file") + writeLimit := fs.Duration("write-timeout", 90*time.Second, "atomic write deadline") + if e := fs.Parse(args); e != nil { + return e + } + if *source == "" || *commit == "" || (!*scanOnly && (*db == "" || *gate == "" || *upstream == "")) { + return fmt.Errorf("source/commit and live db/commit-gate/upstream-receipt required") + } + if *writeLimit <= 0 || *writeLimit > 2*time.Minute { + return fmt.Errorf("write-timeout must be positive and <=2m") + } + started := time.Now().UTC() + scanCtx, done := context.WithTimeout(context.Background(), 45*time.Second) + result, e := codeindex.ScanCommit(scanCtx, codeindex.Options{Path: *source, MaxFileBytes: 524288, MaxTotalBytes: 1 << 30, MaxFiles: 100000}, *commit) + done() + if e != nil { + return e + } + var captured map[string]any + if *upstream != "" { + b, e := os.ReadFile(*upstream) + if e != nil { + return e + } + if e = json.Unmarshal(b, &captured); e != nil { + return e + } + if captured["format"] != "gitcrawl.code-upstream.v1" || captured["repository_url"] != "https://github.com/openclaw/openclaw.git" || captured["branch"] != "main" || captured["commit"] != result.GitSHA || captured["git_dir"] != result.SourceRoot { + return fmt.Errorf("captured upstream source binding differs") + } + captured["receipt_sha256"] = store.NativeSHA256(b) + } + type documentMeta struct { + Path string `json:"path"` + Language string `json:"language"` + Hash string `json:"sha256"` + Bytes int64 `json:"bytes"` + } + if *manifest != "" { + entries := make([]documentMeta, 0, len(result.Documents)) + for _, d := range result.Documents { + entries = append(entries, documentMeta{d.Path, d.Language, d.ContentHash, d.ByteSize}) + } + raw, e := json.Marshal(entries) + if e != nil { + return e + } + f, e := os.OpenFile(*manifest, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600) + if e != nil { + return e + } + _, e = f.Write(raw) + if e == nil { + e = f.Sync() + } + ce := f.Close() + if e != nil { + return e + } + if ce != nil { + return ce + } + } + if *scanOnly { + return json.NewEncoder(os.Stdout).Encode(map[string]any{"format": "gitcrawl.code-scan.v1", "git_sha": result.GitSHA, "source_root": result.SourceRoot, "tracked_dirty": result.WorktreeDirty, "documents": result.FilesIndexed, "text_bytes": result.BytesIndexed, "skip_counts": map[string]int{"binary": result.SkippedBinary, "large": result.SkippedLarge, "nonregular": result.SkippedNonRegular}, "database_writes": false, "scanned_at": time.Now().UTC()}) + } + readCtx, done := context.WithTimeout(context.Background(), 15*time.Second) + ro, e := store.OpenReadOnly(readCtx, *db) + if e != nil { + done() + return e + } + if e = ro.NativeCodeWriterReady(readCtx); e != nil { + ro.Close() + done() + return e + } + var coreComplete int + var coreObserved string + if e = ro.DB().QueryRowContext(readCtx, "select complete,observed_at from analytics_coverage where repository='openclaw/openclaw'").Scan(&coreComplete, &coreObserved); e != nil { + ro.Close() + done() + return fmt.Errorf("core publication baseline required: %w", e) + } + at, parseErr := time.Parse(time.RFC3339Nano, coreObserved) + if parseErr != nil || coreComplete != 1 || time.Since(at) > 5*time.Minute { + ro.Close() + done() + return fmt.Errorf("core collection must be complete and current before code handoff") + } + repo, e := ro.RepositoryByFullName(readCtx, "openclaw/openclaw") + if e != nil { + ro.Close() + done() + return e + } + snap := store.CodeSnapshot{RepoID: repo.ID, SourceRoot: result.SourceRoot, GitSHA: result.GitSHA, WorktreeDirty: result.WorktreeDirty, FileCount: result.FilesIndexed, ByteCount: result.BytesIndexed, IndexedAt: time.Now().UTC().Format(time.RFC3339Nano)} + docs := make([]store.CodeDocument, 0, len(result.Documents)) + type meta struct { + Path, Language, Hash string + Bytes int64 + } + metas := []meta{} + for _, d := range result.Documents { + docs = append(docs, store.CodeDocument{RepoID: repo.ID, Path: d.Path, Language: d.Language, ContentHash: d.ContentHash, Text: d.Text, ByteSize: d.ByteSize, UpdatedAt: snap.IndexedAt}) + metas = append(metas, meta{d.Path, d.Language, d.ContentHash, d.ByteSize}) + } + sort.Slice(metas, func(i, j int) bool { return metas[i].Path < metas[j].Path }) + material, _ := json.Marshal(struct { + Root, SHA string + Dirty bool + Documents []meta + }{snap.SourceRoot, snap.GitSHA, snap.WorktreeDirty, metas}) + digest := store.NativeSHA256(material) + var id int64 + var same bool + e = ro.WithTx(readCtx, func(tx *store.Store) error { + var err error + id, same, err = tx.NativeCodeSnapshotMatches(readCtx, snap, docs) + return err + }) + closeErr := ro.Close() + done() + if e != nil { + return e + } + if closeErr != nil { + return closeErr + } + output := map[string]any{"format": "gitcrawl.code-producer.v1", "upstream": captured, "phase": "prepared", "started_at": started, "scanned_at": time.Now().UTC(), "git_sha": snap.GitSHA, "tracked_dirty": snap.WorktreeDirty, "source_root": snap.SourceRoot, "documents": len(docs), "text_bytes": snap.ByteCount, "content_digest": digest, "unchanged": same, "snapshot_id": id, "skip_counts": map[string]int{"binary": result.SkippedBinary, "large": result.SkippedLarge, "nonregular": result.SkippedNonRegular, "missing": result.SkippedMissing, "outside": result.SkippedOutsideRoot}} + if same { + output["phase"] = "unchanged" + return json.NewEncoder(os.Stdout).Encode(output) + } + info, e := os.Stat(*db) + if e != nil { + return e + } + identity := info.Sys().(*syscall.Stat_t) + output["source_device"] = fmt.Sprint(identity.Dev) + output["source_inode"] = fmt.Sprint(identity.Ino) + if e = json.NewEncoder(os.Stdout).Encode(output); e != nil { + return e + } + // Gate is created by the reviewed owner wrapper only after ordinary settlement. + parent := os.Getppid() + until := time.Now().Add(60 * time.Second) + for { + if os.Getppid() != parent { + return fmt.Errorf("owner wrapper exited before commit") + } + if time.Now().After(until) { + return fmt.Errorf("commit gate timed out") + } + b, err := os.ReadFile(*gate) + if err == nil { + if strings.TrimSpace(string(b)) != digest { + return fmt.Errorf("commit gate digest mismatch") + } + break + } + if !os.IsNotExist(err) { + return err + } + time.Sleep(50 * time.Millisecond) + } + lock, e := os.OpenFile(filepath.Join(filepath.Dir(*db), "runner.lock"), os.O_CREATE|os.O_RDWR, 0600) + if e != nil { + return e + } + defer lock.Close() + if e = syscall.Flock(int(lock.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); e != nil { + return fmt.Errorf("native owner lock busy: %w", e) + } + defer syscall.Flock(int(lock.Fd()), syscall.LOCK_UN) + now, e := os.Stat(*db) + if e != nil { + return e + } + current := now.Sys().(*syscall.Stat_t) + if current.Dev != identity.Dev || current.Ino != identity.Ino { + return fmt.Errorf("source file replaced after scan") + } + ctx, finish := context.WithTimeout(context.Background(), *writeLimit) + defer finish() + probe, e := store.OpenReadOnly(ctx, *db) + if e != nil { + return e + } + e = probe.NativeCodeWriterReady(ctx) + probe.Close() + if e != nil { + return e + } + st, e := store.OpenNativeProducerWriter(ctx, *db) + if e != nil { + return e + } + defer st.Close() + actualRepo, e := st.RepositoryByFullName(ctx, "openclaw/openclaw") + if e != nil { + return e + } + if actualRepo.ID != repo.ID || actualRepo.GitHubRepoID != repo.GitHubRepoID { + return fmt.Errorf("native repository identity changed") + } + id, same, e = st.NativeCodeSnapshotMatches(ctx, snap, docs) + if e != nil { + return e + } + commitStart := time.Now() + if !same { + id, e = st.ReplaceCodeSnapshot(ctx, snap, docs) + if e != nil { + return e + } + } + output["phase"] = "committed" + output["unchanged"] = same + output["snapshot_id"] = id + output["write_seconds"] = time.Since(commitStart).Seconds() + output["finished_at"] = time.Now().UTC() + output["documents"] = len(docs) + return json.NewEncoder(os.Stdout).Encode(output) +} diff --git a/scripts/native-code/cmd/main_test.go b/scripts/native-code/cmd/main_test.go new file mode 100644 index 00000000..716f4cc6 --- /dev/null +++ b/scripts/native-code/cmd/main_test.go @@ -0,0 +1,145 @@ +//go:build native_code_overlay + +package main + +import ( + "context" + "encoding/json" + "os" + "os/exec" + "path/filepath" + "testing" + "time" + + "github.com/openclaw/gitcrawl/internal/codeindex" + "github.com/openclaw/gitcrawl/internal/store" +) + +func TestCodeCommitGatePinsSourceAndReplacesAtomically(t *testing.T) { + for _, fail := range []bool{false, true} { + t.Run(map[bool]string{false: "commit", true: "rollback"}[fail], func(t *testing.T) { + root := t.TempDir() + repo := filepath.Join(root, "source") + if e := os.Mkdir(repo, 0700); e != nil { + t.Fatal(e) + } + git := func(args ...string) string { + t.Helper() + c := exec.Command("git", append([]string{"-C", repo}, args...)...) + b, e := c.CombinedOutput() + if e != nil { + t.Fatalf("fixture git: %v %s", e, b) + } + return string(b) + } + git("init", "-b", "main") + if e := os.WriteFile(filepath.Join(repo, "current.go"), []byte("package captured\n"), 0600); e != nil { + t.Fatal(e) + } + git("add", ".") + git("-c", "user.name=Fixture", "-c", "user.email=fixture@example.invalid", "commit", "-m", "captured") + sha := git("rev-parse", "HEAD") + sha = sha[:len(sha)-1] + scan, e := codeindex.ScanCommit(context.Background(), codeindex.Options{Path: repo}, sha) + if e != nil { + t.Fatal(e) + } + dbpath := filepath.Join(root, "archive.db") + s, e := store.Open(context.Background(), dbpath) + if e != nil { + t.Fatal(e) + } + id, e := s.UpsertRepository(context.Background(), store.Repository{Owner: "openclaw", Name: "openclaw", FullName: "openclaw/openclaw", GitHubRepoID: "42", RawJSON: "{}", UpdatedAt: "t"}) + if e != nil { + t.Fatal(e) + } + _, e = s.DB().Exec("CREATE TABLE IF NOT EXISTS analytics_coverage(repository TEXT PRIMARY KEY,through TEXT,issues INTEGER,pull_requests INTEGER,complete INTEGER,observed_at TEXT)") + if e != nil { + t.Fatal(e) + } + _, e = s.DB().Exec("INSERT INTO analytics_coverage(repository,through,issues,pull_requests,complete,observed_at) VALUES('openclaw/openclaw','fixture',0,0,1,?)", time.Now().UTC().Format(time.RFC3339Nano)) + if e != nil { + t.Fatal(e) + } + old, e := s.ReplaceCodeSnapshot(context.Background(), store.CodeSnapshot{RepoID: id, SourceRoot: "old", GitSHA: "old", FileCount: 1, ByteCount: 3, IndexedAt: "old"}, []store.CodeDocument{{RepoID: id, Path: "obsolete.go", Text: "old", ByteSize: 3, ContentHash: "old"}}) + if e != nil { + t.Fatal(e) + } + if fail { + _, e = s.DB().Exec("CREATE TRIGGER fixture_refuse BEFORE INSERT ON code_documents WHEN NEW.path='current.go' BEGIN SELECT RAISE(ABORT,'fixture'); END") + if e != nil { + t.Fatal(e) + } + } + if e = s.Close(); e != nil { + t.Fatal(e) + } + proof := filepath.Join(root, "source.json") + raw, _ := json.Marshal(map[string]any{"format": "gitcrawl.code-upstream.v1", "repository_url": "https://github.com/openclaw/openclaw.git", "branch": "main", "commit": sha, "git_dir": scan.SourceRoot, "observed_at": time.Now().UTC().Format(time.RFC3339Nano)}) + if e = os.WriteFile(proof, raw, 0600); e != nil { + t.Fatal(e) + } + gate := filepath.Join(root, "commit.gate") + reader, writer, e := os.Pipe() + if e != nil { + t.Fatal(e) + } + stdout := os.Stdout + os.Stdout = writer + defer func() { os.Stdout = stdout; reader.Close(); writer.Close() }() + done := make(chan error, 1) + go func() { + done <- codeRun([]string{"--db", dbpath, "--source", repo, "--commit", sha, "--upstream-receipt", proof, "--commit-gate", gate}) + }() + _ = reader.SetReadDeadline(time.Now().Add(15 * time.Second)) + decoder := json.NewDecoder(reader) + var prepared map[string]any + if e = decoder.Decode(&prepared); e != nil { + t.Fatal(e) + } + if prepared["phase"] != "prepared" || prepared["git_sha"] != sha || prepared["tracked_dirty"] != false { + t.Fatal("wrong prepared source binding") + } + // Advance main and dirty the worktree AFTER preparation, BEFORE the gate. + if e = os.WriteFile(filepath.Join(repo, "current.go"), []byte("package later\n"), 0600); e != nil { + t.Fatal(e) + } + git("add", ".") + git("-c", "user.name=Fixture", "-c", "user.email=fixture@example.invalid", "commit", "-m", "later") + if e = os.WriteFile(gate, []byte(prepared["content_digest"].(string)), 0600); e != nil { + t.Fatal(e) + } + select { + case e = <-done: + case <-time.After(15 * time.Second): + t.Fatal("native commit did not settle") + } + os.Stdout = stdout + if (e != nil) != fail { + t.Fatalf("commit error=%v want_failure=%v", e, fail) + } + ro, e := store.OpenReadOnly(context.Background(), dbpath) + if e != nil { + t.Fatal(e) + } + defer ro.Close() + snap, e := ro.LatestCodeSnapshot(context.Background(), id) + if e != nil { + t.Fatal(e) + } + var path, text string + if e = ro.DB().QueryRow("SELECT path,text_content FROM code_documents WHERE snapshot_id=?", snap.ID).Scan(&path, &text); e != nil { + t.Fatal(e) + } + if fail { + if snap.ID != old || path != "obsolete.go" || text != "old" { + t.Fatal("failed publication damaged prior generation") + } + } else { + if snap.GitSHA != sha || snap.WorktreeDirty || path != "current.go" || text != "package captured\n" { + t.Fatal("publication mixed captured and later commit") + } + } + }) + } +} diff --git a/scripts/native-code/native_store.go b/scripts/native-code/native_store.go new file mode 100644 index 00000000..9ca8fdf2 --- /dev/null +++ b/scripts/native-code/native_store.go @@ -0,0 +1,125 @@ +//go:build native_code_overlay + +// This file is overlaid into internal/store for the isolated schema16 producer. +package store + +import ( + "context" + "crypto/sha256" + "database/sql" + "encoding/hex" + "fmt" + "net/url" + "os" + "path/filepath" + "runtime" +) + +func NativeSHA256(b []byte) string { h := sha256.Sum256(b); return hex.EncodeToString(h[:]) } +func (s *Store) NativeCodeWriterReady(ctx context.Context) error { + var version int + if e := s.q().QueryRowContext(ctx, "pragma user_version").Scan(&version); e != nil { + return e + } + if version != 16 { + return fmt.Errorf("schema16 required") + } + if s.hasTable(ctx, "capture_commits") { + return fmt.Errorf("capture-owned source is outside this isolated producer contract") + } + changes, e := inspectStructuralCompatibilityMigrations(ctx, s, version, inspectPRDetailSchema(ctx, s)) + if e != nil { + return e + } + if len(changes) != 0 { + return fmt.Errorf("native schema needs migration; refusing code-only writer") + } + return nil +} +func (s *Store) NativeCodeSnapshotMatches(ctx context.Context, snap CodeSnapshot, docs []CodeDocument) (int64, bool, error) { + old, e := s.LatestCodeSnapshot(ctx, snap.RepoID) + if e == sql.ErrNoRows { + return 0, false, nil + } + if e != nil { + return 0, false, e + } + if old.RepoID != snap.RepoID || old.SourceRoot != snap.SourceRoot || old.GitSHA != snap.GitSHA || old.WorktreeDirty != snap.WorktreeDirty || old.FileCount != snap.FileCount || old.ByteCount != snap.ByteCount { + return old.ID, false, nil + } + wanted := map[string]CodeDocument{} + for _, d := range docs { + if _, ok := wanted[d.Path]; ok { + return 0, false, fmt.Errorf("duplicate code path") + } + wanted[d.Path] = d + } + rows, e := s.q().QueryContext(ctx, "select repo_id,path,language,content_hash,byte_size from code_documents where snapshot_id=?", old.ID) + if e != nil { + return 0, false, e + } + defer rows.Close() + seen := 0 + for rows.Next() { + var repo, size int64 + var path, lang, hash string + if e = rows.Scan(&repo, &path, &lang, &hash, &size); e != nil { + return 0, false, e + } + d, ok := wanted[path] + if !ok || repo != snap.RepoID || lang != d.Language || hash != d.ContentHash || size != d.ByteSize { + return old.ID, false, nil + } + seen++ + } + if e = rows.Err(); e != nil { + return 0, false, e + } + return old.ID, seen == len(docs), nil +} + +func OpenNativeProducerWriter(ctx context.Context, path string) (*Store, error) { + info, e := os.Stat(path) + if e != nil { + return nil, e + } + if !info.Mode().IsRegular() { + return nil, fmt.Errorf("existing regular native archive required") + } + absolute, e := filepath.Abs(path) + if e != nil { + return nil, e + } + u := url.URL{Scheme: "file", Path: absolute} + q := u.Query() + q.Set("mode", "rw") + q.Add("_pragma", "foreign_keys(1)") + q.Add("_pragma", "busy_timeout(5000)") + q.Add("_pragma", "synchronous(FULL)") + if runtime.GOOS == "darwin" { + q.Add("_pragma", "fullfsync(1)") + q.Add("_pragma", "checkpoint_fullfsync(1)") + } + u.RawQuery = q.Encode() + db, e := sql.Open("sqlite", u.String()) + if e != nil { + return nil, e + } + db.SetMaxOpenConns(1) + db.SetMaxIdleConns(1) + s := &Store{db: db, path: path} + if e = db.PingContext(ctx); e != nil { + db.Close() + return nil, e + } + if e = s.NativeCodeWriterReady(ctx); e != nil { + db.Close() + return nil, e + } + var journal string + if e = db.QueryRowContext(ctx, "pragma journal_mode").Scan(&journal); e != nil || journal != "wal" { + db.Close() + return nil, fmt.Errorf("existing WAL archive required") + } + return s, nil +} diff --git a/scripts/native-code/native_store_test.go b/scripts/native-code/native_store_test.go new file mode 100644 index 00000000..eb7684b5 --- /dev/null +++ b/scripts/native-code/native_store_test.go @@ -0,0 +1,141 @@ +//go:build native_code_overlay + +package store + +import ( + "context" + "os" + "path/filepath" + "runtime" + "testing" +) + +func TestNativeProducerCodeNoopAndRollback(t *testing.T) { + ctx := context.Background() + s, e := Open(ctx, filepath.Join(t.TempDir(), "fixture.db")) + if e != nil { + t.Fatal(e) + } + defer s.Close() + repo, e := s.UpsertRepository(ctx, Repository{Owner: "openclaw", Name: "openclaw", FullName: "openclaw/openclaw", GitHubRepoID: "1103012935", RawJSON: "{}", UpdatedAt: "2026-01-01T00:00:00Z"}) + if e != nil { + t.Fatal(e) + } + snap := CodeSnapshot{RepoID: repo, SourceRoot: "/synthetic", GitSHA: "a", FileCount: 1, ByteCount: 4, IndexedAt: "2026-01-01T00:00:00Z"} + docs := []CodeDocument{{Path: "a.go", Language: "go", ContentHash: "h1", Text: "code", ByteSize: 4, UpdatedAt: snap.IndexedAt}} + id, e := s.ReplaceCodeSnapshot(ctx, snap, docs) + if e != nil { + t.Fatal(e) + } + id2, same, e := s.NativeCodeSnapshotMatches(ctx, snap, docs) + if e != nil || !same || id != id2 { + t.Fatalf("noop %d %v %v", id2, same, e) + } + docs[0].Language = "text" + _, same, e = s.NativeCodeSnapshotMatches(ctx, snap, docs) + if e != nil || same { + t.Fatal("language change missed") + } + docs[0].Language = "go" + // Inject cancellation after a real document insert; old generation must survive. + s.DB().Exec("CREATE TRIGGER fail_fixture_code BEFORE INSERT ON code_documents WHEN NEW.path='reject.go' BEGIN SELECT RAISE(ABORT,'synthetic interruption'); END") + next := append(append([]CodeDocument{}, docs...), CodeDocument{Path: "reject.go", Language: "go", ContentHash: "h2", Text: "next", ByteSize: 4, UpdatedAt: snap.IndexedAt}) + snap.FileCount = 2 + snap.ByteCount = 8 + if _, e = s.ReplaceCodeSnapshot(ctx, snap, next); e == nil { + t.Fatal("synthetic failure ignored") + } + latest, e := s.LatestCodeSnapshot(ctx, repo) + if e != nil || latest.ID != id || latest.FileCount != 1 { + t.Fatal("partial replacement visible") + } + cancelled, cancel := context.WithCancel(ctx) + cancel() + if _, e = s.ReplaceCodeSnapshot(cancelled, snap, next); e == nil { + t.Fatal("cancelled write succeeded") + } +} +func TestNativeProducerExistingWriterNeverMigrates(t *testing.T) { + ctx := context.Background() + p := filepath.Join(t.TempDir(), "source.db") + s, e := Open(ctx, p) + if e != nil { + t.Fatal(e) + } + repo, e := s.UpsertRepository(ctx, Repository{Owner: "openclaw", Name: "openclaw", FullName: "openclaw/openclaw", RawJSON: "{}", UpdatedAt: "2026-01-01T00:00:00Z"}) + if e != nil { + t.Fatal(e) + } + s.DB().Exec("update thread_observation_sequence set value=123 where id=1") + s.DB().Exec("update observation_schema_convergence set checked_observation_sequence=0 where id=1") + s.Close() + os.Chmod(p, 0640) + w, e := OpenNativeProducerWriter(ctx, p) + if e != nil { + t.Fatal(e) + } + defer w.Close() + if _, e = w.ReplaceCodeSnapshot(ctx, CodeSnapshot{RepoID: repo, SourceRoot: "synthetic", GitSHA: "s", IndexedAt: "t"}, nil); e != nil { + t.Fatal(e) + } + var checked, allocated int64 + w.DB().QueryRow("select c.checked_observation_sequence,s.value from observation_schema_convergence c join thread_observation_sequence s on s.id=c.id where c.id=1").Scan(&checked, &allocated) + if checked != 0 || allocated != 123 { + t.Fatal("writer performed convergence repair") + } + info, e := os.Stat(p) + if e != nil || info.Mode().Perm() != 0640 { + t.Fatal("writer changed permissions") + } + missing := filepath.Join(t.TempDir(), "missing.db") + if st, e := OpenNativeProducerWriter(ctx, missing); e == nil { + st.Close() + t.Fatal("missing source accepted") + } + if _, e = os.Stat(missing); !os.IsNotExist(e) { + t.Fatal("source was created") + } +} + +func TestNativeCodeWriterFlushAndObsoleteReplacement(t *testing.T) { + ctx := context.Background() + p := filepath.Join(t.TempDir(), "source.db") + s, e := Open(ctx, p) + if e != nil { + t.Fatal(e) + } + repo, e := s.UpsertRepository(ctx, Repository{Owner: "openclaw", Name: "openclaw", FullName: "openclaw/openclaw", RawJSON: "{}", UpdatedAt: "t"}) + if e != nil { + t.Fatal(e) + } + s.Close() + s, e = OpenNativeProducerWriter(ctx, p) + if e != nil { + t.Fatal(e) + } + defer s.Close() + var level, full, checkpoint int + s.DB().QueryRow("pragma synchronous").Scan(&level) + s.DB().QueryRow("pragma fullfsync").Scan(&full) + s.DB().QueryRow("pragma checkpoint_fullfsync").Scan(&checkpoint) + if level < 2 || runtime.GOOS == "darwin" && (full != 1 || checkpoint != 1) { + t.Fatalf("writer durability %d %d %d", level, full, checkpoint) + } + first := CodeSnapshot{RepoID: repo, SourceRoot: "owned-cache", GitSHA: "first", FileCount: 1, ByteCount: 3, IndexedAt: "first"} + if _, e = s.ReplaceCodeSnapshot(ctx, first, []CodeDocument{{RepoID: repo, Path: "old.go", Text: "old", ContentHash: "h", ByteSize: 3}}); e != nil { + t.Fatal(e) + } + second := first + second.GitSHA = "second" + second.IndexedAt = "second" + id, e := s.ReplaceCodeSnapshot(ctx, second, []CodeDocument{{RepoID: repo, Path: "renamed.go", Text: "new", ContentHash: "new", ByteSize: 3}}) + if e != nil { + t.Fatal(e) + } + var old, foreign int + s.DB().QueryRow("select count(*) from code_documents where path='old.go'").Scan(&old) + s.DB().QueryRow("select count(*) from code_documents where snapshot_id!=?", id).Scan(&foreign) + if old != 0 || foreign != 0 { + t.Fatal("obsolete generation retained in current code index") + } +} diff --git a/scripts/native-code/run_code_update.py b/scripts/native-code/run_code_update.py new file mode 100755 index 00000000..00741cd7 --- /dev/null +++ b/scripts/native-code/run_code_update.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +"""Official-main immutable code producer; existing native handoff lifecycle.""" +from upstream import SourceCache +import argparse,fcntl,json,os,plistlib,select,signal,subprocess,sys,time,hashlib,uuid +from pathlib import Path + +def digest(path): return hashlib.sha256(Path(path).read_bytes()).hexdigest() +def atomic(path,value): + path=Path(path);tmp=path.with_name(path.name+'.tmp-'+uuid.uuid4().hex) + with open(tmp,'x',encoding='utf8') as f: + os.chmod(tmp,0o600);json.dump(value,f,sort_keys=True);f.write('\n');f.flush();os.fsync(f.fileno()) + os.replace(tmp,path) + fd=os.open(path.parent,os.O_RDONLY) + try:os.fsync(fd) + finally:os.close(fd) +def command(argv,timeout=20):return subprocess.run(argv,stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True,timeout=timeout) + +class Lifecycle: + def __init__(self,a,run=command): + self.a=a;self.run=run;self.state=Path(a.state_dir);self.state.mkdir(parents=True,exist_ok=True,mode=0o700) + self.journal=self.state/'handoff.json';self.target=f'gui/{os.getuid()}/{a.label}' + def verify(self): + for path,want in [(self.a.binary,self.a.binary_sha256),(self.a.job_plist,self.a.plist_sha256),(self.a.config,self.a.config_sha256)]: + if digest(path)!=want:raise RuntimeError('pinned producer/job/config changed: '+path) + spec=plistlib.loads(Path(self.a.job_plist).read_bytes()) + if spec.get('Label')!=self.a.label:raise RuntimeError('job label mismatch') + args=spec.get('ProgramArguments',[]) + if not args or args[0]!=self.a.collector_binary or 'analytics' not in args or '--watch' not in args or self.a.config not in args:raise RuntimeError('unexpected analytics job') + def loaded(self):return self.run(['launchctl','print',self.target]).returncode==0 + def restore(self,record): + # A prior disabled/unloaded job is never enabled by this controller. + if record['was_loaded']: + if digest(self.a.job_plist)!=record['plist_sha256'] or digest(self.a.config)!=record['config_sha256']:raise RuntimeError('original job/config changed during handoff; restoration requires owner review') + if not self.loaded(): + result=self.run(['launchctl','bootstrap',f'gui/{os.getuid()}',self.a.job_plist]) + if result.returncode and not self.loaded():raise RuntimeError('original analytics bootstrap failed') + if not self.loaded():raise RuntimeError('original analytics job not loaded after restoration') + record['restored_at']=time.time();record['phase']='restored';atomic(self.state/'last-handoff.json',record) + if self.journal.exists():self.journal.unlink() + def recover(self): + if not self.journal.exists():return False + record=json.loads(self.journal.read_text());self.restore(record);atomic(self.state/'last-result.json',{'status':'interrupted_handoff_restored','index_committed':'unknown','previous':record});return True + def update(self): + if self.recover():return + self.verify() + runid=uuid.uuid4().hex;gate=self.state/(runid+'.gate');record=None;child=None + captured=None + env=dict(os.environ,GIT_OPTIONAL_LOCKS='0',GIT_NO_LAZY_FETCH='1',GIT_TERMINAL_PROMPT='0',GOMAXPROCS='2') + argv=[self.a.binary,'--db',self.a.db,'--commit-gate',str(gate)] + try: + captured,proof=SourceCache(self.state,self.a.source).capture(runid) + argv += ['--source',captured['git_dir'],'--commit',captured['commit'],'--upstream-receipt',str(proof),'--write-timeout',f'{self.a.write_seconds}s'] + child=subprocess.Popen(argv,stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True,env=env) + if not select.select([child.stdout],[],[],65)[0]:raise RuntimeError('native prepare timed out') + line=child.stdout.readline() + if not line: + _,err=child.communicate(timeout=5);raise RuntimeError('native prepare failed: '+err[-2000:]) + prepared=json.loads(line) + if prepared.get('git_sha')!=captured['commit'] or prepared.get('tracked_dirty') is not False or prepared.get('source_root')!=captured['git_dir'] or prepared.get('upstream',{}).get('receipt_sha256')!=digest(proof):raise RuntimeError('prepared upstream binding mismatch') + if prepared.get('phase')=='unchanged': + out,err=child.communicate(timeout=5) + if child.returncode:raise RuntimeError('native no-op failed: '+err[-1000:]) + atomic(self.state/(runid+'-code-result.json'),prepared);atomic(self.state/'last-code-result.json',prepared);atomic(self.state/'last-result.json',prepared);print(json.dumps(prepared));return + if prepared.get('phase')!='prepared':raise RuntimeError('unexpected native producer phase') + self.verify();was_loaded=self.loaded();record={'phase':'settling','was_loaded':was_loaded,'plist_sha256':self.a.plist_sha256,'config_sha256':self.a.config_sha256,'prepared':prepared,'started_at':time.time()};atomic(self.journal,record) + if was_loaded: + result=self.run(['launchctl','bootout',self.target]) + if result.returncode:raise RuntimeError('ordinary analytics settlement failed') + # Wait for graceful native exit; never steal/delete the existing lock. + until=time.monotonic()+20 + while True: + with open(Path(self.a.db).parent/'runner.lock','rb') as probe: + try:fcntl.flock(probe,fcntl.LOCK_EX|fcntl.LOCK_NB);break + except BlockingIOError: + if time.monotonic()>=until:raise RuntimeError('native owner did not settle within20s') + time.sleep(.1) + # Child checks source identity and takes the original runner.lock itself. + with open(gate,'x',encoding='ascii') as f: + os.chmod(gate,0o600);f.write(prepared['content_digest']);f.flush();os.fsync(f.fileno()) + out,err=child.communicate(timeout=self.a.write_seconds+20) + if child.returncode:raise RuntimeError('native atomic commit failed: '+err[-2000:]) + lines=[x for x in out.splitlines() if x.strip()] + if not lines:raise RuntimeError('native commit receipt missing') + result=json.loads(lines[-1]) + if result.get('phase')!='committed':raise RuntimeError('native commit receipt invalid') + atomic(self.state/(runid+'-commit.json'),result);atomic(self.state/'last-code-result.json',result);record['commit_receipt']=runid+'-commit.json';record['phase']='committed';atomic(self.journal,record) + self.restore(record);record=None;atomic(self.state/'last-result.json',result);print(json.dumps(result)) + except BaseException as e: + atomic(self.state/'last-failure.json',{'at':time.time(),'error':str(e),'phase':record['phase'] if record else 'prepare','run_id':runid,'upstream':captured}) + raise + finally: + if child is not None and child.poll() is None: + child.terminate() + try:child.wait(timeout=5) + except subprocess.TimeoutExpired:child.kill();child.wait(timeout=5) + if gate.exists():gate.unlink() + if record is not None:self.restore(record) + +def main(): + p=argparse.ArgumentParser() + for name in ['binary','binary-sha256','db','source','state-dir','job-plist','plist-sha256','config','config-sha256','collector-binary'] :p.add_argument('--'+name,required=True) + p.add_argument('--label',default='com.hrudolph.gitcrawl-analytics');p.add_argument('--write-seconds',type=int,default=90);a=p.parse_args() + if not 1<=a.write_seconds<=120:raise SystemExit('write deadline must be1..120s') + os.umask(0o077);control=Lifecycle(a);lock=open(control.state/'controller.lock','a');os.chmod(control.state/'controller.lock',0o600) + try:fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) + except BlockingIOError:return + def stop(sig,frame):raise KeyboardInterrupt('controller interrupted') + signal.signal(signal.SIGTERM,stop);signal.signal(signal.SIGINT,stop) + control.update() +if __name__=='__main__':main() diff --git a/scripts/native-code/test_lifecycle.py b/scripts/native-code/test_lifecycle.py new file mode 100644 index 00000000..70245db2 --- /dev/null +++ b/scripts/native-code/test_lifecycle.py @@ -0,0 +1,76 @@ +import json +from pathlib import Path +import sys +import tempfile +import types +import unittest +from unittest.mock import patch + +import run_code_update as m + + +class LifecycleTests(unittest.TestCase): + def fixture(self, phase='unchanged', bad_binding=False): + t = tempfile.TemporaryDirectory();self.addCleanup(t.cleanup) + p = Path(t.name).resolve() + (p/'job').write_text('job');(p/'config').write_text('config') + (p/'db').touch();(p/'runner.lock').touch() + source = {'format':'gitcrawl.code-upstream.v1','commit':'a'*40,'git_dir':str(p/'upstream/source.git')} + proof = p/'source.json';m.atomic(proof, source) + prepared = {'phase':phase,'git_sha':source['commit'],'tracked_dirty':False,'source_root':source['git_dir'], + 'content_digest':'fixture','upstream':{'receipt_sha256':m.digest(proof)}} + if bad_binding:prepared['git_sha'] = 'b'*40 + executable = p/'producer' + executable.write_text('#!'+sys.executable+'\nimport json,sys,time,pathlib\nprint('+repr(json.dumps(prepared))+',flush=True)\n'+ + ("gate=pathlib.Path(sys.argv[sys.argv.index('--commit-gate')+1])\nwhile not gate.exists():time.sleep(.01)\nraise SystemExit(1)\n" if phase=='prepared' else '')) + executable.chmod(0o700) + calls=[];state={'loaded':True} + def run(argv, timeout=20): + calls.append(argv) + if argv[1]=='print':return types.SimpleNamespace(returncode=0 if state['loaded'] else 1) + if argv[1]=='bootout':state['loaded']=False;return types.SimpleNamespace(returncode=0) + if argv[1]=='bootstrap':state['loaded']=True;return types.SimpleNamespace(returncode=0) + self.fail('unexpected lifecycle call') + a=types.SimpleNamespace(state_dir=str(p),label='fixture',binary=str(executable),source=source['git_dir'],db=str(p/'db'),write_seconds=1, + job_plist=str(p/'job'),config=str(p/'config'),plist_sha256=m.digest(p/'job'),config_sha256=m.digest(p/'config')) + c=m.Lifecycle(a,run);c.verify=lambda:None + return p,c,calls,state,source,proof + + def test_noop_keeps_owner_running_and_records_observed_upstream(self): + p,c,calls,state,source,proof=self.fixture() + with patch.object(m.SourceCache,'capture',return_value=(source,proof)): + c.update() + self.assertEqual(calls,[]);self.assertTrue(state['loaded']) + self.assertEqual(json.loads((p/'last-code-result.json').read_bytes())['git_sha'],source['commit']) + + def test_fetch_failure_never_settles_owner_or_relabels_old_success(self): + p,c,calls,state,source,proof=self.fixture();m.atomic(p/'last-code-result.json',{'old':True}) + with patch.object(m.SourceCache,'capture',side_effect=RuntimeError('upstream unavailable')): + with self.assertRaisesRegex(RuntimeError,'upstream unavailable'):c.update() + self.assertEqual(calls,[]);self.assertTrue(state['loaded']);self.assertFalse(c.journal.exists()) + self.assertEqual(json.loads((p/'last-code-result.json').read_bytes()),{'old':True}) + + def test_prepared_wrong_sha_refused_before_settlement(self): + p,c,calls,state,source,proof=self.fixture(bad_binding=True) + with patch.object(m.SourceCache,'capture',return_value=(source,proof)): + with self.assertRaisesRegex(RuntimeError,'binding mismatch'):c.update() + self.assertEqual(calls,[]);self.assertTrue(state['loaded']) + + def test_failed_atomic_write_restores_original_owner(self): + p,c,calls,state,source,proof=self.fixture(phase='prepared') + with patch.object(m.SourceCache,'capture',return_value=(source,proof)): + with self.assertRaisesRegex(RuntimeError,'commit failed'):c.update() + self.assertTrue(state['loaded']);self.assertFalse(c.journal.exists()) + self.assertEqual(sum(x[1]=='bootout' for x in calls),1) + self.assertEqual(sum(x[1]=='bootstrap' for x in calls),1) + + def test_recovery_precedes_fetch_and_preserves_previously_unloaded_owner(self): + p,c,calls,state,source,proof=self.fixture();state['loaded']=False + m.atomic(c.journal,{'was_loaded':False}) + with patch.object(m.SourceCache,'capture',side_effect=AssertionError('fetch during recovery')): + c.update() + self.assertEqual(calls,[]);self.assertFalse(state['loaded']);self.assertFalse(c.journal.exists()) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/native-code/test_upstream.py b/scripts/native-code/test_upstream.py new file mode 100644 index 00000000..3afa12cf --- /dev/null +++ b/scripts/native-code/test_upstream.py @@ -0,0 +1,102 @@ +import json +import os +from pathlib import Path +import subprocess +import tempfile +import unittest +import uuid + +import upstream + + +class UpstreamTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.base = Path(self.temp.name).resolve() + self.remote = self.base/'remote' + self.remote.mkdir() + self.git(self.remote, 'init', '-b', 'main') + (self.remote/'old.txt').write_text('old\n') + self.commit('first') + self.state = self.base/'state' + self.state.mkdir() + self.calls = [] + + def git(self, repo, *args): + return subprocess.check_output(['/usr/bin/git', '-C', str(repo), *args], stderr=subprocess.DEVNULL).decode().strip() + + def commit(self, title): + self.git(self.remote, 'add', '.') + self.git(self.remote, '-c', 'user.name=Fixture', '-c', 'user.email=fixture@example.invalid', 'commit', '-m', title) + + def local_git(self, repo, args, timeout=120): + self.calls.append(list(args)) + if args[0] == 'fetch': + args = list(args) + args[args.index('origin')] = self.remote.as_uri() + return upstream.git_command(repo, args, timeout) + + def cache(self, call=None): + return upstream.SourceCache(self.state, self.state/'upstream/source.git', call or self.local_git) + + def test_tracks_remote_main_without_checkout_and_keeps_previous_receipt(self): + c = self.cache() + first, path = c.capture(uuid.uuid4().hex) + before = path.read_bytes() + self.assertEqual(first['commit'], self.git(self.remote, 'rev-parse', 'HEAD')) + self.assertEqual(first['repository_url'], upstream.REMOTE) + self.assertFalse((c.repo/'old.txt').exists()) + (self.remote/'old.txt').unlink() + (self.remote/'new.txt').write_text('new\n') + self.commit('replacement') + second, _ = c.capture(uuid.uuid4().hex) + self.assertNotEqual(first['commit'], second['commit']) + self.assertEqual(second['commit'], self.git(self.remote, 'rev-parse', 'HEAD')) + self.assertEqual(path.read_bytes(), before) + self.assertEqual(self.git(c.repo, 'show', first['commit']+':old.txt'), 'old') + self.assertEqual(self.git(c.repo, 'show', second['commit']+':new.txt'), 'new') + self.assertTrue(all('checkout' not in x and 'reset' not in x for x in self.calls)) + + def test_fetch_failure_is_not_reported_as_current_cached_success(self): + c = self.cache();first, path = c.capture(uuid.uuid4().hex) + def fail(repo, args, timeout=120): + if args[0] == 'fetch': + raise RuntimeError('simulated transport failure') + return self.local_git(repo, args, timeout) + with self.assertRaisesRegex(RuntimeError, 'transport failure'): + self.cache(fail).capture(uuid.uuid4().hex) + self.assertEqual(json.loads(path.read_bytes()), first) + self.assertEqual(len(list(c.root.glob('*-upstream.json'))), 1) + + def test_initialization_interruption_resumes_owned_staging(self): + def interrupt(repo, args, timeout=120): + if args[:2] == ['config', 'remote.origin.fetch']: + raise RuntimeError('interrupted initialization') + return self.local_git(repo, args, timeout) + with self.assertRaisesRegex(RuntimeError, 'interrupted initialization'): + self.cache(interrupt).capture(uuid.uuid4().hex) + self.assertFalse((self.state/'upstream/source.git').exists()) + c = self.cache();receipt, _ = c.capture(uuid.uuid4().hex) + self.assertEqual(receipt['commit'], self.git(self.remote, 'rev-parse', 'HEAD')) + + def test_refuses_external_tree_and_changed_remote(self): + with self.assertRaisesRegex(RuntimeError, 'dedicated'): + upstream.SourceCache(self.state, self.remote) + c = self.cache();c.capture(uuid.uuid4().hex) + self.git(c.repo, 'config', 'remote.origin.url', 'https://example.invalid/wrong.git') + with self.assertRaisesRegex(RuntimeError, 'configuration'): + c.capture(uuid.uuid4().hex) + + def test_refuses_symlink_and_unowned_cache(self): + root = self.state/'upstream' + root.symlink_to(self.remote, target_is_directory=True) + with self.assertRaisesRegex(RuntimeError, 'dedicated|symlink'): + self.cache().capture(uuid.uuid4().hex) + root.unlink();root.mkdir(mode=0o700) + with self.assertRaisesRegex(RuntimeError, 'ownership'): + self.cache().capture(uuid.uuid4().hex) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/native-code/upstream.py b/scripts/native-code/upstream.py new file mode 100644 index 00000000..0fafd324 --- /dev/null +++ b/scripts/native-code/upstream.py @@ -0,0 +1,144 @@ +"""Owned source objects for official-main code indexing. Never a serving checkout.""" +import datetime +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import uuid + +REMOTE = 'https://github.com/openclaw/openclaw.git' +BRANCH = 'main' + + +def now(): + return datetime.datetime.now(datetime.timezone.utc).isoformat() + + +def atomic(path, value): + raw = (json.dumps(value, sort_keys=True) + '\n').encode() + temporary = path.with_name(path.name + '.tmp-' + uuid.uuid4().hex) + with open(temporary, 'xb') as f: + os.chmod(temporary, 0o600) + f.write(raw); f.flush(); os.fsync(f.fileno()) + os.replace(temporary, path) + fd = os.open(path.parent, os.O_RDONLY) + try: + os.fsync(fd) + finally: + os.close(fd) + + +def git_command(repo, args, timeout=120): + env = {k: v for k, v in os.environ.items() if not k.upper().startswith('GIT_')} + env.update(GIT_CONFIG_GLOBAL=os.devnull, GIT_CONFIG_NOSYSTEM='1', + GIT_TERMINAL_PROMPT='0', GIT_ASKPASS='/usr/bin/false', + GIT_OPTIONAL_LOCKS='0', GIT_NO_REPLACE_OBJECTS='1') + command = ['/usr/bin/git', '-c', 'core.hooksPath='+os.devnull, + '-c', 'maintenance.auto=false', '-c', 'gc.auto=0', + '-c', 'protocol.ext.allow=never', '-C', str(repo), *args] + result = subprocess.run(command, env=env, capture_output=True, timeout=timeout) + if result.returncode: + # Do not copy transport/helper messages or potential proxy credentials. + raise RuntimeError('upstream git '+args[0]+' failed, exit='+str(result.returncode)) + if len(result.stdout) > 1 << 20: + raise RuntimeError('upstream metadata output exceeds1MiB') + return result.stdout.decode().strip() + + +class SourceCache: + def __init__(self, state, source, git=git_command): + self.state = Path(state).resolve() + self.root = self.state/'upstream' + self.repo = self.root/'source.git' + # The CLI cannot turn this into a checkout/reset of any external tree. + if Path(source).absolute() != self.repo and Path(source).resolve() != self.repo: + raise RuntimeError('source must be the dedicated state/upstream/source.git cache') + self.git = git + + def owned(self): + if self.root.is_symlink() or self.repo.is_symlink(): + raise RuntimeError('upstream cache symlink refused') + created = not self.root.exists() + self.root.mkdir(mode=0o700, exist_ok=True) + if self.root.stat().st_uid != os.getuid() or self.root.stat().st_mode & 0o077: + raise RuntimeError('upstream cache must be owner-private') + identity = {'format': 'gitcrawl.code-cache.v1', 'repository_url': REMOTE, + 'branch': BRANCH, 'git_dir': str(self.repo)} + marker = self.root/'owner.json' + if created: + atomic(marker, identity) + if marker.is_symlink() or not marker.exists() or json.loads(marker.read_bytes()) != identity: + raise RuntimeError('upstream cache ownership marker differs') + if not self.repo.exists(): + # A killed initial fetch/init never promotes a half-configured cache. + # This fixed staging name belongs only to the checked owner marker. + stage = self.root/'initializing.git' + if stage.is_symlink(): + raise RuntimeError('source initialization symlink refused') + stage.mkdir(mode=0o700, exist_ok=True) + self.git(stage, ['init', '--bare', '--template=', '.']) + self.git(stage, ['config', 'remote.origin.url', REMOTE]) + self.git(stage, ['config', 'remote.origin.fetch', '+refs/heads/main:refs/remotes/origin/main']) + os.rename(stage, self.repo) + fd = os.open(self.root, os.O_RDONLY) + try: + os.fsync(fd) + finally: + os.close(fd) + allowed = {'core.repositoryformatversion': '0', 'core.filemode': 'true', 'core.bare': 'true', + 'remote.origin.url': REMOTE, + 'remote.origin.fetch': '+refs/heads/main:refs/remotes/origin/main'} + # Reject local transport redirection, helpers, includes, filters and aliases. + raw = self.git(self.repo, ['config', '--local', '--null', '--list']) + config = {} + for entry in raw.rstrip('\0').split('\0'): + key, sep, value = entry.partition('\n') + if key in ('core.ignorecase', 'core.precomposeunicode') and value in ('true', 'false'): + continue # Git's filesystem probes; no worktree is created. + if not sep or key in config or key not in allowed or value != allowed[key]: + raise RuntimeError('unsupported owned source Git configuration') + config[key] = value + if config != allowed or self.git(self.repo, ['rev-parse', '--is-bare-repository']) != 'true': + raise RuntimeError('owned source repository identity differs') + for path in ['objects/info/alternates', 'objects/info/http-alternates', 'info/grafts', 'refs/replace']: + if (self.repo/path).exists(): + raise RuntimeError('owned source object redirection refused') + allocated = 0 + for parent, directories, files in os.walk(self.repo, followlinks=False): + for name in directories+files: + path = Path(parent)/name + if path.is_symlink(): + raise RuntimeError('owned source metadata symlink refused') + if path.is_file(): + allocated += path.stat().st_size + if allocated > 4 << 30 or shutil.disk_usage(self.root).free < 2 << 30: + raise RuntimeError('upstream cache disk budget unavailable') + + def capture(self, run_id): + if not re.fullmatch('[a-f0-9]{32}', run_id): + raise RuntimeError('invalid source run ID') + self.owned() + started = now() + # Fetch occurs while ordinary collection is running. No worktree is used. + self.git(self.repo, ['fetch', '--quiet', '--depth=1', '--no-tags', + '--no-auto-maintenance', '--no-recurse-submodules', + 'origin', '+refs/heads/main:refs/remotes/origin/main']) + observed = now() + commit = self.git(self.repo, ['rev-parse', '--verify', 'refs/remotes/origin/main^{commit}']) + if not re.fullmatch('[a-f0-9]{40}', commit): + raise RuntimeError('invalid official upstream commit ID') + committed = self.git(self.repo, ['show', '-s', '--format=%cI', commit]) + datetime.datetime.fromisoformat(committed.replace('Z', '+00:00')) + tree = self.git(self.repo, ['rev-parse', commit+'^{tree}']) + self.owned() + receipt = {'format': 'gitcrawl.code-upstream.v1', 'repository_url': REMOTE, + 'branch': BRANCH, 'git_dir': str(self.repo), 'commit': commit, + 'commit_time': committed, 'tree': tree, + 'fetch_started_at': started, 'observed_at': observed} + path = self.root/(run_id+'-upstream.json') + if path.exists(): + raise RuntimeError('upstream receipt collision') + atomic(path, receipt) + return receipt, path