[LCHIB-786] support symlink files for PTS v2 embedding (main)

src/main/java/com/launchableinc/ingest/commits/CommitGraphCollector.java

@@ -23,6 +23,7 @@
 import org.eclipse.jgit.errors.ConfigInvalidException;
 import org.eclipse.jgit.errors.InvalidObjectIdException;
 import org.eclipse.jgit.errors.MissingObjectException;
+import org.eclipse.jgit.lib.AnyObjectId;
 import org.eclipse.jgit.lib.ConfigConstants;
 import org.eclipse.jgit.lib.FileMode;
 import org.eclipse.jgit.lib.ObjectId;
@@ -49,6 +50,7 @@
 import java.io.OutputStream;
 import java.io.UncheckedIOException;
 import java.net.URL;
+import java.nio.charset.StandardCharsets;
 import java.time.Duration;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -68,6 +70,7 @@
 
 import static com.google.common.collect.ImmutableList.toImmutableList;
 import static java.util.Arrays.stream;
+import static org.eclipse.jgit.lib.Constants.OBJ_BLOB;
 
 /**
  * Compares what commits the local repository and the remote repository have, then send delta over.
@@ -528,26 +531,44 @@ void collectFiles(Collection<ObjectId> advertised, TreeReceiver treeReceiver, Fl
         OUTER:
         while (treeWalk.next()) {
           ObjectId head = treeWalk.getObjectId(0);
-          for (int i = 1; i < c; i++) {
-            if (head.equals(treeWalk.getObjectId(i))) {
-              // file at the head is identical to one of the uninteresting commits,
-              // meaning we have already seen this file/directory on the server.
-              // if it is a dir, there's no need to visit this whole subtree, so skip over
-              continue OUTER;
-            }
-          }
 
           if (treeWalk.isSubtree()) {
+            for (int i = 1; i < c; i++) {
+              if (head.equals(treeWalk.getObjectId(i))) {
+                continue OUTER;
+              }
+            }
             treeWalk.enterSubtree();
+            continue;
+          }
+
+          String filePath = treeWalk.getPathString();
+          FileMode mode = treeWalk.getFileMode(0);
+          ObjectId blobId;
+
+          if (mode == FileMode.SYMLINK) {
+            blobId = resolveSymlinkTarget(start.getTree(), filePath, head);
+            if (blobId == null) {
+              continue;
+            }
+          } else if ((mode.getBits() & FileMode.TYPE_MASK) == FileMode.TYPE_FILE) {
+            blobId = head;
           } else {
-            if ((treeWalk.getFileMode(0).getBits() & FileMode.TYPE_MASK) == FileMode.TYPE_FILE) {
-              GitFile f = new GitFile(name, treeWalk.getPathString(), head, readers::get);
-              // to avoid excessive data transfer, skip files that are too big
-              if (f.size() < 1024 * 1024 && f.isText() && !f.path.equals(HEADER_FILE)) {
-                treeReceiver.accept(f);
-              }
+            continue;
+          }
+
+          // Dedup check uses the actual content blob ID so that symlink target changes are
+          // detected even when the symlink path string (and thus its own blob) is unchanged.
+          for (int i = 1; i < c; i++) {
+            if (blobId.equals(treeWalk.getObjectId(i))) {
+              continue OUTER;
             }
           }
+
+          GitFile f = new GitFile(name, filePath, blobId, readers::get);
+          if (f.size() < 1024 * 1024 && f.isText() && !f.path.equals(HEADER_FILE)) {
+            treeReceiver.accept(f);
+          }
         }
 
         // Now let the server select the files it actually wants to see
@@ -562,6 +583,47 @@ void collectFiles(Collection<ObjectId> advertised, TreeReceiver treeReceiver, Fl
       }
     }
 
+    private ObjectId resolveSymlinkTarget(AnyObjectId treeId, String symlinkPath, ObjectId symlinkBlobId) {
+      try {
+        byte[] raw = objectReader.open(symlinkBlobId, OBJ_BLOB).getCachedBytes(10_000);
+        String targetRelative = new String(raw, StandardCharsets.UTF_8).trim();
+
+        java.nio.file.Path symlinkDir = java.nio.file.Paths.get(symlinkPath).getParent();
+        java.nio.file.Path resolved;
+        if (symlinkDir != null) {
+          resolved = symlinkDir.resolve(targetRelative).normalize();
+        } else {
+          resolved = java.nio.file.Paths.get(targetRelative).normalize();
+        }
+
+        if (resolved.isAbsolute() || resolved.startsWith("..")) {
+          logger.debug("Skipping symlink {} -> {} (points outside repository)", symlinkPath, targetRelative);
+          return null;
+        }
+
+        String resolvedPath = resolved.toString().replace(java.io.File.separatorChar, '/');
+
+        try (TreeWalk tw = TreeWalk.forPath(git, resolvedPath, treeId)) {
+          if (tw == null) {
+            logger.debug("Skipping symlink {} -> {} (target not found in tree)", symlinkPath, resolvedPath);
+            return null;
+          }
+
+          FileMode targetMode = tw.getFileMode(0);
+          if ((targetMode.getBits() & FileMode.TYPE_MASK) == FileMode.TYPE_FILE) {
+            return tw.getObjectId(0);
+          }
+
+          logger.debug("Skipping symlink {} -> {} (target is not a regular file, mode={})",
+              symlinkPath, resolvedPath, targetMode);
+          return null;
+        }
+      } catch (IOException e) {
+        logger.warn("Failed to resolve symlink {}: {}", symlinkPath, e.getMessage());
+        return null;
+      }
+    }
+
     /**
      * Creates a per repository "header" file as a {@link VirtualFile}.
      * Currently, this is just the list of files in the repository.

src/test/java/com/launchableinc/ingest/commits/CommitGraphCollectorTest.java

@@ -29,6 +29,7 @@
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 
 import static com.google.common.truth.Truth.assertThat;
@@ -226,6 +227,96 @@ private void addCommitInSubRepo(Git mainrepo) throws Exception {
     }
   }
 
+  @Test
+  public void symlinkResolvedAsRegularFile() throws Exception {
+    try (Git repo = Git.init().setDirectory(ws).call()) {
+      Files.writeString(ws.toPath().resolve("real.txt"), "hello");
+      repo.add().addFilepattern("real.txt").call();
+      commit(repo).setMessage("add real file").call();
+
+      Files.createSymbolicLink(ws.toPath().resolve("link.txt"), java.nio.file.Path.of("real.txt"));
+      repo.add().addFilepattern("link.txt").call();
+      commit(repo).setMessage("add symlink").call();
+
+      assertThat(collectFilePaths(repo)).containsAtLeast("real.txt", "link.txt");
+    }
+  }
+
+  @Test
+  public void symlinkAcrossDirectories() throws Exception {
+    try (Git repo = Git.init().setDirectory(ws).call()) {
+      Files.createDirectory(ws.toPath().resolve("sub"));
+
+      // subdir -> parent: sub/link.txt -> ../root.txt
+      Files.writeString(ws.toPath().resolve("root.txt"), "root content");
+      Files.createSymbolicLink(ws.toPath().resolve("sub").resolve("link.txt"), java.nio.file.Path.of("../root.txt"));
+      repo.add().addFilepattern("root.txt").addFilepattern("sub/link.txt").call();
+
+      // parent -> subdir: link2.txt -> sub/deep.txt
+      Files.writeString(ws.toPath().resolve("sub").resolve("deep.txt"), "deep content");
+      Files.createSymbolicLink(ws.toPath().resolve("link2.txt"), java.nio.file.Path.of("sub/deep.txt"));
+      repo.add().addFilepattern("sub/deep.txt").addFilepattern("link2.txt").call();
+
+      commit(repo).setMessage("add cross-directory symlinks").call();
+
+      assertThat(collectFilePaths(repo)).containsAtLeast("root.txt", "sub/link.txt", "sub/deep.txt", "link2.txt");
+    }
+  }
+
+  @Test
+  public void brokenSymlinkSkipped() throws Exception {
+    try (Git repo = Git.init().setDirectory(ws).call()) {
+      Files.writeString(ws.toPath().resolve("target.txt"), "content");
+      Files.createSymbolicLink(ws.toPath().resolve("broken.txt"), java.nio.file.Path.of("target.txt"));
+      repo.add().addFilepattern("target.txt").addFilepattern("broken.txt").call();
+      commit(repo).setMessage("add files").call();
+
+      repo.rm().addFilepattern("target.txt").call();
+      commit(repo).setMessage("remove target").call();
+
+      assertThat(collectFilePaths(repo)).doesNotContain("broken.txt");
+    }
+  }
+
+  @Test
+  public void symlinkOutsideRepoSkipped() throws Exception {
+    try (Git repo = Git.init().setDirectory(ws).call()) {
+      Files.createSymbolicLink(ws.toPath().resolve("escape.txt"), java.nio.file.Path.of("../outside.txt"));
+      repo.add().addFilepattern("escape.txt").call();
+      commit(repo).setMessage("add symlink to outside").call();
+
+      assertThat(collectFilePaths(repo)).doesNotContain("escape.txt");
+    }
+  }
+
+  @Test
+  public void regularFilesUnaffectedBySymlinkChange() throws Exception {
+    try (Git repo = Git.init().setDirectory(ws).call()) {
+      Files.writeString(ws.toPath().resolve("a.txt"), "aaa");
+      Files.writeString(ws.toPath().resolve("b.txt"), "bbb");
+      repo.add().addFilepattern("a.txt").addFilepattern("b.txt").call();
+      commit(repo).setMessage("add regular files").call();
+
+      assertThat(collectFilePaths(repo)).containsExactly("a.txt", "b.txt");
+    }
+  }
+
+  private List<String> collectFilePaths(Git repo) throws IOException {
+    List<VirtualFile> files = new ArrayList<>();
+    CommitGraphCollector cgc = new CommitGraphCollector("test", repo.getRepository());
+    cgc.new ByRepository(repo.getRepository(), "main")
+        .collectFiles(Collections.emptyList(), new PassThroughTreeReceiverImpl(),
+            FlushableConsumer.of(files::add));
+
+    List<String> paths = new ArrayList<>();
+    for (VirtualFile f : files) {
+      if (!f.path().equals(CommitGraphCollector.HEADER_FILE)) {
+        paths.add(f.path());
+      }
+    }
+    return paths;
+  }
+
   private CommitCommand commit(Git r) {
     return r.commit().setAll(true).setSign(false);
   }