feature: data management supports nested folders

backend/openapi/specs/data-management.yaml

@@ -330,6 +330,35 @@ paths:
                 type: string
                 format: binary
 
+  /data-management/datasets/{datasetId}/files/upload/add:
+      post:
+          tags: [ DatasetFile ]
+          operationId: addFilesToDataset
+          summary: 添加文件到数据集（仅创建数据库记录）
+          description: 将指定源文件路径列表添加到数据集，仅在数据库中创建记录，不执行物理文件系统操作。
+          parameters:
+              -   name: datasetId
+                  in: path
+                  required: true
+                  schema:
+                      type: string
+                  description: 数据集ID
+          requestBody:
+              required: true
+              content:
+                  application/json:
+                      schema:
+                          $ref: '#/components/schemas/AddFilesRequest'
+          responses:
+              '200':
+                  description: 添加成功，返回创建的文件记录列表
+                  content:
+                      application/json:
+                          schema:
+                              type: array
+                              items:
+                                  $ref: '#/components/schemas/DatasetFileResponse'
+
   /data-management/datasets/{datasetId}/files/upload/pre-upload:
     post:
       tags: [ DatasetFile ]
@@ -805,3 +834,19 @@ components:
         path:
           type: string
           description: 请求路径
+
+    AddFilesRequest:
+        type: object
+        description: 将源文件路径添加到数据集的请求
+        properties:
+            sourcePaths:
+                type: array
+                items:
+                    type: string
+                description: 源文件路径列表（相对或绝对路径），每个元素表示一个要添加的文件或目录路径
+            softAdd:
+                type: boolean
+                description: 如果为 true，则仅在数据库中创建记录（默认 false）
+                default: false
+        required:
+            - sourcePaths

backend/services/data-management-service/src/main/java/com/datamate/datamanagement/application/DatasetFileApplicationService.java

@@ -1,6 +1,7 @@
 package com.datamate.datamanagement.application;
 
 import com.baomidou.mybatisplus.core.metadata.IPage;
+import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
 import com.datamate.common.domain.model.ChunkUploadPreRequest;
 import com.datamate.common.domain.model.FileUploadResult;
 import com.datamate.common.domain.service.FileService;
@@ -29,6 +30,9 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import jakarta.servlet.http.HttpServletResponse;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
+import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream;
+import org.apache.commons.io.IOUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.core.io.Resource;
@@ -37,22 +41,22 @@
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
 
-import java.io.BufferedInputStream;
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.MalformedURLException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.nio.file.attribute.BasicFileAttributes;
 import java.time.LocalDateTime;
+import java.time.ZoneId;
 import java.time.format.DateTimeFormatter;
 import java.util.*;
 import java.util.concurrent.CompletableFuture;
 import java.util.function.Function;
 import java.util.stream.Collectors;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipOutputStream;
+import java.util.stream.Stream;
 
 /**
  * 数据集文件应用服务
@@ -85,11 +89,77 @@ public DatasetFileApplicationService(DatasetFileRepository datasetFileRepository
      */
     @Transactional(readOnly = true)
     public PagedResponse<DatasetFile> getDatasetFiles(String datasetId, String fileType, String status, String name, PagingQuery pagingQuery) {
-        IPage<DatasetFile> page = new com.baomidou.mybatisplus.extension.plugins.pagination.Page<>(pagingQuery.getPage(), pagingQuery.getSize());
+        IPage<DatasetFile> page = new Page<>(pagingQuery.getPage(), pagingQuery.getSize());
         IPage<DatasetFile> files = datasetFileRepository.findByCriteria(datasetId, fileType, status, name, page);
         return PagedResponse.of(files);
     }
 
+    /**
+     * 获取数据集文件列表
+     */
+    @Transactional(readOnly = true)
+    public PagedResponse<DatasetFile> getDatasetFilesWithDirectory(String datasetId, String prefix, PagingQuery pagingQuery) {
+        Dataset dataset = datasetRepository.getById(datasetId);
+        int page = Math.max(pagingQuery.getPage(), 1);
+        int size = pagingQuery.getSize() == null || pagingQuery.getSize() < 0 ? 20 : pagingQuery.getSize();
+        if (dataset == null) {
+            return PagedResponse.of(new Page<>(page, size));
+        }
+        String datasetPath = dataset.getPath();
+        Path queryPath = Path.of(dataset.getPath() + File.separator + prefix);
+        Map<String, DatasetFile> datasetFilesMap = datasetFileRepository.findAllByDatasetId(datasetId)
+            .stream().collect(Collectors.toMap(DatasetFile::getFilePath, Function.identity()));
+        try (Stream<Path> pathStream = Files.list(queryPath)) {
+            List<Path> allFiles = pathStream
+                .filter(path -> path.toString().startsWith(datasetPath))
+                .sorted(Comparator
+                    .comparing((Path path) -> !Files.isDirectory(path))
+                    .thenComparing(path -> path.getFileName().toString()))
+                .collect(Collectors.toList());
+
+            // 计算分页
+            int total = allFiles.size();
+            int totalPages = (int) Math.ceil((double) total / size);
+
+            // 获取当前页数据
+            int fromIndex = (page - 1) * size;
+            fromIndex = Math.max(fromIndex, 0);
+            int toIndex = Math.min(fromIndex + size, total);
+
+            List<Path> pageData = new ArrayList<>();
+            if (fromIndex < total) {
+                pageData = allFiles.subList(fromIndex, toIndex);
+            }
+            List<DatasetFile> datasetFiles = pageData.stream().map(path -> getDatasetFile(path, datasetFilesMap)).toList();
+
+            return new PagedResponse<>(page, size, total, totalPages, datasetFiles);
+        } catch (IOException e) {
+            log.error("list dataset path error", e);
+            return PagedResponse.of(new Page<>(page, size));
+        }
+    }
+
+    private DatasetFile getDatasetFile(Path path, Map<String, DatasetFile> datasetFilesMap) {
+        DatasetFile datasetFile = new DatasetFile();
+        LocalDateTime localDateTime = LocalDateTime.now();
+        try {
+            localDateTime = Files.getLastModifiedTime(path).toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime();
+        } catch (IOException e) {
+            log.error("get last modified time error", e);
+        }
+        datasetFile.setFileName(path.getFileName().toString());
+        datasetFile.setUploadTime(localDateTime);
+        if (Files.isDirectory(path)) {
+            datasetFile.setId("directory-" + datasetFile.getFileName());
+        } else if (Objects.isNull(datasetFilesMap.get(path.toString()))) {
+            datasetFile.setId("file-" + datasetFile.getFileName());
+            datasetFile.setFileSize(path.toFile().length());
+        } else {
+            datasetFile = datasetFilesMap.get(path.toString());
+        }
+        return datasetFile;
+    }
+
     /**
      * 获取文件详情
      */
@@ -151,58 +221,61 @@ public Resource downloadFile(String datasetId, String fileId) {
      */
     @Transactional(readOnly = true)
     public void downloadDatasetFileAsZip(String datasetId, HttpServletResponse response) {
+        Dataset dataset = datasetRepository.getById(datasetId);
+        if (Objects.isNull(dataset)) {
+            throw BusinessException.of(DataManagementErrorCode.DATASET_NOT_FOUND);
+        }
         List<DatasetFile> allByDatasetId = datasetFileRepository.findAllByDatasetId(datasetId);
-        fileRename(allByDatasetId);
+        Set<String> filePaths = allByDatasetId.stream().map(DatasetFile::getFilePath).collect(Collectors.toSet());
+        String datasetPath = dataset.getPath();
+        Path downloadPath = Path.of(datasetPath);
         response.setContentType("application/zip");
         String zipName = String.format("dataset_%s.zip",
                 LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyyMMddHHmmss")));
         response.setHeader(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=" + zipName);
-        try (ZipOutputStream zos = new ZipOutputStream(response.getOutputStream())) {
-            for (DatasetFile file : allByDatasetId) {
-                addToZipFile(file, zos);
+        try (ZipArchiveOutputStream zos = new ZipArchiveOutputStream(response.getOutputStream())) {
+            try (Stream<Path> pathStream = Files.walk(downloadPath)) {
+                List<Path> allPaths = pathStream.filter(path -> path.toString().startsWith(datasetPath))
+                    .filter(path -> filePaths.stream().anyMatch(filePath -> filePath.startsWith(path.toString())))
+                    .toList();
+                for (Path path : allPaths) {
+                    addToZipFile(path, downloadPath, zos);
+                }
             }
         } catch (IOException e) {
             log.error("Failed to download files in batches.", e);
             throw BusinessException.of(SystemErrorCode.FILE_SYSTEM_ERROR);
         }
     }
 
-    private void fileRename(List<DatasetFile> files) {
-        Set<String> uniqueFilenames = new HashSet<>();
-        for (DatasetFile file : files) {
-            String originalFilename = file.getFileName();
-            if (!uniqueFilenames.add(originalFilename)) {
-                String newFilename;
-                int counter = 1;
-                do {
-                    newFilename = generateNewFilename(originalFilename, counter);
-                    counter++;
-                } while (!uniqueFilenames.add(newFilename));
-                file.setFileName(newFilename);
+    private void addToZipFile(Path path, Path basePath, ZipArchiveOutputStream zos) throws IOException {
+        String entryName = basePath.relativize(path)
+            .toString()
+            .replace(File.separator, "/");
+
+        // 处理目录
+        if (Files.isDirectory(path)) {
+            if (!entryName.isEmpty()) {
+                entryName += "/";
+                ZipArchiveEntry dirEntry = new ZipArchiveEntry(entryName);
+                zos.putArchiveEntry(dirEntry);
+                zos.closeArchiveEntry();
             }
-        }
-    }
+        } else {
+            // 处理文件
+            ZipArchiveEntry fileEntry = new ZipArchiveEntry(path.toFile(), entryName);
 
-    private String generateNewFilename(String oldFilename, int counter) {
-        int dotIndex = oldFilename.lastIndexOf(".");
-        return oldFilename.substring(0, dotIndex) + "-(" + counter + ")" + oldFilename.substring(dotIndex);
-    }
+            // 设置更多属性
+            BasicFileAttributes attrs = Files.readAttributes(path, BasicFileAttributes.class);
+            fileEntry.setSize(attrs.size());
+            fileEntry.setLastModifiedTime(attrs.lastModifiedTime());
 
-    private void addToZipFile(DatasetFile file, ZipOutputStream zos) throws IOException {
-        if (file.getFilePath() == null || !Files.exists(Paths.get(file.getFilePath()))) {
-            log.warn("The file hasn't been found on filesystem, id: {}", file.getId());
-            return;
-        }
-        try (InputStream fis = Files.newInputStream(Paths.get(file.getFilePath()));
-             BufferedInputStream bis = new BufferedInputStream(fis)) {
-            ZipEntry zipEntry = new ZipEntry(file.getFileName());
-            zos.putNextEntry(zipEntry);
-            byte[] buffer = new byte[8192];
-            int length;
-            while ((length = bis.read(buffer)) >= 0) {
-                zos.write(buffer, 0, length);
+            zos.putArchiveEntry(fileEntry);
+
+            try (InputStream is = Files.newInputStream(path)) {
+                IOUtils.copy(is, zos);
             }
-            zos.closeEntry();
+            zos.closeArchiveEntry();
         }
     }
 

backend/services/data-management-service/src/main/java/com/datamate/datamanagement/interfaces/rest/DatasetFileController.java

@@ -46,12 +46,10 @@ public Response<PagedResponse<DatasetFile>> getDatasetFiles(
             @PathVariable("datasetId") String datasetId,
             @RequestParam(value = "page", required = false, defaultValue = "0") Integer page,
             @RequestParam(value = "size", required = false, defaultValue = "20") Integer size,
-            @RequestParam(value = "fileType", required = false) String fileType,
-            @RequestParam(value = "status", required = false) String status,
-            @RequestParam(value = "name", required = false) String name) {
+            @RequestParam(value = "prefix", required = false) String prefix) {
         PagingQuery pagingQuery = new PagingQuery(page, size);
-        PagedResponse<DatasetFile> filesPage = datasetFileApplicationService.getDatasetFiles(
-                datasetId, fileType, status, name, pagingQuery);
+        PagedResponse<DatasetFile> filesPage = datasetFileApplicationService.getDatasetFilesWithDirectory(
+                datasetId, prefix, pagingQuery);
         return Response.ok(filesPage);
     }
 

backend/shared/domain-common/src/main/java/com/datamate/common/domain/utils/ArchiveAnalyzer.java

@@ -12,6 +12,7 @@
 import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
 import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream;
 import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
+import org.apache.commons.io.FileUtils;
 
 import java.io.BufferedInputStream;
 import java.io.BufferedOutputStream;
@@ -145,7 +146,7 @@ private static Optional<FileUploadResult> extractEntity(ArchiveInputStream<?> ar
         Path path = Paths.get(archivePath.getParent().toString(), archiveEntry.getName());
         File file = path.toFile();
         long fileSize = 0L;
-        String extension = AnalyzerUtils.getExtension(archiveEntry.getName());
+        FileUtils.createParentDirectories(file);
 
         long supportFileSize = 1024*1024*1024; // 上传大小暂定为1个G
         try (OutputStream outputStream = new BufferedOutputStream(Files.newOutputStream(file.toPath()))) {

deployment/helm/datamate/values.yaml

@@ -95,6 +95,24 @@ backend:
     - name: operator-volume
       mountPath: /operators
 
+backend-python:
+  env:
+    - name: DB_PASSWORD
+      value: *dbPass
+  volumes:
+    - *datasetVolume
+    - *flowVolume
+    - *logVolume
+  volumeMounts:
+    - name: dataset-volume
+      mountPath: /dataset
+    - name: flow-volume
+      mountPath: /flow
+    - name: log-volume
+      mountPath: /var/log/datamate
+    - name: operator-volume
+      mountPath: /operators
+
 frontend:
   service:
     type: NodePort