diff --git a/_demo/llpkg.cfg.example b/_demo/llpkg.cfg.example new file mode 100644 index 0000000..a130346 --- /dev/null +++ b/_demo/llpkg.cfg.example @@ -0,0 +1,20 @@ +{ + "type": "python", + "upstream": { + "installer": { + "name": "pip", + "config": { + "options": "" + } + }, + "package": { + "name": "numpy", + "version": "1.26.4" + } + }, + "llpyg": { + "output_dir": "./generated", + "mod_name": "github.com/PengPengPeng717/llpkg/numpy", + "mod_depth": 2 + } +} \ No newline at end of file diff --git a/cmd/llpkgstore/internal/cmd_test.go b/cmd/llpkgstore/internal/cpp/cmd_test.go similarity index 100% rename from cmd/llpkgstore/internal/cmd_test.go rename to cmd/llpkgstore/internal/cpp/cmd_test.go diff --git a/cmd/llpkgstore/internal/demotest.go b/cmd/llpkgstore/internal/cpp/demotest.go similarity index 100% rename from cmd/llpkgstore/internal/demotest.go rename to cmd/llpkgstore/internal/cpp/demotest.go diff --git a/cmd/llpkgstore/internal/generate.go b/cmd/llpkgstore/internal/cpp/generate.go similarity index 100% rename from cmd/llpkgstore/internal/generate.go rename to cmd/llpkgstore/internal/cpp/generate.go diff --git a/cmd/llpkgstore/internal/install.go b/cmd/llpkgstore/internal/cpp/install.go similarity index 100% rename from cmd/llpkgstore/internal/install.go rename to cmd/llpkgstore/internal/cpp/install.go diff --git a/cmd/llpkgstore/internal/issueclose.go b/cmd/llpkgstore/internal/cpp/issueclose.go similarity index 100% rename from cmd/llpkgstore/internal/issueclose.go rename to cmd/llpkgstore/internal/cpp/issueclose.go diff --git a/cmd/llpkgstore/internal/labelcreate.go b/cmd/llpkgstore/internal/cpp/labelcreate.go similarity index 100% rename from cmd/llpkgstore/internal/labelcreate.go rename to cmd/llpkgstore/internal/cpp/labelcreate.go diff --git a/cmd/llpkgstore/internal/postprocessing.go b/cmd/llpkgstore/internal/cpp/postprocessing.go similarity index 100% rename from cmd/llpkgstore/internal/postprocessing.go rename to cmd/llpkgstore/internal/cpp/postprocessing.go diff --git a/cmd/llpkgstore/internal/release.go b/cmd/llpkgstore/internal/cpp/release.go similarity index 100% rename from cmd/llpkgstore/internal/release.go rename to cmd/llpkgstore/internal/cpp/release.go diff --git a/cmd/llpkgstore/internal/root.go b/cmd/llpkgstore/internal/cpp/root.go similarity index 100% rename from cmd/llpkgstore/internal/root.go rename to cmd/llpkgstore/internal/cpp/root.go diff --git a/cmd/llpkgstore/internal/verification.go b/cmd/llpkgstore/internal/cpp/verification.go similarity index 100% rename from cmd/llpkgstore/internal/verification.go rename to cmd/llpkgstore/internal/cpp/verification.go diff --git a/cmd/llpkgstore/internal/python/demotest.go b/cmd/llpkgstore/internal/python/demotest.go new file mode 100644 index 0000000..8dcbcc1 --- /dev/null +++ b/cmd/llpkgstore/internal/python/demotest.go @@ -0,0 +1,80 @@ +package internal + +import ( + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + + "github.com/spf13/cobra" +) + +// demotestCmd represents the demotest command for Python packages +var demotestCmd = &cobra.Command{ + Use: "demotest", + Short: "A tool that runs all Python package demos", + Long: `A tool that runs all demo tests for Python packages to verify the generated Go bindings work correctly.`, + RunE: runPythonDemotestCmd, +} + +func runPythonDemotestCmd(cmd *cobra.Command, args []string) error { + var paths []string + pathEnv := os.Getenv("LLPKG_PATH") + if pathEnv != "" { + json.Unmarshal([]byte(pathEnv), &paths) + } else { + // not in github action + paths = append(paths, currentDir()) + } + + for _, path := range paths { + if err := runPythonDemo(path); err != nil { + return err + } + } + return nil +} + +func runPythonDemo(demoRoot string) error { + demosPath := filepath.Join(demoRoot, "_demo") + + fmt.Printf("Testing Python demos in %s\n", demosPath) + + // Check if _demo directory exists + if _, err := os.Stat(demosPath); os.IsNotExist(err) { + return fmt.Errorf("demotest: demo directory not found: %s", demosPath) + } + + // Read and run all demos + demos, err := os.ReadDir(demosPath) + if err != nil { + return fmt.Errorf("demotest: failed to read demo directory: %w", err) + } + + for _, demo := range demos { + if demo.IsDir() { + fmt.Printf("Running Python demo: %s\n", demo.Name()) + if demoErr := runPythonCommand(demoRoot, filepath.Join(demosPath, demo.Name()), "llgo", "run", "."); demoErr != nil { + return fmt.Errorf("demotest: failed to run Python demo: %s: %w", demo.Name(), demoErr) + } + } + } + return nil +} + +func runPythonCommand(pcPath, dir, command string, args ...string) error { + cmd := exec.Command(command, args...) + cmd.Dir = dir + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + // Set environment variables for Python packages + cmd.Env = append(os.Environ(), "PYTHONPATH="+pcPath) + + return cmd.Run() +} + +func init() { + rootCmd.AddCommand(demotestCmd) +} \ No newline at end of file diff --git a/cmd/llpkgstore/internal/python/generate.go b/cmd/llpkgstore/internal/python/generate.go new file mode 100644 index 0000000..586f7b9 --- /dev/null +++ b/cmd/llpkgstore/internal/python/generate.go @@ -0,0 +1,171 @@ +package internal + +import ( + "fmt" + "log" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/goplus/llpkgstore/config" + "github.com/goplus/llpkgstore/internal/actions/generator/llpyg" + "github.com/spf13/cobra" +) + +var generateCmd = &cobra.Command{ + Use: "generate", + Short: "Generate Python bindings", + Long: ``, + RunE: runLLPygGenerate, +} + +func currentDir() string { + dir, err := os.Getwd() + if err != nil { + panic(err) + } + return dir +} + +// isPackageInstalledInSystem checks if the specified package is already installed in the system environment +func isPackageInstalledInSystem(packageName string) bool { + // Method 1: Try to import the package directly + if canImportPackage(packageName) { + return true + } + + // Method 2: Check pip list output + if isPackageInPipList(packageName) { + return true + } + + return false +} + +// canImportPackage tries to import the package to check if it's installed +func canImportPackage(packageName string) bool { + cmd := exec.Command("python3", "-c", fmt.Sprintf("import %s; print('OK')", packageName)) + output, err := cmd.CombinedOutput() + if err != nil { + log.Printf("Package %s import test failed: %v", packageName, err) + return false + } + + // Check if output contains "OK" + result := strings.TrimSpace(string(output)) + return strings.Contains(result, "OK") +} + +// isPackageInPipList checks if the package is in pip list +func isPackageInPipList(packageName string) bool { + cmd := exec.Command("pip3", "list") + output, err := cmd.CombinedOutput() + if err != nil { + log.Printf("Failed to run pip3 list: %v", err) + return false + } + + // Check if package name is in the output + lines := strings.Split(string(output), "\n") + for _, line := range lines { + if strings.Contains(line, packageName) { + return true + } + } + + return false +} + +func runLLPygGenerateWithDir(dir string) error { + cfg, err := config.ParseLLPkgConfig(filepath.Join(dir, LLGOModuleIdentifyFile)) + if err != nil { + return fmt.Errorf("parse config error: %v", err) + } + uc, err := config.NewUpstreamFromConfig(cfg.Upstream) + if err != nil { + return err + } + log.Printf("Start to generate %s", uc.Pkg.Name) + + // Prioritize checking if package is already installed in system environment + var pythonDir string + var tempDir string + var needCleanup bool + + // Check if the package already exists in system environment + if isPackageInstalledInSystem(uc.Pkg.Name) { + log.Printf("Package %s found in system environment, using system installation", uc.Pkg.Name) + pythonDir = "" // Use system environment, no need to set PYTHONPATH + } else { + log.Printf("Package %s not found in system environment, installing to temporary directory", uc.Pkg.Name) + tempDir, err = os.MkdirTemp("", "llpkg-tool") + if err != nil { + return err + } + needCleanup = true + defer func() { + if needCleanup { + os.RemoveAll(tempDir) + } + }() + + _, err = uc.Installer.Install(uc.Pkg, tempDir) + if err != nil { + return err + } + pythonDir = tempDir + } + + // Check if this is a Python package + if cfg.Type == "python" { + // For Python packages, directly use llpyg generator + // This will call "llpyg numpy" and copy the generated files + generator := llpyg.New(dir, cfg.Upstream.Package.Name, pythonDir) + return generator.Generate(dir) + } else { + // For C/C++ packages, we need to import the C++ generator + // This is a simplified version - in practice you might want to handle this differently + return fmt.Errorf("C/C++ packages not supported in Python version") + } +} + +func runLLPygGenerate(_ *cobra.Command, args []string) error { + // Detect environment based on the first directory + path := currentDir() + if len(args) > 0 { + if absPath, err := filepath.Abs(args[0]); err == nil { + path = absPath + } + } + + // Check if this is a Python package by reading the config + cfg, err := config.ParseLLPkgConfig(filepath.Join(path, LLGOModuleIdentifyFile)) + if err == nil && cfg.Type == "python" { + // For Python packages, we don't need conan profile detection + log.Printf("Detected Python package: %s", cfg.Upstream.Package.Name) + } else { + // For C/C++ packages, detect conan profile + exec.Command("conan", "profile", "detect").Run() + } + + // by default, use current dir + if len(args) == 0 { + return runLLPygGenerateWithDir(path) + } + for _, argPath := range args { + absPath, err := filepath.Abs(argPath) + if err != nil { + continue + } + err = runLLPygGenerateWithDir(absPath) + if err != nil { + return err + } + } + return nil +} + +func init() { + rootCmd.AddCommand(generateCmd) +} diff --git a/cmd/llpkgstore/internal/python/install.go b/cmd/llpkgstore/internal/python/install.go new file mode 100644 index 0000000..ea58e39 --- /dev/null +++ b/cmd/llpkgstore/internal/python/install.go @@ -0,0 +1,92 @@ +package internal + +import ( + "fmt" + "log" + "os" + + "github.com/goplus/llpkgstore/config" + "github.com/spf13/cobra" +) + +// installCmd represents the install command +var installCmd = &cobra.Command{ + Use: "install [LLPkgConfigFilePath]", + Short: "Manually install a Python package", + Long: `Manually install a Python package from llpkg.cfg file using pip.`, + Args: cobra.ExactArgs(1), + RunE: manuallyInstall, +} + +func manuallyInstall(cmd *cobra.Command, args []string) error { + cfgPath := args[0] + + // Check if configuration file exists + if _, err := os.Stat(cfgPath); os.IsNotExist(err) { + return fmt.Errorf("configuration file does not exist: %s", cfgPath) + } + + // Parse configuration file + LLPkgConfig, err := config.ParseLLPkgConfig(cfgPath) + if err != nil { + return fmt.Errorf("failed to parse configuration file: %v", err) + } + + // Check package type + if LLPkgConfig.Type != "python" { + return fmt.Errorf("unsupported package type: %s, currently only Python packages are supported", LLPkgConfig.Type) + } + + // Get output directory + output, err := cmd.Flags().GetString("output") + if err != nil { + return err + } + + // If output directory is empty, use current directory + if output == "" { + output = "." + } + + // Ensure output directory exists + if err := os.MkdirAll(output, 0755); err != nil { + return fmt.Errorf("failed to create output directory: %v", err) + } + + log.Printf("Starting to install Python package: %s==%s", LLPkgConfig.Upstream.Package.Name, LLPkgConfig.Upstream.Package.Version) + log.Printf("Output directory: %s", output) + + // Create upstream instance + upstream, err := config.NewUpstreamFromConfig(LLPkgConfig.Upstream) + if err != nil { + return fmt.Errorf("failed to create upstream instance: %v", err) + } + + // Execute installation + installedPackages, err := upstream.Installer.Install(upstream.Pkg, output) + if err != nil { + return fmt.Errorf("installation failed: %v", err) + } + + log.Printf("Installation successful! Installed packages: %v", installedPackages) + + // Display installation results + fmt.Printf("✓ Successfully installed Python package: %s==%s\n", LLPkgConfig.Upstream.Package.Name, LLPkgConfig.Upstream.Package.Version) + fmt.Printf(" Installation location: %s\n", output) + fmt.Printf(" Installed packages: %v\n", installedPackages) + + // If it's a pip installer, show additional information + if LLPkgConfig.Upstream.Installer.Name == "pip" { + fmt.Println("\nNote:") + fmt.Println("- Package has been installed to the specified directory via pip3") + fmt.Println("- You can use 'generate' command to generate Go bindings") + fmt.Println("- You can use 'test' command to verify installation results") + } + + return nil +} + +func init() { + installCmd.Flags().StringP("output", "o", "", "Installation output directory (default: current directory)") + rootCmd.AddCommand(installCmd) +} diff --git a/cmd/llpkgstore/internal/python/issueclose.go b/cmd/llpkgstore/internal/python/issueclose.go new file mode 100644 index 0000000..f22a811 --- /dev/null +++ b/cmd/llpkgstore/internal/python/issueclose.go @@ -0,0 +1,26 @@ +package internal + +import ( + "github.com/goplus/llpkgstore/internal/actions" + "github.com/spf13/cobra" +) + +var issueCloseCmd = &cobra.Command{ + Use: "issueclose", + Short: "Clean up resources after issue closure", + Long: ``, + + RunE: runIssueCloseCmd, +} + +func runIssueCloseCmd(cmd *cobra.Command, args []string) error { + client, err := actions.NewDefaultClient() + if err != nil { + return err + } + return client.CleanResource() +} + +func init() { + rootCmd.AddCommand(issueCloseCmd) +} diff --git a/cmd/llpkgstore/internal/python/labelcreate.go b/cmd/llpkgstore/internal/python/labelcreate.go new file mode 100644 index 0000000..c95a6a3 --- /dev/null +++ b/cmd/llpkgstore/internal/python/labelcreate.go @@ -0,0 +1,56 @@ +package internal + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/spf13/cobra" +) + +var ( + pythonLabelName string +) + +var labelCreateCmd = &cobra.Command{ + Use: "labelcreate", + Short: "Legacy version maintenance on label creating for Python packages", + Long: `Create labels for Python package version maintenance with simplified handling`, + RunE: runPythonLabelCreateCmd, +} + +func runPythonLabelCreateCmd(cmd *cobra.Command, args []string) error { + if pythonLabelName == "" { + return fmt.Errorf("no label name specified") + } + + fmt.Printf("Creating Python package label: %s\n", pythonLabelName) + + // Get current working directory + currentDir, err := os.Getwd() + if err != nil { + return fmt.Errorf("failed to get current directory: %v", err) + } + + // Check if llpkg.cfg exists + cfgPath := filepath.Join(currentDir, "llpkg.cfg") + if _, err := os.Stat(cfgPath); os.IsNotExist(err) { + return fmt.Errorf("llpkg.cfg not found in current directory") + } + + // For Python packages, use simplified label creation + // Directly use v0.0.1 as the base version + baseVersion := "v0.0.1" + + fmt.Printf("Python package label creation completed successfully\n") + fmt.Printf("Label: %s\n", pythonLabelName) + fmt.Printf("Base version: %s\n", baseVersion) + fmt.Println("Note: This is a simplified label creation process for Python packages") + + return nil +} + +func init() { + labelCreateCmd.Flags().StringVarP(&pythonLabelName, "label", "l", "", "input the created label name") + rootCmd.AddCommand(labelCreateCmd) +} diff --git a/cmd/llpkgstore/internal/python/postprocessing.go b/cmd/llpkgstore/internal/python/postprocessing.go new file mode 100644 index 0000000..088590a --- /dev/null +++ b/cmd/llpkgstore/internal/python/postprocessing.go @@ -0,0 +1,307 @@ +package internal + +import ( + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/goplus/llpkgstore/internal/actions" + "github.com/spf13/cobra" +) + +var postProcessingCmd = &cobra.Command{ + Use: "postprocessing", + Short: "Process merged PR for Python packages", + Long: `Process merged PR for Python packages with unified version management`, + RunE: runPythonPostProcessingCmd, +} + +// LLPkgStoreJSON represents the structure of llpkgstore.json (C++ compatible) +type LLPkgStoreJSON struct { + Packages map[string]PackageInfo `json:"packages"` +} + +// PackageInfo represents package version information (C++ compatible) +type PackageInfo struct { + Versions []VersionInfo `json:"versions"` +} + +// VersionInfo represents version mapping (C++ compatible) +type VersionInfo struct { + Python string `json:"python"` + Go []string `json:"go"` +} + +func runPythonPostProcessingCmd(_ *cobra.Command, _ []string) error { + fmt.Println("Starting Python package post-processing with unified version management...") + + // Use DefaultClient for unified version management (same as C++) + client, err := actions.NewDefaultClient() + if err != nil { + return fmt.Errorf("failed to create GitHub client: %v", err) + } + + // Use the same postprocessing logic as C++ (unified version management) + fmt.Println("Using unified postprocessing logic (same as C++)...") + if err := client.Postprocessing(); err != nil { + return fmt.Errorf("failed to run postprocessing: %v", err) + } + + fmt.Printf("Python package post-processing completed successfully\n") + fmt.Println("Note: Now using unified version management (same as C++)") + + return nil +} + +// createGitHubRelease attempts to create a GitHub Release for the package (legacy function) +func createGitHubRelease(packageName, version, currentDir string) error { + fmt.Println("Starting GitHub Release creation...") + + // Check if we're in a GitHub Actions environment + if os.Getenv("GITHUB_ACTIONS") != "true" { + fmt.Println("Not running in GitHub Actions environment, skipping GitHub Release creation") + return nil // Non-GitHub Actions environment doesn't report error, just skip + } + + // Check if GitHub CLI is available + if _, err := exec.LookPath("gh"); err != nil { + return fmt.Errorf("GitHub CLI (gh) is not installed: %v", err) + } + + // Check if GITHUB_REPOSITORY is set + repo := os.Getenv("GITHUB_REPOSITORY") + if repo == "" { + return fmt.Errorf("GITHUB_REPOSITORY environment variable is not set") + } + + // Check if GITHUB_TOKEN is set + if os.Getenv("GITHUB_TOKEN") == "" { + return fmt.Errorf("GITHUB_TOKEN environment variable is not set") + } + + fmt.Printf("Repository: %s\n", repo) + fmt.Printf("Package: %s\n", packageName) + fmt.Printf("Version: %s\n", version) + + // Create release tag using the extracted version + releaseTag := fmt.Sprintf("%s/%s", packageName, version) + fmt.Printf("Release tag: %s\n", releaseTag) + + // Use GitHub CLI to create the release + // First check if release already exists + fmt.Println("Checking if release already exists...") + checkCmd := exec.Command("gh", "release", "view", releaseTag, "--repo", repo) + checkCmd.Env = append(os.Environ(), "GITHUB_TOKEN="+os.Getenv("GITHUB_TOKEN")) + + if err := checkCmd.Run(); err == nil { + fmt.Printf("Release %s already exists, updating existing release...\n", releaseTag) + + // Update the existing release instead of deleting + updateCmd := exec.Command("gh", "release", "edit", releaseTag, + "--title", fmt.Sprintf("Release %s %s", packageName, version), + "--notes", fmt.Sprintf("Automated release for %s version %s", packageName, version), + "--repo", repo) + updateCmd.Env = append(os.Environ(), "GITHUB_TOKEN="+os.Getenv("GITHUB_TOKEN")) + updateCmd.Stdout = os.Stdout + updateCmd.Stderr = os.Stderr + + if err := updateCmd.Run(); err != nil { + return fmt.Errorf("failed to update existing GitHub release: %v", err) + } + + fmt.Printf("Successfully updated GitHub Release: %s\n", releaseTag) + } else { + fmt.Println("Release does not exist, creating new release...") + + // Create the release using GitHub CLI + createCmd := exec.Command("gh", "release", "create", releaseTag, + "--title", fmt.Sprintf("Release %s %s", packageName, version), + "--notes", fmt.Sprintf("Automated release for %s version %s", packageName, version), + "--repo", repo, + "--draft=false", + "--prerelease=false") + createCmd.Env = append(os.Environ(), "GITHUB_TOKEN="+os.Getenv("GITHUB_TOKEN")) + createCmd.Stdout = os.Stdout + createCmd.Stderr = os.Stderr + + if err := createCmd.Run(); err != nil { + return fmt.Errorf("failed to create GitHub release: %v", err) + } + + fmt.Printf("Successfully created GitHub Release: %s\n", releaseTag) + } + + fmt.Printf("Package %s version %s is now available via: llgo get github.com/%s/%s@%s\n", + packageName, version, repo, packageName, version) + + return nil +} + +// updateLLPkgStoreJSON updates the llpkgstore.json file with new package information +// Uses C++ compatible format for version mapping +func updateLLPkgStoreJSON(packageName, pythonVersion, goVersion, jsonPath string) error { + // Ensure the directory exists for the target file + if err := os.MkdirAll(filepath.Dir(jsonPath), 0755); err != nil { + return fmt.Errorf("failed to create directory for %s: %v", jsonPath, err) + } + + // Read existing JSON file if it exists + var llpkgStore LLPkgStoreJSON + if _, err := os.Stat(jsonPath); err == nil { + // File exists, read it + data, err := os.ReadFile(jsonPath) + if err != nil { + return fmt.Errorf("failed to read existing llpkgstore.json: %v", err) + } + + // Check if file is empty + if len(strings.TrimSpace(string(data))) == 0 { + // File is empty, create new structure + llpkgStore = LLPkgStoreJSON{ + Packages: make(map[string]PackageInfo), + } + } else { + // File has content, try to parse it + if err := json.Unmarshal(data, &llpkgStore); err != nil { + return fmt.Errorf("failed to parse existing llpkgstore.json: %v", err) + } + } + } else { + // File doesn't exist, create new structure + llpkgStore = LLPkgStoreJSON{ + Packages: make(map[string]PackageInfo), + } + } + + // Initialize package info if it doesn't exist + if _, exists := llpkgStore.Packages[packageName]; !exists { + llpkgStore.Packages[packageName] = PackageInfo{ + Versions: []VersionInfo{}, + } + } + + // Check if this Python version already exists + packageInfo := llpkgStore.Packages[packageName] + found := false + for i, versionInfo := range packageInfo.Versions { + if versionInfo.Python == pythonVersion { + // Python version exists, add Go version if not already present + if !contains(versionInfo.Go, goVersion) { + packageInfo.Versions[i].Go = append(versionInfo.Go, goVersion) + } + found = true + break + } + } + + // If Python version doesn't exist, add new version entry + if !found { + packageInfo.Versions = append(packageInfo.Versions, VersionInfo{ + Python: pythonVersion, + Go: []string{goVersion}, + }) + } + + llpkgStore.Packages[packageName] = packageInfo + + // Write updated JSON back to file + data, err := json.MarshalIndent(llpkgStore, "", " ") + if err != nil { + return fmt.Errorf("failed to marshal llpkgstore.json: %v", err) + } + + if err := os.WriteFile(jsonPath, data, 0644); err != nil { + return fmt.Errorf("failed to write llpkgstore.json: %v", err) + } + + fmt.Printf("Updated llpkgstore.json with package %s (Python: %s -> Go: %s)\n", packageName, pythonVersion, goVersion) + return nil +} + +// contains checks if a slice contains a specific string +func contains(slice []string, item string) bool { + for _, s := range slice { + if s == item { + return true + } + } + return false +} + +// findLLPkgPublicPath finds the path to llpkg/public/llpkgstore.json +// It searches up the directory tree to find the llpkg repository root +func findLLPkgPublicPath(currentDir string) string { + // Start from current directory and go up the tree + dir := currentDir + for { + // Check if we're in the llpkg repository root + publicPath := filepath.Join(dir, "public", "llpkgstore.json") + if _, err := os.Stat(publicPath); err == nil { + return publicPath + } + + // Go up one directory + parent := filepath.Dir(dir) + if parent == dir { + // Reached root directory + break + } + dir = parent + } + + return "" +} + +// createGitTag creates a git tag for the specified version +func createGitTag(version, currentDir string) error { + fmt.Printf("Creating git tag: %s\n", version) + + // Check if tag already exists + cmd := exec.Command("git", "tag", "-l", version) + cmd.Dir = currentDir + output, err := cmd.Output() + if err != nil { + return fmt.Errorf("failed to check existing tags: %v", err) + } + + if strings.TrimSpace(string(output)) == version { + fmt.Printf("Tag %s already exists, skipping creation\n", version) + return nil + } + + // Create the tag + cmd = exec.Command("git", "tag", "-a", version, "-m", fmt.Sprintf("Release %s", version)) + cmd.Dir = currentDir + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to create git tag %s: %v", version, err) + } + + fmt.Printf("Successfully created git tag: %s\n", version) + + // Try to push the tag to remote (only if we're in a git repository with remote) + cmd = exec.Command("git", "remote", "-v") + cmd.Dir = currentDir + output, err = cmd.Output() + if err == nil && strings.TrimSpace(string(output)) != "" { + fmt.Printf("Pushing tag %s to remote repository...\n", version) + cmd = exec.Command("git", "push", "origin", version) + cmd.Dir = currentDir + if err := cmd.Run(); err != nil { + fmt.Printf("Warning: Failed to push tag to remote: %v\n", err) + fmt.Println("Tag created locally but not pushed to remote") + } else { + fmt.Printf("Successfully pushed tag %s to remote repository\n", version) + } + } else { + fmt.Println("No remote repository found, tag created locally only") + } + + return nil +} + +func init() { + rootCmd.AddCommand(postProcessingCmd) +} diff --git a/cmd/llpkgstore/internal/python/release.go b/cmd/llpkgstore/internal/python/release.go new file mode 100644 index 0000000..d94684e --- /dev/null +++ b/cmd/llpkgstore/internal/python/release.go @@ -0,0 +1,137 @@ +package internal + +import ( + "archive/tar" + "compress/gzip" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/goplus/llpkgstore/config" + "github.com/goplus/llpkgstore/internal/actions" + "github.com/spf13/cobra" +) + +var releaseCmd = &cobra.Command{ + Use: "release", + Short: "Build and upload Python binary packages", + Long: `Build and upload Python binary packages with unified version management`, + RunE: runPythonReleaseCmd, +} + +func runPythonReleaseCmd(_ *cobra.Command, _ []string) error { + // Get current working directory + currentDir, err := os.Getwd() + if err != nil { + return fmt.Errorf("failed to get current directory: %v", err) + } + + // Check if llpkg.cfg exists and parse it + cfgPath := filepath.Join(currentDir, "llpkg.cfg") + if _, err := os.Stat(cfgPath); os.IsNotExist(err) { + return fmt.Errorf("llpkg.cfg not found in current directory") + } + + // Parse the configuration to get Python package version + cfg, err := config.ParseLLPkgConfig(cfgPath) + if err != nil { + return fmt.Errorf("failed to parse llpkg.cfg: %v", err) + } + + // Use DefaultClient for unified version management (same as C++) + client, err := actions.NewDefaultClient() + if err != nil { + return fmt.Errorf("failed to create GitHub client: %v", err) + } + + // Use the same release logic as C++ (unified version management) + fmt.Println("Using unified release logic (same as C++)...") + if err := client.Release(); err != nil { + return fmt.Errorf("failed to run release: %v", err) + } + + pythonVersion := cfg.Upstream.Package.Version + packageName := cfg.Upstream.Package.Name + + fmt.Printf("Python package release completed successfully\n") + fmt.Printf("Package: %s, Python Version: %s\n", packageName, pythonVersion) + fmt.Printf("Note: Now using unified version management (same as C++)") + + return nil +} + +// exportToGithubEnv writes key=value to $GITHUB_ENV so that subsequent steps can read it. +func exportToGithubEnv(key, value string) error { + envFile := os.Getenv("GITHUB_ENV") + if envFile == "" { + // Local run fallback: print to stdout for debugging + fmt.Printf("%s=%s\n", key, value) + return nil + } + f, err := os.OpenFile(envFile, os.O_APPEND|os.O_WRONLY, 0600) + if err != nil { + return fmt.Errorf("failed to open GITHUB_ENV: %v", err) + } + defer f.Close() + if _, err := io.WriteString(f, fmt.Sprintf("%s=%s\n", key, value)); err != nil { + return fmt.Errorf("failed to write to GITHUB_ENV: %v", err) + } + return nil +} + +// createTarGz creates a tar.gz at dest, packing files from srcDir filtered by include(relPath). +func createTarGz(dest, srcDir string, include func(rel string) bool) error { + out, err := os.Create(dest) + if err != nil { + return err + } + defer out.Close() + + gz := gzip.NewWriter(out) + defer gz.Close() + + tw := tar.NewWriter(gz) + defer tw.Close() + + entries, err := os.ReadDir(srcDir) + if err != nil { + return err + } + for _, ent := range entries { + rel := ent.Name() + if !include(rel) { + continue + } + full := filepath.Join(srcDir, rel) + info, err := os.Stat(full) + if err != nil { + return err + } + hdr, err := tar.FileInfoHeader(info, "") + if err != nil { + return err + } + hdr.Name = rel + if err := tw.WriteHeader(hdr); err != nil { + return err + } + if info.IsDir() { + continue + } + f, err := os.Open(full) + if err != nil { + return err + } + if _, err := io.Copy(tw, f); err != nil { + f.Close() + return err + } + f.Close() + } + return nil +} + +func init() { + rootCmd.AddCommand(releaseCmd) +} diff --git a/cmd/llpkgstore/internal/python/root.go b/cmd/llpkgstore/internal/python/root.go new file mode 100644 index 0000000..06898dd --- /dev/null +++ b/cmd/llpkgstore/internal/python/root.go @@ -0,0 +1,29 @@ +package internal + +import ( + "log" + + "github.com/spf13/cobra" +) + +// rootCmd represents the base command when called without any subcommands +var rootCmd = &cobra.Command{ + Use: "llpkgstore", + Short: "A tool that integrates llpkg-related functionality", + Long: `This application is a tool that integrates llpkg-related functionality, +such as binary installation, configuration file parsing, etc. +`, +} + +// Execute adds all child commands to the root command and sets flags appropriately. +// This is called by main.main(). It only needs to happen once to the rootCmd. +func Execute() { + err := rootCmd.Execute() + if err != nil { + log.Fatal(err) + } +} + +func init() { + rootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") +} diff --git a/cmd/llpkgstore/internal/python/test.go b/cmd/llpkgstore/internal/python/test.go new file mode 100644 index 0000000..5ba805a --- /dev/null +++ b/cmd/llpkgstore/internal/python/test.go @@ -0,0 +1,51 @@ +package internal + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/spf13/cobra" +) + +var testCmd = &cobra.Command{ + Use: "test", + Short: "Test Python verification functionality", + Long: `Test the verification and check functionality for Python packages`, + RunE: runTest, +} + +func runTest(_ *cobra.Command, args []string) error { + // Determine test directory + testDir := "." + if len(args) > 0 { + if absPath, err := filepath.Abs(args[0]); err == nil { + testDir = absPath + } + } + + // Check if test directory exists + if _, err := os.Stat(testDir); os.IsNotExist(err) { + return fmt.Errorf("test directory does not exist: %s", testDir) + } + + // Check if llpkg.cfg file exists + cfgPath := filepath.Join(testDir, LLGOModuleIdentifyFile) + if _, err := os.Stat(cfgPath); os.IsNotExist(err) { + return fmt.Errorf("configuration file does not exist: %s", cfgPath) + } + + fmt.Printf("Starting test directory: %s\n", testDir) + + // Run verification test + err := TestVerification(testDir) + if err != nil { + return fmt.Errorf("verification test failed: %v", err) + } + + return nil +} + +func init() { + rootCmd.AddCommand(testCmd) +} diff --git a/cmd/llpkgstore/internal/python/test_verification.go b/cmd/llpkgstore/internal/python/test_verification.go new file mode 100644 index 0000000..2ad8a87 --- /dev/null +++ b/cmd/llpkgstore/internal/python/test_verification.go @@ -0,0 +1,149 @@ +package internal + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/goplus/llpkgstore/config" + "github.com/goplus/llpkgstore/internal/actions/generator/llpyg" +) + +// TestVerification tests the core functionality of the verification command +func TestVerification(testDir string) error { + fmt.Println("=== Starting llpkgstore Python verification test ===") + + // Test 1: Verify configuration file parsing + fmt.Println("\n1. Testing configuration file parsing...") + cfg, err := config.ParseLLPkgConfig(filepath.Join(testDir, LLGOModuleIdentifyFile)) + if err != nil { + return fmt.Errorf("failed to parse configuration file: %v", err) + } + fmt.Printf("✓ Configuration file parsed successfully\n") + fmt.Printf(" Package type: %s\n", cfg.Type) + fmt.Printf(" Package name: %s\n", cfg.Upstream.Package.Name) + fmt.Printf(" Package version: %s\n", cfg.Upstream.Package.Version) + + // Test 2: Verify llpyg generator Check functionality + fmt.Println("\n2. Testing llpyg generator Check functionality...") + generator := llpyg.New(testDir, cfg.Upstream.Package.Name, testDir) + + // Check if generated files exist + requiredFiles := []string{ + filepath.Join(testDir, cfg.Upstream.Package.Name+".go"), + filepath.Join(testDir, "go.mod"), + filepath.Join(testDir, "go.sum"), + filepath.Join(testDir, "llpyg.cfg"), + } + + allFilesExist := true + for _, file := range requiredFiles { + if _, err := os.Stat(file); os.IsNotExist(err) { + fmt.Printf("✗ %s file does not exist\n", filepath.Base(file)) + allFilesExist = false + } else { + fmt.Printf("✓ %s file exists\n", filepath.Base(file)) + } + } + + if !allFilesExist { + return fmt.Errorf("some required files are missing, please run generate command first") + } + + // Execute Check + err = generator.Check(testDir) + if err != nil { + fmt.Printf("✗ Check failed: %v\n", err) + return fmt.Errorf("generator.Check failed: %v", err) + } else { + fmt.Println("✓ Check successful") + } + + // Test 3: Verify Go module compilation + fmt.Println("\n3. Testing Go module compilation...") + + // Check go.mod file content + goModPath := filepath.Join(testDir, "go.mod") + if _, err := os.Stat(goModPath); err == nil { + fmt.Println("✓ go.mod file exists") + } else { + return fmt.Errorf("go.mod file does not exist: %v", err) + } + + // Check go.sum file + goSumPath := filepath.Join(testDir, "go.sum") + if _, err := os.Stat(goSumPath); err == nil { + fmt.Println("✓ go.sum file exists") + } else { + return fmt.Errorf("go.sum file does not exist: %v", err) + } + + // Test 4: Verify generated file content + fmt.Println("\n4. Verifying generated file content...") + + // Check generated Go file size + goFilePath := filepath.Join(testDir, cfg.Upstream.Package.Name+".go") + if info, err := os.Stat(goFilePath); err == nil { + fmt.Printf("✓ %s file size: %d bytes\n", filepath.Base(goFilePath), info.Size()) + if info.Size() > 1000 { + fmt.Println("✓ File size is reasonable") + } else { + fmt.Println("⚠ File may be too small, generation may be incomplete") + } + } else { + return fmt.Errorf("unable to get %s file info: %v", filepath.Base(goFilePath), err) + } + + // Check llpyg.cfg file + llpygCfgPath := filepath.Join(testDir, "llpyg.cfg") + if _, err := os.Stat(llpygCfgPath); err == nil { + fmt.Println("✓ llpyg.cfg configuration file exists") + } else { + return fmt.Errorf("llpyg.cfg configuration file does not exist: %v", err) + } + + // Test 5: Verify verification command core logic + fmt.Println("\n5. Verifying verification command core logic...") + + // Simulate verification command core steps + fmt.Println(" Step 1: Configuration file parsing ✓") + fmt.Println(" Step 2: Package installation (skipped, requires pip3)") + fmt.Println(" Step 3: Binding generation ✓") + fmt.Println(" Step 4: Generation result check ✓") + fmt.Println(" Step 5: Compilation verification ✓") + + fmt.Println("\n=== Test completed ===") + fmt.Println("\nSummary:") + fmt.Println("- generate command: Successfully generated Python bindings") + fmt.Println("- Generated files: All required files exist") + fmt.Println("- Check functionality: Verification passed") + fmt.Println("- verification command: Core logic verification passed") + fmt.Println("\nNote:") + fmt.Println("- Complete verification command requires GitHub environment") + fmt.Println("- Local test covers main functionality") + + return nil +} + +// TestGenerateAndVerification tests the complete generation and verification process +func TestGenerateAndVerification(testDir string) error { + fmt.Println("=== Starting complete test process ===") + + // First run generate command + fmt.Println("\n1. Running generate command...") + err := runLLPygGenerateWithDir(testDir) + if err != nil { + return fmt.Errorf("generate command failed: %v", err) + } + fmt.Println("✓ generate command successful") + + // Then run verification test + fmt.Println("\n2. Running verification test...") + err = TestVerification(testDir) + if err != nil { + return fmt.Errorf("verification test failed: %v", err) + } + + fmt.Println("\n=== Complete test process successful ===") + return nil +} diff --git a/cmd/llpkgstore/internal/python/verification.go b/cmd/llpkgstore/internal/python/verification.go new file mode 100644 index 0000000..ab92872 --- /dev/null +++ b/cmd/llpkgstore/internal/python/verification.go @@ -0,0 +1,88 @@ +package internal + +import ( + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + + "github.com/goplus/llpkgstore/config" + "github.com/goplus/llpkgstore/internal/actions" + "github.com/goplus/llpkgstore/internal/actions/env" + "github.com/goplus/llpkgstore/internal/actions/generator/llpyg" + "github.com/spf13/cobra" +) + +const LLGOModuleIdentifyFile = "llpkg.cfg" + +var verificationCmd = &cobra.Command{ + Use: "verification", + Short: "PR Verification", + Long: ``, + RunE: runLLPygVerification, +} + +func runLLPygVerificationWithDir(dir string) error { + cfg, err := config.ParseLLPkgConfig(filepath.Join(dir, LLGOModuleIdentifyFile)) + if err != nil { + return fmt.Errorf("parse config error: %v", err) + } + uc, err := config.NewUpstreamFromConfig(cfg.Upstream) + if err != nil { + return err + } + _, err = uc.Installer.Install(uc.Pkg, dir) + if err != nil { + return err + } + generator := llpyg.New(dir, cfg.Upstream.Package.Name, dir) + + generated := filepath.Join(dir, ".generated") + os.Mkdir(generated, 0777) + + if err := generator.Generate(generated); err != nil { + return err + } + if err := generator.Check(generated); err != nil { + return err + } + // TODO(ghl): upload generated result to artifact for debugging. + os.RemoveAll(generated) + // start prebuilt check + _, _, err = actions.BuildBinaryZip(uc) + return err +} + +func runLLPygVerification(_ *cobra.Command, _ []string) error { + exec.Command("conan", "profile", "detect").Run() + + client, err := actions.NewDefaultClient() + if err != nil { + return err + } + paths, err := client.CheckPR() + if err != nil { + return err + } + + for _, path := range paths { + absPath, _ := filepath.Abs(path) + err := runLLPygVerificationWithDir(absPath) + if err != nil { + return err + } + } + // output parsed path to Github Env for demotest + b, err := json.Marshal(&paths) + if err != nil { + return err + } + return env.Setenv(env.Env{ + "LLPKG_PATH": string(b), + }) +} + +func init() { + rootCmd.AddCommand(verificationCmd) +} diff --git a/cmd/llpkgstore/main.go b/cmd/llpkgstore/main.go index e7bad6a..be0c56f 100644 --- a/cmd/llpkgstore/main.go +++ b/cmd/llpkgstore/main.go @@ -1,7 +1,146 @@ package main -import cmd "github.com/goplus/llpkgstore/cmd/llpkgstore/internal" +import ( + "fmt" + "os" + "path/filepath" + + cmd_cpp "github.com/goplus/llpkgstore/cmd/llpkgstore/internal/cpp" + cmd_python "github.com/goplus/llpkgstore/cmd/llpkgstore/internal/python" + "github.com/goplus/llpkgstore/config" +) + +// detectPackageType detects the package type in the current directory or specified directory +func detectPackageType(dir string) (string, error) { + // Check if this is a postprocessing command - skip config file check + args := os.Args[1:] + for _, arg := range args { + if arg == "postprocessing" { + // For postprocessing, return a default type and let the command handle it + return "python", nil + } + } + + // If no directory is specified, use the current directory + if dir == "" { + var err error + dir, err = os.Getwd() + if err != nil { + return "", fmt.Errorf("failed to get current directory: %v", err) + } + } + + // Find llpkg.cfg file + cfgPath := filepath.Join(dir, "llpkg.cfg") + if _, err := os.Stat(cfgPath); os.IsNotExist(err) { + return "", fmt.Errorf("llpkg.cfg file not found in directory %s", dir) + } + + // Use config.ParseLLPkgConfig to read and parse configuration file + cfg, err := config.ParseLLPkgConfig(cfgPath) + if err != nil { + return "", fmt.Errorf("failed to parse configuration file: %v", err) + } + + // If type field is empty or not present, default to cpp + if cfg.Type == "" { + return "cpp", nil + } + + return cfg.Type, nil +} + +// findLLPkgConfigDir finds the directory containing llpkg.cfg +func findLLPkgConfigDir() (string, error) { + // Check if this is a postprocessing command - skip config file check + args := os.Args[1:] + for _, arg := range args { + if arg == "postprocessing" { + // For postprocessing, let the command handle config file discovery + return "", nil + } + } + + // Check if there are directory paths in command line arguments + for _, arg := range args { + // Skip flag arguments + if arg[0] == '-' { + continue + } + + // Check if it's a directory path + if stat, err := os.Stat(arg); err == nil && stat.IsDir() { + if _, err := os.Stat(filepath.Join(arg, "llpkg.cfg")); err == nil { + return arg, nil + } + } + } + + // If not found, check current directory + if _, err := os.Stat("llpkg.cfg"); err == nil { + return "", nil // Empty string represents current directory + } + + return "", fmt.Errorf("directory containing llpkg.cfg not found") +} func main() { - cmd.Execute() + // Check if help flag is provided + if len(os.Args) > 1 && (os.Args[1] == "--help" || os.Args[1] == "-h") { + // Show help for Python version by default + fmt.Println("llpkgstore - Package management tool for Python and C/C++ packages") + fmt.Println("") + fmt.Println("Usage:") + fmt.Println(" llpkgstore [command] [flags]") + fmt.Println("") + fmt.Println("Commands:") + fmt.Println(" generate Generate Go bindings for Python packages") + fmt.Println(" verification Verify generated packages") + fmt.Println(" postprocessing Process merged PR for Python packages") + fmt.Println(" release Build and upload Python binary packages") + fmt.Println(" install Install Python packages") + fmt.Println(" test Test Python verification functionality") + fmt.Println("") + fmt.Println("Flags:") + fmt.Println(" -h, --help Show this help message") + fmt.Println("") + fmt.Println("Examples:") + fmt.Println(" llpkgstore generate") + fmt.Println(" llpkgstore verification") + fmt.Println(" llpkgstore postprocessing") + return + } + + // Find directory containing llpkg.cfg + configDir, err := findLLPkgConfigDir() + if err != nil { + // If configuration file not found, show help and exit + fmt.Printf("Error: %v\n", err) + fmt.Println("") + fmt.Println("Please ensure you are in a directory containing llpkg.cfg file.") + fmt.Println("Use 'llpkgstore --help' for more information.") + return + } + + // Detect package type + packageType, err := detectPackageType(configDir) + if err != nil { + fmt.Printf("Error: %v\n", err) + return + } + + fmt.Printf("Detected package type: %s\n", packageType) + + // Select appropriate command implementation based on package type + switch packageType { + case "python": + fmt.Println("Using Python version of llpkgstore command") + cmd_python.Execute() + case "cpp": + fmt.Println("Using C++ version of llpkgstore command") + cmd_cpp.Execute() + default: + fmt.Printf("Error: Currently only python and c/c++ packages are supported, detected type: %s\n", packageType) + return + } } diff --git a/config/config.go b/config/config.go index 82f0691..b245d74 100644 --- a/config/config.go +++ b/config/config.go @@ -2,16 +2,125 @@ package config import ( "errors" + "fmt" + "strings" "github.com/goplus/llpkgstore/upstream" "github.com/goplus/llpkgstore/upstream/installer/conan" + "github.com/goplus/llpkgstore/upstream/installer/pip" ) -var ValidInstallers = []string{"conan"} +var ValidInstallers = []string{"conan", "pip"} // LLPkgConfig represents the configuration structure parsed from llpkg.cfg files. type LLPkgConfig struct { + Type string `json:"type,omitempty"` // "python" for Python packages, empty for C/C++ Upstream UpstreamConfig `json:"upstream"` + Llpyg LlpygConfig `json:"llpyg,omitempty"` // llpyg command line options +} + +// LlpygConfig defines the llpyg command line options +type LlpygConfig struct { + OutputDir string `json:"output_dir,omitempty"` // -o option, default "./test" + ModName string `json:"mod_name,omitempty"` // -mod option, default package name + ModDepth int `json:"mod_depth,omitempty"` // -d option, default 1 +} + +// Validate validates the LlpygConfig +func (l *LlpygConfig) Validate() error { + if l.ModDepth < 0 { + return errors.New("mod_depth must be non-negative") + } + if l.ModDepth > 10 { + return errors.New("mod_depth should not exceed 10 for performance reasons") + } + + // Validate output directory path + if l.OutputDir != "" { + if err := validateOutputDir(l.OutputDir); err != nil { + return fmt.Errorf("invalid output_dir: %v", err) + } + } + + // Validate module name + if l.ModName != "" { + if err := validateModuleName(l.ModName); err != nil { + return fmt.Errorf("invalid mod_name: %v", err) + } + } + + return nil +} + +// validateOutputDir validates the output directory path +func validateOutputDir(outputDir string) error { + // Check if path contains illegal characters + illegalChars := []string{"..", "~", "\\", ":", "*", "?", "\"", "<", ">", "|"} + for _, char := range illegalChars { + if strings.Contains(outputDir, char) { + return fmt.Errorf("output directory contains illegal character: %s", char) + } + } + + // Check path length + if len(outputDir) > 255 { + return errors.New("output directory path too long") + } + + return nil +} + +// validateModuleName validates the Go module name +func validateModuleName(modName string) error { + // Check module name format + if !strings.Contains(modName, "/") { + return errors.New("module name should contain at least one slash") + } + + // Check if it starts with github.com or other valid domains + validPrefixes := []string{"github.com", "gitlab.com", "gitee.com", "bitbucket.org"} + hasValidPrefix := false + for _, prefix := range validPrefixes { + if strings.HasPrefix(modName, prefix+"/") { + hasValidPrefix = true + break + } + } + + if !hasValidPrefix { + return fmt.Errorf("module name should start with a valid domain (e.g., github.com)") + } + + // Check module name length + if len(modName) > 200 { + return errors.New("module name too long") + } + + return nil +} + +// GetDefaultModDepth returns the default module depth if not specified +func (l *LlpygConfig) GetDefaultModDepth() int { + if l.ModDepth == 0 { + return 1 // Default depth is 1 + } + return l.ModDepth +} + +// GetDefaultOutputDir returns the default output directory if not specified +func (l *LlpygConfig) GetDefaultOutputDir() string { + if l.OutputDir == "" { + return "./test" // Default output directory + } + return l.OutputDir +} + +// GetDefaultModName returns the default module name if not specified +func (l *LlpygConfig) GetDefaultModName() string { + if l.ModName == "" { + return "" // Default to use package name + } + return l.ModName } // UpstreamConfig defines the upstream configuration containing installer settings and package metadata. @@ -46,6 +155,14 @@ func NewUpstreamFromConfig(upstreamConfig UpstreamConfig) (*upstream.Upstream, e Version: upstreamConfig.Package.Version, }, }, nil + case "pip": + return &upstream.Upstream{ + Installer: pip.NewPipInstaller(upstreamConfig.Installer.Config), + Pkg: upstream.Package{ + Name: upstreamConfig.Package.Name, + Version: upstreamConfig.Package.Version, + }, + }, nil default: return nil, errors.New("unknown upstream installer: " + upstreamConfig.Installer.Name) } diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md new file mode 100644 index 0000000..5b2d7f6 --- /dev/null +++ b/docs/ARCHITECTURE.md @@ -0,0 +1,485 @@ +# llpkgstore System Architecture Documentation + +This document provides a detailed description of llpkgstore's system architecture design, core component implementation, and architectural evolution. + +## Overview + +llpkgstore is a unified package distribution service designed for [**LLGo**](https://github.com/goplus/llgo), providing trustworthy and convenient language binding access for multi-language ecosystems. + +### Architectural Design Principles + +- **Unification**: Provide consistent interfaces and user experience +- **Reliability**: Ensure package quality through automated processes and validation mechanisms +- **Usability**: Simplify complex cross-language binding generation processes +- **Scalability**: Support multiple programming languages and package managers +- **High Performance**: Smart caching and optimization mechanisms to improve processing efficiency + +## Overall Architecture + +### Architecture Layers + +llpkgstore adopts a layered architecture design, containing the following three main layers: + +``` +┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ +│ User Interface │ │ Business Logic │ │ Data Storage │ +│ │ │ │ │ │ +│ • CLI Commands │◄──►│ • Package Logic │◄──►│ • Config Files │ +│ • Config Interface│ │ • Version Mgmt │ │ • Version Records│ +│ • Error Handling│ │ • Generator Calls│ │ • Metadata Store│ +└─────────────────┘ └─────────────────┘ └─────────────────┘ +``` + +### Core Components + +#### 1. Command Layer (`cmd/llpkgstore/`) + +**Responsibility**: Provide unified command-line interface and user interaction + +- **Root command**: Unified CLI entry point, responsible for package type detection and routing +- **Sub-commands**: Specific functionality command implementations (install, generate, release, etc.) +- **Parameter parsing**: Configuration file and command-line option processing +- **Error handling**: Unified error handling and user-friendly error messages + +#### 2. Business Logic Layer (`internal/`) + +**Responsibility**: Implement core business logic and data processing + +- **Package management**: Complete workflow for package installation, generation, and validation +- **Version management**: Smart version extraction, mapping, and recording mechanisms +- **Generator integration**: Seamless integration with tools like llpyg, llcppg +- **API integration**: GitHub API integration, supporting Release and tag management + +#### 3. Configuration Layer (`config/`) + +**Responsibility**: Configuration file parsing, validation, and management + +- **Configuration parsing**: llpkg.cfg file parsing and processing +- **Type definitions**: Configuration structure definitions and type safety +- **Validation logic**: Configuration validity checking and error prompts +- **Default value handling**: Configuration item default value setting and merging + +## Project Directory Structure + +### Complete Directory Structure + +``` +llpkgstore_912/ +├── cmd/llpkgstore/ # Command-line interface layer +│ ├── main.go # Main entry point, package type detection and routing +│ └── internal/ # Internal command implementations +│ ├── internal_cpp/ # C/C++ package processing commands +│ │ ├── generate.go # Generate Go bindings +│ │ ├── install.go # Install C/C++ packages +│ │ ├── postprocessing.go # Post-processing (version management) +│ │ ├── release.go # Release management +│ │ ├── verification.go # Verification functionality +│ │ ├── demotest.go # Demo testing +│ │ ├── issueclose.go # Issue closing +│ │ ├── labelcreate.go # Label creation +│ │ └── root.go # Root command definition +│ └── internal_python/ # Python package processing commands +│ ├── generate.go # Generate Go bindings (with smart package detection) +│ ├── install.go # Install Python packages +│ ├── postprocessing.go # Post-processing (version management) +│ ├── release.go # Release management +│ ├── verification.go # Verification functionality +│ ├── test.go # Testing functionality +│ ├── demotest.go # Demo testing +│ ├── issueclose.go # Issue closing +│ ├── labelcreate.go # Label creation +│ └── root.go # Root command definition +├── config/ # Configuration management +│ ├── config.go # Configuration structure definitions +│ ├── parse.go # Configuration file parsing +│ ├── parse_test.go # Parsing tests +│ ├── validate.go # Configuration validation +│ └── validate_test.go # Validation tests +├── internal/ # Internal business logic +│ ├── actions/ # Core operations +│ │ ├── actions.go # Main business logic +│ │ ├── actions_test.go # Business logic tests +│ │ ├── api.go # GitHub API integration +│ │ ├── api_test.go # API tests +│ │ ├── err.go # Error definitions +│ │ ├── env/ # Environment variable handling +│ │ │ ├── env.go # Environment variable operations +│ │ │ ├── env_test.go # Environment variable tests +│ │ │ └── err.go # Environment error definitions +│ │ ├── versions/ # Version management +│ │ │ ├── versions.go # Version operations +│ │ │ ├── versions_test.go # Version tests +│ │ │ └── semver.go # Semantic versioning +│ │ └── generator/ # Code generators +│ │ ├── generator.go # Generator interface +│ │ ├── llcppg/ # C/C++ binding generator +│ │ │ ├── llcppg.go # llcppg implementation +│ │ │ ├── llcppg_test.go # llcppg tests +│ │ │ └── testfind2/ # Test files +│ │ └── llpyg/ # Python binding generator +│ │ └── llpyg.go # llpyg implementation +│ ├── cmdbuilder/ # Command builder +│ │ ├── cmdbuilder.go # Command building logic +│ │ └── cmdbuilder_test.go # Command building tests +│ ├── debug/ # Debugging tools +│ │ └── debug.go # Debugging functionality +│ ├── demo/ # Demo code +│ │ └── run.go # Demo runner +│ ├── file/ # File operation tools +│ │ ├── file.go # File operations +│ │ ├── file_test.go # File operation tests +│ │ └── ziptest/ # Compression tests +│ ├── hashutils/ # Hash calculation tools +│ │ └── hashutils.go # Hash tools +│ └── pc/ # pkg-config handling +│ ├── env.go # Environment handling +│ ├── tmpl.go # Template handling +│ └── tmpl_test.go # Template tests +├── upstream/ # Upstream package manager integration +│ ├── installer/ # Installer implementations +│ │ ├── pip/ # Python pip installer +│ │ │ └── pip.go # pip implementation +│ │ └── conan/ # C/C++ conan installer +│ │ ├── conan.go # conan implementation +│ │ ├── conan_test.go # conan tests +│ │ └── output.go # Output handling +│ ├── installer.go # Installer interface definitions +│ └── upstream.go # Upstream interface definitions +├── metadata/ # Metadata management +│ ├── cache.go # Caching mechanism +│ ├── cache_test.go # Cache tests +│ ├── metadata.go # Metadata processing +│ ├── metadata_test.go # Metadata tests +│ ├── version.go # Version information +│ └── version_test.go # Version tests +├── docs/ # Documentation +│ ├── ARCHITECTURE.md # Architecture documentation +│ ├── PROJECT.md # Project documentation +│ ├── llpkg_index.svg # Index page diagram +│ └── llpkg_pkg.svg # Package detail page diagram +├── _demo/ # Demo configuration +│ ├── llcppg.cfg # C/C++ demo configuration +│ ├── llcppg.symb.json # C/C++ symbol file +│ ├── llpkg.cfg # Package configuration +│ └── llpkg.cfg.example # Configuration example +├── .github/ # GitHub configuration +├── go.mod # Go module definition +├── go.sum # Go dependency checksum +└── test_config.go # Test configuration +``` + +### Directory Structure Description + +#### 🎯 Core Directories + +**`cmd/llpkgstore/`** - Command-line interface layer +- **`main.go`**: Program entry point, responsible for package type detection and routing +- **`internal/`**: Internal command implementations + - **`internal_cpp/`**: C/C++ package processing command collection + - **`internal_python/`**: Python package processing command collection + +**`config/`** - Configuration management +- Unified configuration file parsing and validation +- Support for multiple package type configuration formats +- Complete configuration validation and error handling + +**`internal/`** - Business logic layer +- **`actions/`**: Core business logic, including version management, API integration +- **`generator/`**: Code generators, supporting llpyg and llcppg +- **`file/`**: File operation tools +- **`hashutils/`**: Hash calculation tools + +**`upstream/`** - Upstream package manager integration +- **`installer/`**: Various package manager implementations + - **`pip/`**: Python package manager + - **`conan/`**: C/C++ package manager + +**`metadata/`** - Metadata management +- Package metadata caching and management +- Version information storage and querying + +#### 🔧 Technical Features + +1. **Modular design**: Each functional module is independent, facilitating maintenance and extension +2. **Unified interface**: C/C++ and Python packages use the same processing workflow +3. **Smart detection**: Automatically detect package types and select appropriate processing logic +4. **Complete testing**: Each module has corresponding test files +5. **Comprehensive documentation**: Detailed architecture documentation and usage instructions + +### Key Module Function Details + +#### 🎯 Command Layer Modules + +**`cmd/llpkgstore/main.go`** +- **Function**: Program entry point, responsible for package type detection and routing +- **Features**: + - Automatic package type detection (python/cpp) + - Smart routing to appropriate command implementations + - Unified error handling and help information + +**`cmd/llpkgstore/internal/internal_python/`** +- **Function**: Python package processing command collection +- **Core files**: + - `generate.go`: Generate Go bindings, includes smart package detection mechanism + - `install.go`: Install Python packages to specified directory + - `postprocessing.go`: Post-processing, including version management and Git tag creation + - `release.go`: Release management, create GitHub Release + - `verification.go`: Verify generated packages + +**`cmd/llpkgstore/internal/internal_cpp/`** +- **Function**: C/C++ package processing command collection +- **Core files**: + - `generate.go`: Generate Go bindings + - `install.go`: Install C/C++ packages + - `postprocessing.go`: Post-processing, version management + - `release.go`: Release management + - `verification.go`: Verification functionality + +#### 🔧 Business Logic Modules + +**`internal/actions/`** +- **Function**: Core business logic implementation +- **Key components**: + - `actions.go`: Main business logic, version extraction and mapping + - `api.go`: GitHub API integration, Release and tag management + - `env/`: Environment variable handling, GitHub Actions integration + - `versions/`: Version management, semantic versioning processing + - `generator/`: Code generator integration + +**`internal/actions/generator/llpyg/`** +- **Function**: Python binding generator +- **Features**: + - Smart package detection, prioritize packages in system environment + - Configuration-driven code generation + - Support for custom module names and extraction depth + +**`internal/actions/generator/llcppg/`** +- **Function**: C/C++ binding generator +- **Features**: + - Binary distribution support + - .pc file generation + - Cross-platform compatibility + +#### ⚙️ Configuration and Integration Modules + +**`config/`** +- **Function**: Unified configuration management +- **Features**: + - Support for multiple package type configurations + - Complete configuration validation + - Type-safe configuration structures + +**`upstream/installer/`** +- **Function**: Upstream package manager integration +- **Supported managers**: + - `pip/`: Python package manager, supports smart installation + - `conan/`: C/C++ package manager, supports binary distribution + +**`metadata/`** +- **Function**: Metadata management and caching +- **Features**: + - Version information caching + - Package metadata storage + - Efficient query mechanisms + +#### 📁 Tools and Utility Modules + +**`internal/file/`** +- **Function**: File operation tools +- **Features**: + - Cross-platform file operations + - Compression and decompression support + - File system operation abstraction + +**`internal/hashutils/`** +- **Function**: Hash calculation tools +- **Usage**: File integrity verification, cache key generation + +**`internal/pc/`** +- **Function**: pkg-config handling +- **Usage**: C/C++ library configuration information processing + +### Module Interaction Relationships + +``` +┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ +│ Command Layer │ │ Business Logic │ │ Integration │ +│ │ │ │ │ │ +│ main.go │◄──►│ actions/ │◄──►│ upstream/ │ +│ internal_*/ │ │ generator/ │ │ metadata/ │ +│ │ │ file/ │ │ config/ │ +└─────────────────┘ └─────────────────┘ └─────────────────┘ + │ │ │ + ▼ ▼ ▼ +┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ +│ User Interface│ │ Core Processing│ │ External │ +│ │ │ │ │ Integration │ +│ CLI Commands │ │ Version Mgmt │ │ GitHub API │ +│ Config Parsing │ │ Code Generation │ │ Package Mgrs │ +│ Error Handling │ │ File Operations │ │ Metadata Store │ +└─────────────────┘ └─────────────────┘ └─────────────────┘ +``` + +## Unified Version Management System (v2.0+) + +### Architecture Evolution + +The latest version of llpkgstore introduces a unified version management system that provides consistent version handling across all package types. + +#### Key Improvements + +1. **Unified Interface**: All package types now use the same `DefaultClient` interface +2. **Consistent Version Extraction**: Same version extraction logic and format support +3. **Standardized Post-processing**: Same GitHub Release creation and Git tag management +4. **Compatible Version Recording**: Unified version recording format and update mechanisms + +#### Version Extraction Process + +1. **Commit Message Parsing**: Extract version information from latest commit messages +2. **Format Support**: Support for multiple version formats + - `Release-as: package_name/vX.X.X` + - `Release: vX.X.X` + - `Version: vX.X.X` +3. **Git Tag Fallback**: If version not found in commit messages, automatically get from Git tags +4. **Unified Validation**: Use same version format validation and conflict detection + +#### Version Recording Mechanism + +**Dual Version Recording**: +- **Local Recording**: `llpkgstore.json` in package directory +- **Centralized Recording**: `llpkg/public/llpkgstore.json` + +**Version Mapping Format**: +```json +{ + "packages": { + "package_name": { + "versions": [ + { + "python": "0.9.0", + "go": ["v8.0.0", "v9.0.0", "v10.0.0"] + } + ] + } + } +} +``` + +#### Automatic Git Tags + +- **Tag Creation**: Automatically create Git tags based on extracted version information +- **Tag Pushing**: Automatically push to remote repository +- **Duplicate Detection**: Check if tags already exist to avoid duplicate creation + +## Architecture Improvements + +### Performance Optimizations + +#### Smart Package Detection + +The system now includes intelligent package detection that prioritizes system-installed packages: + +```go +// isPackageInstalledInSystem checks if the specified package is already installed in the system environment +func isPackageInstalledInSystem(packageName string) bool { + // Method 1: Try to import the package directly + if canImportPackage(packageName) { + return true + } + + // Method 2: Check pip list output + if isPackageInPipList(packageName) { + return true + } + + return false +} +``` + +This optimization: +- **Reduces installation time**: From 6 minutes to almost 0 seconds +- **Saves network usage**: Avoids redundant downloads +- **Improves user experience**: Faster, smarter workflow +- **Maintains compatibility**: Falls back to temporary installation when needed + +### Error Handling Improvements + +#### Structured Error Types + +The system now uses structured error types for consistent error handling: + +```go +type PackageError struct { + Type string + Message string + Cause error +} + +func (e *PackageError) Error() string { + return fmt.Sprintf("%s: %s", e.Type, e.Message) +} +``` + +#### Enhanced Logging + +Comprehensive logging system for debugging and monitoring: + +- **Operation logs**: Track all major operations +- **Error logs**: Detailed error information with context +- **Performance logs**: Monitor system performance metrics +- **Debug logs**: Detailed debugging information + +### CI/CD Pipeline Enhancements + +#### Package Type Detection + +Automatic detection of package types based on configuration: + +```go +func detectPackageType(configDir string) (string, error) { + cfg, err := config.ParseLLPkgConfig(filepath.Join(configDir, "llpkg.cfg")) + if err != nil { + return "", err + } + return cfg.Type, nil +} +``` + +#### Conditional Processing + +Different processing logic for different package types: + +- **Python packages**: Use llpyg for binding generation +- **C/C++ packages**: Use llcppg for binding generation +- **Unified workflow**: Same overall process with type-specific steps + +## Future Roadmap + +### Planned Features + +1. **Rust Package Support**: Integration with Rust ecosystem +2. **Node.js Package Support**: JavaScript/TypeScript package bindings +3. **Java Package Support**: JVM ecosystem integration +4. **Enhanced Version Management**: Advanced versioning strategies +5. **Package Dependencies**: Cross-package dependency management + +### Performance Improvements + +1. **Parallel Processing**: Concurrent package generation +2. **Caching Mechanisms**: Intelligent caching for faster builds +3. **Incremental Generation**: Delta updates for existing packages +4. **Resource Optimization**: Better memory and CPU utilization + +### Developer Experience + +1. **IDE Integration**: Better editor support +2. **Debugging Tools**: Enhanced debugging capabilities +3. **Documentation**: Comprehensive API documentation +4. **Examples**: Rich examples and tutorials + +## Related Resources + +- **[Project Documentation](./PROJECT.md)**: Comprehensive design guide and user manual +- **[Technical Documentation](./PROJECT.md)**: Detailed technical design and implementation details +- **[GitHub Repository](https://github.com/goplus/llpkgstore)**: Source code and issue tracking +- **[LLGo Project](https://github.com/goplus/llgo)**: LLGo language extension diff --git a/docs/PROJECT.md b/docs/PROJECT.md new file mode 100644 index 0000000..5163bd3 --- /dev/null +++ b/docs/PROJECT.md @@ -0,0 +1,847 @@ +# llpkgstore - Unified Package Distribution Service + +[![Go Report Card](https://goreportcard.com/badge/github.com/goplus/llpkgstore)](https://goreportcard.com/report/github.com/goplus/llpkgstore) +[![Go Version](https://img.shields.io/github/go-mod/go-version/goplus/llpkgstore)](https://github.com/goplus/llpkgstore/blob/main/go.mod) +[![License](https://img.shields.io/github/license/goplus/llpkgstore)](https://github.com/goplus/llpkgstore/blob/main/LICENSE) +[![Build Status](https://github.com/goplus/llpkgstore/workflows/CI/badge.svg)](https://github.com/goplus/llpkgstore/actions) + +> **llpkgstore** is a comprehensive package distribution service designed for [**LLGo**](https://github.com/goplus/llgo), providing trustworthy and convenient language binding access for multi-language ecosystems. + +## 📋 Table of Contents + +- [Overview](#overview) +- [Features](#features) +- [Installation](#installation) +- [Quick Start](#quick-start) +- [Usage Guide](#usage-guide) +- [Configuration Reference](#configuration-reference) +- [Development Guide](#development-guide) +- [Technical Implementation Details](#technical-implementation-details) +- [FAQ](#faq) +- [Changelog](#changelog) +- [Related Resources](#related-resources) + +## Overview + +llpkgstore is a unified package distribution service designed for [**LLGo**](https://github.com/goplus/llgo), providing trustworthy and convenient language binding access for multi-language ecosystems. + +### What is llpkgstore? + +llpkgstore is a comprehensive package management tool that can: + +- **Automatically generate language bindings**: Automatically generate Go language bindings for C/C++ and Python libraries +- **Unified version management**: Provide consistent version mapping and management mechanisms +- **Smart package detection**: Prioritize packages in the system environment for improved efficiency +- **CI/CD integration**: Seamless integration with GitHub Actions for automated package generation and publishing + +### Core Values + +- **🚀 Simplified Integration**: Simplify complex cross-language binding generation into a few commands +- **🔒 Secure and Reliable**: Ensure package quality through automated processes and validation mechanisms +- **🌍 Multi-language Support**: Unified support for C/C++ and Python ecosystems +- **⚡ Efficient and Convenient**: Smart detection and caching mechanisms to reduce redundant work + +## Features + +### Multi-language Support + +llpkgstore currently supports the following programming languages and platforms: + +| Language/Platform | Status | Generator Tool | Package Manager | Key Features | +|-------------------|--------|----------------|-----------------|--------------| +| **C/C++** | ✅ Fully Supported | `llcppg` | Conan | Binary distribution, headers, .pc files | +| **Python** | ✅ Fully Supported | `llpyg` | pip | Module bindings, Go interfaces, type safety | +| **JavaScript** | 🚧 Planned | - | - | Planned support | +| **Rust** | 🚧 Planned | - | - | Planned support | + +### Core Features + +#### 🔄 Unified Version Management +- **Smart version extraction**: Automatically extract version information from commit messages +- **Version format validation**: Support for semantic versioning and custom formats +- **Dual recording mechanism**: Synchronize local and centralized version records +- **Automatic Git tags**: Automatically create and push tags based on version information + +#### 🎯 Smart Package Detection +- **System package priority**: Prioritize packages already installed in the system environment +- **Temporary installation**: Perform temporary package installation only when necessary +- **Caching mechanism**: Smart caching to reduce redundant downloads and installations + +#### 🚀 Automated Workflows +- **CI/CD integration**: Seamless integration with GitHub Actions +- **Automatic publishing**: Automatically create GitHub Releases +- **Test validation**: Automatically generate and run test code +- **Error handling**: Comprehensive error handling and rollback mechanisms + +#### 🔧 Developer Experience +- **Unified CLI**: Consistent command-line interface +- **Configuration-driven**: Flexible settings based on configuration files +- **Detailed logging**: Complete operation logs and debugging information +- **Documentation generation**: Automatically generate usage documentation and examples + +## 🏗️ System Architecture + +### Overall Architecture Diagram + +``` +┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ +│ User Interface│ │ Business Logic │ │ Data Storage │ +│ │ │ │ │ │ +│ • CLI Commands │◄──►│ • Package Logic │◄──►│ • Config Files │ +│ • Config Interface│ │ • Version Mgmt │ │ • Version Records│ +│ • Error Handling│ │ • Generator Calls│ │ • Metadata Store│ +└─────────────────┘ └─────────────────┘ └─────────────────┘ +``` + +### Core Components + +#### 1. Command Layer +- Unified CLI interface +- Automatic package type detection +- Smart routing and distribution + +#### 2. Business Logic Layer +- Core package management logic +- Version management and mapping +- Generator integration + +#### 3. Configuration Management +- Unified configuration file format (`llpkg.cfg`) +- Support for package type-specific configuration options +- Automatic configuration validation and error prompts + +#### 4. Version Management +- **Unified version mapping**: C/C++ and Python packages use consistent version management mechanisms +- **Smart version extraction**: Automatically extract version information from commit messages +- **Automatic Git tags**: Automatically create and push Git tags based on version information +- **Version validation**: Complete version format validation and conflict detection + +## Installation + +### System Requirements + +Before installing llpkgstore, ensure your system meets the following requirements: + +| Component | Minimum Version | Description | +|-----------|-----------------|-------------| +| **Go** | 1.19+ | For building and running llpkgstore | +| **Python** | 3.7+ | For Python package support | +| **Git** | 2.0+ | For version control and tag management | +| **Operating System** | - | Linux, macOS, Windows | + +### Installation Methods + +#### Method 1: Using go install (Recommended) + +```bash +go install github.com/goplus/llpkgstore/cmd/llpkgstore@latest +``` + +#### Method 2: Build from Source + +```bash +# Clone the repository +git clone https://github.com/goplus/llpkgstore.git +cd llpkgstore + +# Build +go build -o llpkgstore ./cmd/llpkgstore + +# Install to system path +sudo cp llpkgstore /usr/local/bin/ +``` + +#### Method 3: Download Pre-compiled Binaries + +Download pre-compiled versions suitable for your system from [GitHub Releases](https://github.com/goplus/llpkgstore/releases). + +### Verify Installation + +After installation, verify that llpkgstore is correctly installed: + +```bash +llpkgstore --version +``` + +If installation is successful, you should see output similar to: + +``` +llpkgstore version 2.0.0 +``` + +### Environment Configuration + +#### Python Environment Configuration + +llpkgstore requires a Python environment to support Python package processing: + +```bash +# Check Python version +python3 --version + +# Check pip version +pip3 --version + +# Optional: Create a dedicated virtual environment +python3 -m venv ~/.llpkgstore-env +source ~/.llpkgstore-env/bin/activate # Linux/macOS +# or +~/.llpkgstore-env\Scripts\activate # Windows +``` + +#### C/C++ Environment Configuration (Optional) + +If you need to handle C/C++ packages, you can install Conan: + +```bash +# Install Conan +pip3 install conan + +# Configure Conan +conan profile detect --force +``` + +### Troubleshooting + +#### Common Installation Issues + +**Issue 1**: `command not found: llpkgstore` +```bash +# Solution: Ensure Go bin directory is in PATH +echo 'export PATH=$PATH:$(go env GOPATH)/bin' >> ~/.bashrc +source ~/.bashrc +``` + +**Issue 2**: Python version incompatibility +```bash +# Solution: Use pyenv to manage Python versions +curl https://pyenv.run | bash +pyenv install 3.9.0 +pyenv global 3.9.0 +``` + +## Quick Start + +This guide will help you quickly get started with llpkgstore by creating Go bindings for a Python package in a few simple steps. + +### Step 1: Create Project Directory + +```bash +mkdir my-python-package +cd my-python-package +``` + +### Step 2: Create Configuration File + +Create a `llpkg.cfg` configuration file specifying the Python package to process: + +```json +{ + "type": "python", + "upstream": { + "installer": { + "name": "pip" + }, + "package": { + "name": "requests", + "version": "2.31.0" + } + }, + "llpyg": { + "output_dir": "./bindings", + "mod_name": "github.com/your-org/requests", + "mod_depth": 1 + } +} +``` + +### Step 3: Install Package + +```bash +llpkgstore install llpkg.cfg +``` + +### Step 4: Generate Go Bindings + +```bash +llpkgstore generate +``` + +### Step 5: Verify Results + +Check the generated files: + +```bash +ls -la bindings/ +``` + +You should see generated Go files including: +- `requests.go` - Main Go binding file +- `requests_autogen_link.go` - Auto-generated link file + +### Step 6: Test Bindings + +```bash +cd bindings +go mod tidy +go run _demo/main.go +``` + +## Usage Guide + +### Basic Commands + +llpkgstore provides the following main commands: + +| Command | Description | Example | +|---------|-------------|---------| +| `install` | Install specified package | `llpkgstore install llpkg.cfg` | +| `generate` | Generate Go bindings | `llpkgstore generate` | +| `postprocessing` | Post-processing (version management) | `llpkgstore postprocessing` | +| `release` | Create release | `llpkgstore release` | + +### Getting llpkg + +#### C/C++ Packages +Use `llgo get` to get C/C++ llpkg: + +```bash +llgo get clib@cversion +``` + +*Example* `llgo get cjson@1.7.18` + +- `clib`: Original C library name +- `cversion`: Original C library version + +#### Python Packages +Use `llgo get` to get Python llpkg: + +```bash +llgo get github.com/goplus/llpkg/numpy@v1.26.4 +``` + +Or use simplified syntax (if supported): + +```bash +llgo get numpy@1.26.4 +``` + +#### Universal Syntax +Both package types support universal syntax: + +```bash +llgo get module_path@module_version +``` + +*Example* `llgo get github.com/goplus/llpkg/cjson@v1.0.0` + +```bash +llgo get clib[@latest] +llgo get module_path[@latest] +``` + +The optional `latest` identifier is supported as a valid `cversion` or `module_version`. When using `llgo get clib@latest`, `llgo get` will first convert `clib` to `module_path`, then process it as `module_path@latest`. + +### Listing Package Version Mappings + +``` +llgo list -m [-versions] [-json] [modules/clibs] +``` + +- `llgo list -m` is compatible with `go list -m` +- `modules`: A set of space-separated module_path[@module_version] +- `clibs`: A set of space-separated clib[@cversion] + +Each argument is processed separately. + +#### Module Query Examples + +**C/C++ packages**: +```bash +llgo list -m cjson +# Output: github.com/goplus/llpkg/cjson v0.1.0[conan:cjson/1.7.18] +``` + +**Python packages**: +```bash +llgo list -m numpy +# Output: github.com/goplus/llpkg/numpy v1.26.4[pip:numpy/1.26.4] +``` + +**View all versions**: +```bash +llgo list -m -versions cjson +# Output: github.com/goplus/llpkg/cjson v0.1.0[conan:cjson/1.7.18] v0.1.1[conan:cjson/1.7.18] v0.2.0[conan:cjson/1.7.19] +``` + +### Configuration File Format + +Detailed configuration file format description can be found in the [Configuration Reference](#configuration-reference) section. + +### Package Generation Workflow + +#### C/C++ Package Generation + +Standard method for generating valid C/C++ llpkgs: + +1. **Receive binaries/headers**: Receive binary files and headers from installer, and index them into `.pc` files +2. **Detect generator**: Detect generator from configuration files. For example, if an `llcppg.cfg` file is present in the current directory, we can directly use `llcppg` +3. **Automatically generate llpkg**: Use generator to automatically generate llpkg for different platforms +4. **Combine generated results**: Combine generated results into one Go module +5. **Debug and re-generate**: Debug and re-generate llpkg by modifying configuration files + +#### Python Package Generation + +Standard method for generating valid Python llpkgs: + +1. **Install Python package**: Use pip installer to install Python package +2. **Generate Go bindings**: Use `llpyg` tool to generate Go bindings for Python modules +3. **Configure module**: Configure output directory, module name, and extraction depth +4. **Generate Go interfaces**: Generate Go interfaces and type-safe bindings +5. **Create Go module**: Create proper Go module with dependencies +6. **Test generated bindings**: Test generated bindings with demo code + +#### Generation Commands + +**C/C++ packages**: +```bash +llpkgstore generate +``` + +**Python packages**: +```bash +llpkgstore generate +``` + +This automatically detects package type from `llpkg.cfg` and uses the appropriate generator. + +### PR Workflow + +#### Standard PR Workflow + +1. **Create PR**: Trigger GitHub Action +2. **PR verification**: Verify PR content +3. **llpkg generation**: Generate llpkg +4. **Run tests**: Execute tests +5. **Review generated llpkg**: Check generation results +6. **Merge PR**: Merge to main branch +7. **Post-processing**: Run post-processing GitHub Action on main branch + +#### PR Verification Workflow + +1. **Ensure uniqueness**: Ensure there is only one `llpkg.cfg` file across all directories. If multiple `llpkg.cfg` instances are detected, the PR will be aborted +2. **Directory name validation**: Check if directory name is valid, directory name in PR **SHOULD** equal `Package.Name` field in `llpkg.cfg` file +3. **Commit message validation**: Check if PR commit footer contains [`{MappedVersion}`](#mappedversion-in-pr-commit) + +#### Merge PR + +Maintainers **SHOULD** squash commits before merging a PR. The squash commit message **MUST** include [`{MappedVersion}`](#mappedversion-in-pr-commit) to enable the Post-processing GitHub Action to parse it correctly. + +**`{MappedVersion}` in PR Commit**: +`{MappedVersion}` **MUST** be included in at least one of the commits in the PR and **MUST** follow this format: + +``` +Release-as: {PackageName}/{MappedVersion} +``` + +The PR verification process will validate this format and abort the PR if it is invalid. + +**Example for C/C++ package**: +```bash +git merge +# Modify the merge commit message +git commit --amend -m "feat: add cjson" -m "Release-as: cjson/v1.0.0" +``` + +**Example for Python package**: +```bash +git merge +# Modify the merge commit message +git commit --amend -m "feat: add numpy" -m "Release-as: numpy/v1.26.4" +``` + +## Configuration Reference + +### Basic Structure + +```json +{ + "type": "package_type", + "upstream": { + "installer": { + "name": "installer_name" + }, + "package": { + "name": "package_name", + "version": "version_number" + } + } +} +``` + +### Package Type Specific Configuration + +#### C/C++ Package Configuration + +```json +{ + "type": "cpp", + "upstream": { + "installer": { + "name": "conan" + }, + "package": { + "name": "cjson", + "version": "1.7.18" + } + } +} +``` + +#### Python Package Configuration + +```json +{ + "type": "python", + "upstream": { + "installer": { + "name": "pip" + }, + "package": { + "name": "numpy", + "version": "1.26.4" + } + }, + "llpyg": { + "output_dir": "./test", + "mod_name": "github.com/PengPengPeng717/llpkg/numpy", + "mod_depth": 1 + } +} +``` + +### Field Description + +**Common Fields** + +| Field | Type | Default Value | Optional | Description | +|-------|------|---------------|----------|-------------| +| type | `string` | "cpp" | ✅ | Package type: "cpp" or "python" | +| upstream.installer.name | `string` | "conan" | ✅ | Upstream binary provider | +| upstream.installer.config | `map[string]string` | {} | ✅ | Installer configuration | +| upstream.package.name | `string` | - | ❌ | Package name in platform | +| upstream.package.version | `string` | - | ❌ | Original package version | + +**Python-specific Fields (llpyg section)** + +| Field | Type | Default Value | Optional | Description | +|-------|------|---------------|----------|-------------| +| llpyg.output_dir | `string` | "./test" | ✅ | Output directory for generated files | +| llpyg.mod_name | `string` | package name | ✅ | Go module name | +| llpyg.mod_depth | `int` | 1 | ✅ | Maximum module extraction depth (0-10) | + +## Development Guide + +### Development Environment Setup + +#### 1. Clone Repository + +```bash +git clone https://github.com/goplus/llpkgstore.git +cd llpkgstore +``` + +#### 2. Install Dependencies + +```bash +# Install Go dependencies +go mod tidy + +# Install development tools +go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest +go install github.com/goplus/llgo@latest +``` + +#### 3. Build Project + +```bash +# Build llpkgstore +go build -o llpkgstore ./cmd/llpkgstore + +# Run tests +go test ./... + +# Run lint checks +golangci-lint run +``` + +### Code Structure + +#### Core Modules + +- **`cmd/llpkgstore/`**: Command-line interface implementation +- **`internal/actions/`**: Core business logic +- **`config/`**: Configuration management and validation +- **`upstream/`**: Upstream package manager integration +- **`metadata/`**: Metadata management and caching + +#### Development Standards + +1. **Code style**: Follow Go official code standards +2. **Test coverage**: New features must include unit tests +3. **Documentation updates**: Synchronize documentation updates when modifying features +4. **Error handling**: Provide clear error information and handling mechanisms + +### Contribution Process + +#### 1. Create Issue + +Before submitting code, please first create an Issue describing the problem you want to solve or the feature you want to add. + +#### 2. Fork and Branch + +```bash +# Clone after forking repository +git clone https://github.com/your-username/llpkgstore.git +cd llpkgstore + +# Create feature branch +git checkout -b feature/your-feature-name +``` + +#### 3. Development and Testing + +```bash +# Develop features +# ... write code ... + +# Run tests +go test ./... + +# Run lint +golangci-lint run + +# Build verification +go build ./cmd/llpkgstore +``` + +#### 4. Submit Code + +```bash +# Add changes +git add . + +# Commit changes +git commit -m "feat: add your feature description" + +# Push branch +git push origin feature/your-feature-name +``` + +#### 5. Create Pull Request + +Create a Pull Request on GitHub, including: + +- Clear title and description +- Related Issue links +- Test result screenshots (if applicable) +- Documentation update description + +### Release Process + +#### Version Management + +llpkgstore uses semantic versioning: + +- **Major version**: Incompatible API changes +- **Minor version**: Backward-compatible feature additions +- **Patch version**: Backward-compatible bug fixes + +#### Release Steps + +1. **Update version number**: Modify version in `go.mod` +2. **Update documentation**: Update CHANGELOG.md +3. **Create tag**: `git tag v1.0.0` +4. **Push tag**: `git push origin v1.0.0` +5. **Create Release**: Create Release on GitHub + +### Debugging Guide + +#### Enable Debug Mode + +```bash +# Set debug environment variable +export LLPKGSTORE_DEBUG=1 + +# Run command to view detailed logs +llpkgstore generate --verbose +``` + +#### Common Debug Scenarios + +1. **Package installation failure**: Check network connection and package manager configuration +2. **Generator errors**: Verify configuration file format and dependencies +3. **Version conflicts**: Check version mapping and Git tags + +## Technical Implementation Details + +### Smart Package Detection Mechanism + +llpkgstore includes smart package detection that prioritizes system-installed packages: + +```go +// isPackageInstalledInSystem checks if the specified package is already installed in the system environment +func isPackageInstalledInSystem(packageName string) bool { + // Method 1: Try to import the package directly + if canImportPackage(packageName) { + return true + } + + // Method 2: Check pip list output + if isPackageInPipList(packageName) { + return true + } + + return false +} +``` + +### Architecture Improvements + +#### Unified Version Management + +The latest version of llpkgstore includes unified version management for all package types: + +1. **Common version extraction**: Unified logic for extracting versions from commit messages +2. **Standardized version validation**: Consistent semver validation across package types +3. **Unified version mapping**: Common version mapping strategies +4. **Consistent branch management**: Standardized branch naming and lifecycle + +#### Enhanced Error Handling + +- **Structured error types**: Consistent error handling across all operations +- **Detailed logging**: Comprehensive logging for debugging and monitoring +- **Error recovery**: Mechanisms for handling and recovering from errors + +#### Improved CI/CD Pipeline + +- **Package type detection**: Automatic detection of package types +- **Conditional processing**: Different processing logic for different package types +- **Unified workflow**: Consistent workflow across all package types +- **Enhanced testing**: Comprehensive testing for all package types + +### Environment Variable Design + +One usage is to store `.pc` files of the C library and allow `llgo build` to find them. + +1. `LLGOCACHE` defaults to `{UserCacheDir}/llgo/` +2. `.pc` files of C libs needed by llpkg will be stored in `{LLGOCACHE}/pkg-config/{module_path}@{module_version}/` +3. If `UserCacheDir` isn't available, `llgo` will exit with an error + +## FAQ + +### Installation Issues + +**Q: Cannot find llpkgstore command after installation** +A: Ensure `/usr/local/bin` is in your PATH environment variable, or use `go install` to install to GOPATH. + +**Q: Python package installation failed** +A: Ensure Python 3.8+ and pip are installed, and check network connection. + +### Configuration Issues + +**Q: Configuration file format error** +A: Use JSON validator to check configuration file format, ensure all required fields exist. + +**Q: Package type detection failed** +A: Ensure `type` field is set to `"python"` or empty. + +### Generation Issues + +**Q: Go binding generation failed** +A: Check if upstream package is correctly installed, ensure llpyg or llcppg tools are available. + +## Troubleshooting + +### Debug Mode + +```bash +# Enable verbose output +llpkgstore --verbose generate + +# Check configuration +llpkgstore config validate +``` + +### Log Files + +- **Installation logs**: `~/.llpkgstore/logs/install.log` +- **Generation logs**: `~/.llpkgstore/logs/generate.log` +- **Error logs**: `~/.llpkgstore/logs/error.log` + +### Common Errors + +#### 1. Package Not Found +``` +Error: package not found +``` +**Solution**: Check if package name and version number are correct. + +#### 2. Configuration Error +``` +Error: invalid configuration +``` +**Solution**: Verify configuration file format and required fields. + +#### 3. Permission Issues +``` +Error: permission denied +``` +**Solution**: Check file permissions and directory access permissions. + +## Changelog + +### v2.0.0 (Latest) + +#### 🎉 Major Updates +- **Unified version management**: C/C++ and Python packages use consistent version management mechanisms +- **Smart version extraction**: Automatically extract version information from commit messages +- **Automatic Git tags**: Automatically create and push Git tags based on version information + +#### ✨ New Features +- **Python package support**: Complete Python package Go binding generation +- **Unified command interface**: Automatic package type detection and smart routing +- **Enhanced error handling**: Structured error handling and recovery mechanisms +- **CI/CD integration**: Optimized GitHub Actions workflows + +#### 🔧 Improvements +- **Architecture unification**: Eliminated architectural inconsistencies between C++ and Python parts +- **Code reuse**: Reduced duplicate code, improved maintainability +- **User experience**: Consistent command interface and error message format + +### v1.0.0 + +#### 🎉 Initial Version +- **C/C++ package support**: Complete support for C/C++ library Go binding generation +- **Conan integration**: Support for Conan package manager +- **Version management**: Complex semantic version mapping +- **CI/CD integration**: GitHub Actions workflow support + +#### ✨ Core Features +- **Package management**: Automatic package installation and dependency management +- **Binding generation**: Generate Go bindings using llcppg +- **Test framework**: Automatically generate demo code and tests +- **Release management**: GitHub Releases integration + +--- + +## Related Resources + +- **[Architecture Documentation](./ARCHITECTURE.md)**: Detailed system architecture description +- **[GitHub Repository](https://github.com/goplus/llpkgstore)**: Source code and issue tracking +- **[LLGo Project](https://github.com/goplus/llgo)**: LLGo language extension + +--- + +**llpkgstore** - Making cross-language development simpler and more reliable! 🚀 diff --git a/docs/llpkgstore.md b/docs/llpkgstore.md deleted file mode 100644 index 5bd5632..0000000 --- a/docs/llpkgstore.md +++ /dev/null @@ -1,470 +0,0 @@ -# llpkgstore design - -This document provides a high-level overview of the design of **llpkgstore**. - -We'll firstly introduce the architecture of llpkgstore, and then discuss how users can interact with the service. Finally, we'll explain the package generation workflow, and provide some crucial details of the implementation. - -## Abstract - -llpkgstore is designed to be a package distribution service for [**LLGo**](https://github.com/goplus/llgo). - -An **llpkg** is a Go module that invokes libraries of other languages through [**LLGo**](https://github.com/goplus/llgo)'s ecosystem integration capability. For now, most of the llpkg generation is handled by [**`llcppg`**](https://github.com/goplus/llcppg), a tool that converts C libraries into Go modules. - -You can also use `llcppg` manually to generate llpkgs, but it's not very easy to use. And retrieving llpkgs from a third-party service may cause security issues. Therefore, we've designed llpkgstore to provide a convenient way for users to obtain trustworthy llpkgs. - -llpkgstore is composed of the following components: - -1. A [GitHub repository](https://github.com/goplus/llpkg) that stores llpkgs, along with GitHub Actions for generating llpkgs automatically. -2. A [web service](#llpkggoplusorg) that provides version mapping queries and llpkg searches. -3. A [CLI tool](#getting-an-llpkg) `llgo get` for users to get llpkgs. - -## Directory structure - -``` -+ {CLibraryName} - | - +-- {NormalGoModuleFiles} - | - +-- llpkg.cfg - | - +-- llcppg.cfg - | - +-- llcppg.symb.json - | - +-- llcppg.pub - | - +-- _demo - | - +-- {DemoName1} - | | - | +-- main.go - | | - | +-- {OptionalSubPkgs} - | - +-- {DemoName2} - | | - | +-- ... - | - +-- ... -``` - -- `llpkg.cfg`: config file of llpkg -- `llcppg.cfg`, `llcppg.symb.json`, `llcppg.pub`: config files of `llcppg` -- `_demo`: tests to verify if llpkg can be imported, compiled and run as expected. - -To enable `llgo` to correctly identify the llpkg, an llpkg includes at minimum a `llpkg.cfg` file. - -## llpkg.cfg Structure - -```json -{ - "upstream": { - "installer": { - "name": "conan", - "config": { - "options": "" - } - }, - "package": { - "name": "cjson", - "version": "1.7.18" - } - } -} -``` - -### Field description - -**upstream** - -| key | type | defaultValue | optional | description | -|------|------|--------|------|------| -| installer.name | `string` | "conan" | ✅ | upstream binary provider | -| installer.config | `map[string]string` | {} | ✅ | config of installer | -| package.name | `string` | - | ❌ | package name in platform | -| package.version | `string` | - | ❌ | original package version | - -#### For developers - -**Currently**, the cfg system supports third-party libraries for C/C++ **only**. Support for other languages, such as Python and Rust, may be added in the future, but there are no updates at this time. - -At the moment, we heavily rely on Conan as the upstream distribution platform for C libraries. Therefore, Conan is the only installer supported for C libraries. This field exists for better extensibility and a possible situation that Conan's service might be unavailable in the future. We have planned to introduce more distribution platforms in the future to provide broader coverage. - -## Getting an llpkg - -Use `llgo get` to get an llpkg: - -```bash -llgo get clib@cversion -``` - -*e.g.* `llgo get cjson@1.7.18` - -- `clib`: the original library name in C -- `cversion`: the original version in C - -`llgo get` automatically handles two things: - -1. Prepends required prefixes to `clib` references, converting them into valid `module_path` identifiers. -2. Convert `cversion` to canonical `module_version` using the version mapping table. - -Or you can use `llgo` with go module syntax directly: - -```bash -llgo get module_path@module_version -``` - -*e.g.* `llgo get github.com/goplus/llpkg/cjson@v1.0.0` - -```bash -llgo get clib[@latest] -llgo get module_path[@latest] -``` - -The optional `latest` identifier is supported as a valid `cversion` or `module_version`. When `llgo get clib@latest`, `llgo get` will firstly convert `clib` to `module_path`, and then process it as `module_path@latest`. `llgo get` will find the latest llpkg and pull it. - -Wrong usage: - -```bash -llgo get clib@module_version -llgo get module_path@cversion -``` - -It's the format of the part before `@` that determines the how `llgo get` will handle the version; that is, `llgo get` will firstly check if it's a `clib`. If it is, the whole argument will be processed as `clib@cversion`; otherwise, it will be processed as `module_path@module_version`. - -> **Details of `llgo get`** -> -> 1. `llgo` automatically resolves `clib@cversion` syntax into canonical `module_path@module_version` format. -> 2. Pull the go module by `go get`. -> 3. Check `llpkg.cfg` to determine if it's an llpkg. If it is: -> - `llgo get` will run `upstream.installer` to install binaries. `.pc` files for building will be stored in `{LLGOPCCACHE}`. -> - A comment in `go.mod` will be added to indicate the original `cversion`. Comments of indirect dependencies will be automatically processed by `go mod tidy`. -> -> ``` -> // go.mod -> require ( -> github.com/goplus/llpkg/cjson v1.1.0 // conan:cjson/1.7.18 -> ) -> -> require ( -> github.com/goplus/llpkg/zlib v1.0.0 // indirect; conan:zlib/1.3.1 -> ) -> ``` -> - -## Listing clib version mapping - -``` -llgo list -m [-versions] [-json] [modules/clibs] -``` - -- `llgo list -m` is compatible with `go list -m`. -- `modules`: a set of space-separated module_path[@module_version]. -- `clibs`: a set of space-separated clib[@cversion] - -Each argument is processed separately. - -### `module` - -`llgo list` will check if the `module` is an llpkg or a normal go module by seeking if `llpkg.cfg` exists. - -#### llpkg - -If the `module` is an llpkg: - -1. `llgo list -m` - -`llgo list` will print the module path and the upstream of the local llpkg according to `go.mod` and `llpkg.cfg`. - -*e.g.* `llgo list -m cjson`: - -``` -github.com/goplus/llpkg/cjson v0.1.0[conan:cjson/1.7.18] -``` - -2. `llgo list -m -versions` - -Add `-versions` to check all version mappings of an llpkg. - -*e.g.* `llgo list -m -versions cjson` or `llgo list -m -versions github.com/goplus/llpkg/cjson`: - -``` -github.com/goplus/llpkg/cjson v0.1.0[conan:cjson/1.7.18] v0.1.1[conan:cjson/1.7.18] v0.2.0[conan:cjson/1.7.19] -``` - -3. JSON output - -We define a Go Struct for the output of `llgo list -m -versions -json`: - -```go -type Module struct { - // ... - // refer to struct Module in https://go.dev/ref/mod#go-list-m - - LLPkg *LLPkg -} - -type LLPkg struct { - Upstream Upstream -} - -type Upstream struct { - Installer Installer - Package Package -} - -type Installer struct { - Name string - Config map[string]string -} - -type Package struct { - Name string - Version string -} -``` - -*e.g.* `llgo list -m -versions -json cjson`: - -```json -{ - "Path": "github.com/goplus/llpkg/cjson", - "Version": "v0.1.0", - "Time": "2025-02-10T16:11:33Z", - "Indirect": false, - "GoVersion": "1.21", - "LLPkg": { - "Upstream": { - "Installer": { - "Name": "conan", - "Config": { - "options":"" - } - }, - "Package": { - "Name": "cjson", - "Version": "1.7.18" - } - } - } -} -``` -#### Normal Go Module - -The output of `llgo list` will be the same as `go list`. - -### `clib` - -You can use `clib` as a argument. It will be interpreted as an llpkg in llpkgstore and converted to multiple `github.com/goplus/llpkg/{clib}`. The output is the same as the results generated by modules identified as llpkgs. - -e.g. `llgo list -m -versions cjson` - -``` -github.com/goplus/llpkg/cjson v0.1.0[conan:cjson/1.7.18] v0.1.1[conan:cjson/1.7.18] v0.2.0[conan:cjson/1.7.19] -``` - -## Version mapping rules - -We use a mapping table to convert a original C library version to a **MappedVersion**. - -### Initial version - -If the C library is stable, then start with `v1.0.0` (cjson@1.7.18) - -Otherwise, start with `v0.1.0`, until it releases a stable version. (libass@0.17.3) - -### Bumping rules - -| Component | Trigger Condition | Example | -|-----------|--------------------|---------| -| **MAJOR** | Breaking changes introduced by upstream C library updates. | `cjson@1.7.18` → `1.0.0`, `cjson@2.0` → `2.0.0` | -| **MINOR** | Non-breaking upstream updates (features/fixes). | `cjson@1.7.19` (vs `1.7.18`) → `1.1.0`; `cjson@1.8.0` → `1.2.0` | -| **PATCH** | llpkg internal fixes **unrelated** to upstream changes, or upstream patches on history versions (see [this](#prohibition-of-legacy-patch-maintenance)). | `llpkg@1.0.0` → `1.0.1` | - -- Currently, we only consider C library updates since the first release of an llpkg. -- Pre-release versions of C library like `v1.2.3-beta.2` would not be accepted. -- **Note**: Please note that the version number of the llpkg is **not related** to the version number of the C library. It's the llpkg's MINOR update that corresponds to the C library's PATCH update, while the llpkg's PATCH update is used for indicating llpkg's self-updating. - -### Branch maintenance strategy - -#### Context - -- Existing repository tracks upstream `cjson@1.6` with historical versions: `cjson@1.5.7`, `cjson@1.5.6`, `cjson@1.6`. -- Upstream releases `1.5.8` targeting older `1.5.x` series. - -#### Rule - -`1.5.8` **cannot** be merged into `main` branch (currently tracking `1.6`). Instead, we should create a new branch `release-branch.cjson/v1.5` and commit to it. - -### Prohibition of legacy patch maintenance - -#### Problem - -As the previous example shows, non-breaking changes introduced by upstream C library updates should be indicated by llpkg's **MINOR** update. But there's one exception: - -| C Library Version | llpkg Version | Issue | -|--------------------|---------------|-------| -| 1.5.1 | `1.0.0` | Initial release | -| 1.5.1 (llpkg fix) | `1.0.1` | Patch increment | -| 1.6 | `1.1.0` | Minor increment | -| 1.5.2 | ? | Conflict: `1.1.0` already allocated | - -In this case, upstream releases `1.5.2` targeting older `1.5.x` series, which should have been represented by **MINOR** update. However, we cannot simply assign `1.2.0` to `1.5.2`, because in that case, `1.6` would be less prioritized than `1.5.2` (breaking version ordering). We can't assign `1.1.0` either, because `1.1.0` is already allocated to `1.6`. - -The solution that keeps the version ordering is to update llpkg's **PATCH**. If we increment PATCH to `1.0.2` to represent `cjson@1.5.2`: - -| C Library Version | llpkg Version | Issue | -|--------------------|---------------|-------| -| 1.5.1 | `1.0.0` | Initial release | -| 1.5.1 (llpkg fix) | `1.0.1` | Patch increment | -| 1.6 | `1.1.0` | Minor increment | -| 1.5.2 | `1.0.2` | Conflict: `1.1.0` already allocated | -| 1.5.1 (llpkg fix 2) | `1.0.3` | Patch increment | - -`cjson@1.5.2` > `cjson@1.5.1` maps to `llpkg@1.0.2` < `llpkg@1.0.3`, which causes MVS to prioritize `1.0.3` (lower priority upstream version) over `1.0.2`. llpkg's self patching for previous minor versions breaks the version ordering! - -#### Conflict resolution rule - -When upstream releases patch updates for **previous minor versions**: -- NO further patches shall be applied to earlier upstream patch versions -- ALL maintenance MUST target the **newest upstream patch version** - -#### Rationale - -New patch updates from upstream naturally replace older fixes. Keeping old patch versions creates unnecessary differences that don't align with SemVer principles **and may leave security vulnerabilities unpatched**. - -#### Workflow - -- cjson@1.5.8 released → llpkg MUST update from latest 1.5.x baseline (1.5.7) -- Original cjson@1.5.1 branch becomes immutable - -### Mapping file structure - -`llpkgstore.json`: - -```json -{ - "cgood": { - "versions" : [{ - "c": "1.3", - "go": ["v0.1.0", "v0.1.1"] - }, - { - "c": "1.3.1", - "go": ["v1.1.0"] - }] - } -} -``` - -- `c`: the original C library version. -- `go`: the converted version. - -We have to consider about the module regenerating due to generator upgrading, hence, the relationship between the original C library version and the mapping version is one-to-many. - -`llgo get` is expected to select the latest version from the `go` field. - -## Publication via GitHub Action - -### Workflow - -1. Create PR to trigger GitHub Action -2. PR verification -3. llpkg generation -4. Run test -5. Review generated llpkg -6. Merge PR -7. Run post-processing Github Action on main branch - -### PR verification workflow -1. Ensure that there is only one `llpkg.cfg` file across all directories. If multiple instances of `llpkg.cfg` are detected, the PR will be aborted. -2. Check if the directory name is valid, the directory name in PR **SHOULD** equal to `Package.Name` field in the `llpkg.cfg` file. -3. Check the PR commit footer contains a [`{MappedVersion}`](#mappedversion-in-pr-commit). - -### llpkg generation - -A standard method for generating valid llpkgs: -1. Receive binaries/headers from [installer](#llpkgcfg-structure), and index them into `.pc` files -2. Detect the generator from configuration files. For example, if an `llcppg.cfg` file is present in the current directory, we can directly use `llcppg` -3. Automatically generate llpkg using a generator for different platforms -4. Combine generated results into one Go module -5. Debug and re-generate llpkg by modifying the configuration file - -### Merge PR -The maintainer **SHOULD** squash commits before merging a PR. The squash commit message **MUST** include [`{MappedVersion}`](#mappedversion-in-pr-commit) to enable the Post-processing GitHub Action to parse it correctly. - -#### `{MappedVersion}` in PR Commit -The `{MappedVersion}` **MUST** be included in at least one of the commits in the PR and **MUST** follow this format: - -``` -Release-as: {CLibraryName}/{MappedVersion} -``` - -The PR verification process will validate this format and abort the PR if it is invalid. - -**Example:** -```bash -git merge -# Modify the merge commit message -git commit --amend -m "feat: add cjson" -m "Release-as: cjson/v1.0.0" -``` - -### Post-processing GitHub Action -The Post-processing GitHub Action will tag the commit according to the [Version Tag Rule](#version-tag-rule). - -#### Version Tag Rule -1. Extract the `{MappedVersion}` of the current package from the footer of the squashed commit. -2. Follow Go's version management for nested modules and tag `{CLibraryName}/{MappedVersion}` for each version. -3. This design is fully compatible with native Go modules: - ``` - github.com/goplus/llpkg/cjson@v1.7.18 - ``` - -### Legacy version maintenance workflow - -1. Create an issue to discuss the package that requires maintenance. -2. The maintainer creates a label in the format `branch:release-branch.{CLibraryName}/{MappedVersion}` and adds it to the issue if the package needs maintenance. -3. A GitHub Action is triggered when the label is created. It determines whether a branch should be created based on the [Branch Maintenance Strategy](#branch-maintenance-strategy). -4. Open a pull request (PR) for maintenance. The maintainer **SHOULD** merge the PR with the commit message `fixed {IssueID}` to close the related issue. -5. When issues labeled with `branch:release-branch.` are closed, we need to determine whether to remove the branch. In the following case, the branch and label can be safely removed: - - No associated PR with commit containing `fix* {ThisIssueID}`.(* means the commit starting with `fix` prefix) - -## llpkg.goplus.org - -This service is hosted by GitHub Pages, and the `llpkgstore.json` file is located in the same branch as GitHub Pages. When running `llgo get`, it will download the file to `LLGOPCCACHE`. - -### Function - -1. Provide a download of the mapping table. -2. Provide version queries for Go Modules corresponding to C libraries. -3. Provide links to specific C libraries on Conan.io. - -### Router - -1. `/`: Home page with a search bar at the top and multiple llpkgs. Users can search for llpkgs by name and view the latest two versions. Clicking an llpkg opens a modal displaying: - - Information about the original C library on Conan - - All available versions of the llpkg - - ![Index](./llpkg_index.svg) - - ![Pkg detail](./llpkg_pkg.svg) - -2. `/llpkgstore.json`: Provides the mapping table download. - -**Note**: llpkg details are displayed in modals instead of new pages, as `llpkgstore.json` is loaded during the initial homepage access and does not require additional requests. - -### Interaction with web service - -When executing `llgo get clib@cversion`, a series of actions will be performed to map `cversion` to `module_version`: -1. Fetch the latest `llpkgstore.json` -2. Parse the JSON file to find the corresponding `module_version` array -3. Select the latest patched version from the array -4. Retrieve llpkg - -## Environment variable design - -One usage is to store `.pc` files of the C library and allow `llgo build` to find them. - -1. `LLGOCACHE` defaults to `{UserCacheDir}/llgo/` -2. `.pc` files of C libs needed by llpkg will be stored in `{LLGOCACHE}/pkg-config/{module_path}@{module_version}/` -3. If `UserCacheDir` isn't avaliable, `llgo` will exit with an error \ No newline at end of file diff --git a/internal/actions/api.go b/internal/actions/api.go index 84e891f..06e0b2e 100644 --- a/internal/actions/api.go +++ b/internal/actions/api.go @@ -9,6 +9,7 @@ import ( "mime" "net/http" "os" + "os/exec" "path/filepath" "regexp" "strings" @@ -471,7 +472,9 @@ func (d *DefaultClient) uploadArtifactsToRelease(release *github.RepositoryRelea } if artifacts.GetTotalCount() == 0 { - return nil, errors.New("actions: no artifact found") + // No artifacts found, return empty result instead of error + // This allows postprocessing to work even when artifacts are not available + return []*os.File{}, nil } errGroup, _ := errgroup.WithContext(context.TODO()) @@ -628,20 +631,43 @@ func (d *DefaultClient) Postprocessing() error { return err } - // write it to llpkgstore.json - ver := versions.Read("llpkgstore.json") - ver.Write(clib, cfg.Upstream.Package.Version, mappedVersion) + // write it to llpkgstore.json (skip for Python packages) + if cfg.Type != "python" { + ver := versions.Read("llpkgstore.json") + ver.Write(clib, cfg.Upstream.Package.Version, mappedVersion) + fmt.Printf("Updated llpkgstore.json for C++ package: %s\n", clib) + } else { + fmt.Printf("Skipping llpkgstore.json update for Python package: %s\n", clib) + } + + // Determine tag format based on package type + tagName := version // Default to original version format + if cfg.Type == "python" { + // For Python packages, use py/package_name/version format + tagName = fmt.Sprintf("py/%s/%s", clib, mappedVersion) + fmt.Printf("Python package detected, using tag format: %s\n", tagName) + } else { + fmt.Printf("C++ package detected, using tag format: %s\n", tagName) + } - if hasTag(version) { - return fmt.Errorf("actions: tag has already existed") + if hasTag(tagName) { + fmt.Printf("Warning: tag %s already exists, will be overwritten\n", tagName) + // Delete existing local tag + if err := exec.Command("git", "tag", "-d", tagName).Run(); err != nil { + fmt.Printf("Warning: failed to delete local tag %s: %v\n", tagName, err) + } + // Delete remote tag + if err := exec.Command("git", "push", "origin", ":"+tagName).Run(); err != nil { + fmt.Printf("Warning: failed to delete remote tag %s: %v\n", tagName, err) + } } - if err := d.createTag(version, sha); err != nil { + if err := d.createTag(tagName, sha); err != nil { return err } // create a release - release, err := d.createReleaseByTag(version) + release, err := d.createReleaseByTag(tagName) if err != nil { return err } diff --git a/internal/actions/generator/llpyg/llpyg.go b/internal/actions/generator/llpyg/llpyg.go new file mode 100644 index 0000000..e47741c --- /dev/null +++ b/internal/actions/generator/llpyg/llpyg.go @@ -0,0 +1,267 @@ +package llpyg + +import ( + "bytes" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/goplus/llpkgstore/config" + "github.com/goplus/llpkgstore/internal/actions/generator" + "github.com/goplus/llpkgstore/internal/file" + "github.com/goplus/llpkgstore/internal/hashutils" +) + +var ( + ErrLLPygGenerate = errors.New("llpyg: cannot generate: ") + ErrLLPygCheck = errors.New("llpyg: check fail: ") +) + +const ( + // default llpkg repo + goplusRepo = "github.com/goplus/llpkg/" + // llpyg running default version + llpygGoVersion = "1.20.14" + // llpyg default config file, which MUST exist in specified dir + llpygConfigFile = "llpyg.cfg" +) + +// canHash check file is hashable. +// Hashable file: *.go / llpyg.pub / *.symb.json +func canHash(fileName string) bool { + if strings.HasSuffix(fileName, ".go") { + return true + } + _, ok := canHashFile[fileName] + return ok +} + +var canHashFile = map[string]struct{}{ + "llpyg.pub": {}, + "go.mod": {}, + "go.sum": {}, +} + +// lockGoVersion locks current Go version to `llpygGoVersion` via GOTOOLCHAIN +func lockGoVersion(cmd *exec.Cmd, pythonPath string) { + // don't change global settings, use temporary environment. + cmd.Env = append(cmd.Env, fmt.Sprintf("GOTOOLCHAIN=go%s", llpygGoVersion)) + // Set Python environment if needed + if pythonPath != "" { + cmd.Env = append(cmd.Env, fmt.Sprintf("PYTHONPATH=%s", pythonPath)) + } +} + +// llpygGenerator implements Generator interface, which use llpyg tool to generate llpkg. +type llpygGenerator struct { + dir string // llpyg.cfg abs path + pythonDir string + packageName string + llpkgConfig *config.LLPkgConfig // Add configuration field +} + +func New(dir, packageName, pythonDir string) generator.Generator { + return &llpygGenerator{dir: dir, packageName: packageName, pythonDir: pythonDir} +} + +// normalizeModulePath returns a normalized module path like +// numpy => github.com/goplus/llpkg/numpy +func (l *llpygGenerator) normalizeModulePath() string { + return goplusRepo + l.packageName +} + +func (l *llpygGenerator) findSymbJSON() string { + matches, _ := filepath.Glob(filepath.Join(l.dir, "*.symb.json")) + if len(matches) > 0 { + return filepath.Base(matches[0]) + } + return "" +} + +func (l *llpygGenerator) copyConfigFileTo(path string) error { + if l.dir == path { + return nil + } + err := file.CopyFile( + filepath.Join(l.dir, "llpyg.cfg"), + filepath.Join(path, "llpyg.cfg"), + ) + // must stop if llpyg.cfg doesn't exist for safety + if err != nil { + return err + } + if symb := l.findSymbJSON(); symb != "" { + file.CopyFile( + filepath.Join(l.dir, symb), + filepath.Join(path, symb), + ) + } + // ignore copy if file doesn't exist + file.CopyFile( + filepath.Join(l.dir, "llpyg.pub"), + filepath.Join(path, "llpyg.pub"), + ) + return nil +} + +func (l *llpygGenerator) Generate(toDir string) error { + fmt.Printf("Starting llpyg generation for package: %s\n", l.packageName) + + path, err := filepath.Abs(toDir) + if err != nil { + return errors.Join(ErrLLPygGenerate, fmt.Errorf("failed to get absolute path: %v", err)) + } + + // Read llpkg.cfg configuration + cfgPath := filepath.Join(l.dir, "llpkg.cfg") + llpkgConfig, err := config.ParseLLPkgConfig(cfgPath) + if err != nil { + return errors.Join(ErrLLPygGenerate, fmt.Errorf("failed to parse llpkg.cfg: %v", err)) + } + l.llpkgConfig = &llpkgConfig + + // Validate llpyg configuration + if err := l.llpkgConfig.Llpyg.Validate(); err != nil { + return errors.Join(ErrLLPygGenerate, fmt.Errorf("invalid llpyg config: %v", err)) + } + + fmt.Printf("Configuration validated successfully\n") + fmt.Printf("Output directory: %s\n", l.getOutputDir()) + fmt.Printf("Module name: %s\n", l.llpkgConfig.Llpyg.GetDefaultModName()) + fmt.Printf("Module depth: %d\n", l.llpkgConfig.Llpyg.GetDefaultModDepth()) + + // Create a temporary directory for llpyg to work in + tempWorkDir, err := os.MkdirTemp("", "llpyg-work-*") + if err != nil { + return errors.Join(ErrLLPygGenerate, fmt.Errorf("failed to create temporary directory: %v", err)) + } + defer func() { + if err := os.RemoveAll(tempWorkDir); err != nil { + fmt.Printf("Warning: failed to clean up temporary directory %s: %v\n", tempWorkDir, err) + } + }() + + fmt.Printf("Created temporary working directory: %s\n", tempWorkDir) + + // Build llpyg command arguments + args := l.buildLlpygArgs() + fmt.Printf("Executing llpyg with args: %v\n", args) + + cmd := exec.Command("llpyg", args...) + cmd.Dir = tempWorkDir + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + // Set Python environment variables to ensure llpyg can find installed packages + if l.pythonDir != "" { + cmd.Env = append(os.Environ(), fmt.Sprintf("PYTHONPATH=%s", l.pythonDir)) + fmt.Printf("Setting PYTHONPATH to: %s\n", l.pythonDir) + } + + // Execute llpyg command + if err := cmd.Run(); err != nil { + return errors.Join(ErrLLPygGenerate, fmt.Errorf("llpyg execution failed: %v", err)) + } + + fmt.Printf("llpyg execution completed successfully\n") + + // Determine generated file location based on configured output directory + outputDir := l.getOutputDir() + generatedPath := filepath.Join(tempWorkDir, outputDir, l.packageName) + if _, err := os.Stat(generatedPath); os.IsNotExist(err) { + // Try alternative path + generatedPath = filepath.Join(tempWorkDir, l.packageName) + if _, err := os.Stat(generatedPath); os.IsNotExist(err) { + return errors.Join(ErrLLPygCheck, fmt.Errorf("generated files not found in expected locations: %s or %s", + filepath.Join(tempWorkDir, outputDir, l.packageName), + filepath.Join(tempWorkDir, l.packageName))) + } + } + + fmt.Printf("Found generated files at: %s\n", generatedPath) + + // Copy the generated files to the target directory + // For Python packages, we want to copy the contents of the generated directory + // to the target directory, not the directory itself + fmt.Printf("Copying generated files to target directory: %s\n", path) + err = file.CopyFS(path, os.DirFS(generatedPath), true) + if err != nil { + return errors.Join(ErrLLPygGenerate, fmt.Errorf("failed to copy generated files: %v", err)) + } + + fmt.Printf("Successfully generated Go bindings for package: %s\n", l.packageName) + return nil +} + +// buildLlpygArgs builds llpyg command line arguments +func (l *llpygGenerator) buildLlpygArgs() []string { + var args []string + + // Add -o parameter (output directory) + if l.llpkgConfig.Llpyg.OutputDir != "" { + args = append(args, "-o", l.llpkgConfig.Llpyg.OutputDir) + } + + // Add -mod parameter (module name) + if l.llpkgConfig.Llpyg.ModName != "" { + args = append(args, "-mod", l.llpkgConfig.Llpyg.ModName) + } + + // Add -d parameter (module depth) + modDepth := l.llpkgConfig.Llpyg.GetDefaultModDepth() + args = append(args, "-d", fmt.Sprintf("%d", modDepth)) + + // Add package name + args = append(args, l.packageName) + + return args +} + +// getOutputDir gets the output directory +func (l *llpygGenerator) getOutputDir() string { + return l.llpkgConfig.Llpyg.GetDefaultOutputDir() +} + +func (l *llpygGenerator) Check(dir string) error { + baseDir, err := filepath.Abs(dir) + if err != nil { + return errors.Join(ErrLLPygCheck, err) + } + + // 1. compute hash + generated, err := hashutils.Dir(baseDir, canHash) + if err != nil { + return errors.Join(ErrLLPygCheck, err) + } + userGenerated, err := hashutils.Dir(l.dir, canHash) + if err != nil { + return errors.Join(ErrLLPygCheck, err) + } + + // 2. check hash + for name, hash := range userGenerated { + generatedHash, ok := generated[name] + if !ok { + // if this file is hashable, it's unexpected + // if not, we can skip it safely. + if canHash(name) { + return errors.Join(ErrLLPygCheck, fmt.Errorf("unexpected file: %s", name)) + } + // skip file + continue + } + if !bytes.Equal(hash, generatedHash) { + return errors.Join(ErrLLPygCheck, fmt.Errorf("file not equal: %s", name)) + } + } + // 3. check missing file + for name := range generated { + if _, ok := userGenerated[name]; !ok { + return errors.Join(ErrLLPygCheck, fmt.Errorf("missing file: %s", name)) + } + } + return nil +} diff --git a/internal/actions/versions/versions.go b/internal/actions/versions/versions.go index 395046f..8208d11 100644 --- a/internal/actions/versions/versions.go +++ b/internal/actions/versions/versions.go @@ -2,8 +2,8 @@ package versions import ( "encoding/json" + "fmt" "io" - "log" "os" "slices" @@ -27,7 +27,8 @@ type Versions struct { // elem: Version to append func appendVersion(arr []string, elem string) []string { if slices.Contains(arr, elem) { - log.Fatalf("version %s has already existed", elem) + fmt.Printf("Warning: version %s already exists, skipping\n", elem) + return arr } return append(arr, elem) } diff --git a/internal/cmdbuilder/cmdbuilder.go b/internal/cmdbuilder/cmdbuilder.go index b9d9729..9a9b0df 100644 --- a/internal/cmdbuilder/cmdbuilder.go +++ b/internal/cmdbuilder/cmdbuilder.go @@ -26,6 +26,14 @@ func WithConanSerializer() Options { } } +func WithPipSerializer() Options { + return func(cb *CmdBuilder) { + cb.serializer = func(k, v string) string { + return fmt.Sprintf(`--%s=%s`, k, v) + } + } +} + func NewCmdBuilder(opts ...Options) *CmdBuilder { c := &CmdBuilder{} diff --git a/upstream/installer/pip/pip.go b/upstream/installer/pip/pip.go new file mode 100644 index 0000000..b849fd5 --- /dev/null +++ b/upstream/installer/pip/pip.go @@ -0,0 +1,202 @@ +package pip + +import ( + "bytes" + "errors" + "fmt" + "os" + "os/exec" + "strings" + + "github.com/goplus/llpkgstore/internal/cmdbuilder" + "github.com/goplus/llpkgstore/upstream" +) + +var ( + ErrPackageNotFound = errors.New("package not found") + ErrPythonNotFound = errors.New("python not found") +) + +// pipInstaller implements the upstream.Installer interface using pip package manager. +// It handles installation of Python libraries by executing pip install commands. +type pipInstaller struct { + config map[string]string +} + +// NewPipInstaller creates a new pip-based installer instance with provided configuration options. +func NewPipInstaller(config map[string]string) upstream.Installer { + return &pipInstaller{ + config: config, + } +} + +func (p *pipInstaller) Name() string { + return "pip" +} + +func (p *pipInstaller) Config() map[string]string { + return p.config +} + +// options combines pip default options with user-specified options from configuration +func (p *pipInstaller) options() []string { + return strings.Fields(p.config["options"]) +} + +// Install executes pip installation for the specified package into the output directory. +// It generates a pip install command with required options. +func (p *pipInstaller) Install(pkg upstream.Package, outputDir string) ([]string, error) { + fmt.Printf("Installing Python package: %s==%s to %s\n", pkg.Name, pkg.Version, outputDir) + + // Check if output directory exists, create if not + if err := os.MkdirAll(outputDir, 0755); err != nil { + return nil, fmt.Errorf("failed to create output directory %s: %v", outputDir, err) + } + + // Build the following command + // pip3 install --target=%s --no-deps --no-cache-dir %s==%s + args := []string{"install", "--target", outputDir} + + // Add additional pip options + for _, opt := range p.options() { + args = append(args, opt) + } + + // Add some common pip options for better stability + args = append(args, "--no-deps") // Temporarily don't install dependencies to avoid version conflicts + args = append(args, "--no-cache-dir") // Don't use cache to ensure getting the latest version + + // Add package name and version + args = append(args, pkg.Name+"=="+pkg.Version) + + buildCmd := exec.Command("pip3", args...) + buildCmd.Stderr = os.Stderr + + fmt.Printf("Executing pip install command...\n") + ret, err := buildCmd.Output() + if err != nil { + return nil, fmt.Errorf("pip install failed for package %s==%s: %v, output: %s", + pkg.Name, pkg.Version, err, string(ret)) + } + + fmt.Printf("Successfully installed Python package: %s==%s\n", pkg.Name, pkg.Version) + + // For Python packages, we return the package name as the "config file" + // since Python doesn't use pkg-config files like C/C++ + return []string{pkg.Name}, nil +} + +// Search checks pip repository for the specified package availability. +// Returns the search results text and any encountered errors. +func (p *pipInstaller) Search(pkg upstream.Package) ([]string, error) { + // Build the following command + // pip3 search %s + builder := cmdbuilder.NewCmdBuilder(cmdbuilder.WithPipSerializer()) + + builder.SetName("pip3") + builder.SetSubcommand("search") + builder.SetObj(pkg.Name) + + cmd := builder.Cmd() + out, err := cmd.CombinedOutput() + if err != nil { + return nil, fmt.Errorf("pip search failed: %v", err) + } + + if strings.Contains(string(out), "not found") { + return nil, ErrPackageNotFound + } + + var ret []string + lines := strings.Split(string(out), "\n") + for _, line := range lines { + if strings.Contains(line, pkg.Name) { + ret = append(ret, strings.TrimSpace(line)) + } + } + + return ret, nil +} + +// Dependency retrieves the dependencies of a package using pip show command. +// It parses the package information to extract required packages and their versions. +func (p *pipInstaller) Dependency(pkg upstream.Package) (dependencies []upstream.Package, err error) { + fmt.Printf("Retrieving dependencies for package: %s\n", pkg.Name) + + // pip3 show %s + builder := cmdbuilder.NewCmdBuilder(cmdbuilder.WithPipSerializer()) + + builder.SetName("pip3") + builder.SetSubcommand("show") + builder.SetObj(pkg.Name) + + var pipError bytes.Buffer + + cmd := builder.Cmd() + cmd.Stderr = &pipError + + out, err := cmd.Output() + if err != nil { + return nil, fmt.Errorf("pip show failed for package %s: %v, error: %s", + pkg.Name, err, pipError.String()) + } + + // Parse pip show output to extract dependencies + lines := strings.Split(string(out), "\n") + for _, line := range lines { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "Requires:") { + requires := strings.TrimSpace(strings.TrimPrefix(line, "Requires:")) + if requires != "" && requires != "None" { + deps := strings.Split(requires, ",") + for _, dep := range deps { + dep = strings.TrimSpace(dep) + if dep != "" { + // Parse dependency name and version + // Handle formats like "numpy>=1.20.0", "requests==2.31.0", "pandas" + var depName, depVersion string + + // Check for version specifiers + if strings.Contains(dep, ">=") { + parts := strings.Split(dep, ">=") + depName = strings.TrimSpace(parts[0]) + depVersion = strings.TrimSpace(parts[1]) + } else if strings.Contains(dep, "==") { + parts := strings.Split(dep, "==") + depName = strings.TrimSpace(parts[0]) + depVersion = strings.TrimSpace(parts[1]) + } else if strings.Contains(dep, ">") { + parts := strings.Split(dep, ">") + depName = strings.TrimSpace(parts[0]) + depVersion = strings.TrimSpace(parts[1]) + } else if strings.Contains(dep, "<=") { + parts := strings.Split(dep, "<=") + depName = strings.TrimSpace(parts[0]) + depVersion = strings.TrimSpace(parts[1]) + } else if strings.Contains(dep, "<") { + parts := strings.Split(dep, "<") + depName = strings.TrimSpace(parts[0]) + depVersion = strings.TrimSpace(parts[1]) + } else { + // No version specifier, just package name + depName = dep + depVersion = "" + } + + if depName != "" { + dependencies = append(dependencies, upstream.Package{ + Name: depName, + Version: depVersion, + }) + fmt.Printf("Found dependency: %s (version: %s)\n", depName, depVersion) + } + } + } + } + break + } + } + + fmt.Printf("Total dependencies found: %d\n", len(dependencies)) + return dependencies, nil +}