diff --git a/buildtools/cli.go b/buildtools/cli.go index 642d310d8..bc029d45a 100644 --- a/buildtools/cli.go +++ b/buildtools/cli.go @@ -5,12 +5,13 @@ import ( "fmt" "os" "os/exec" + "path/filepath" "sort" "strconv" "strings" + "github.com/BurntSushi/toml" "github.com/jfrog/jfrog-cli-artifactory/artifactory/commands/container/strategies" - "github.com/jfrog/jfrog-cli-artifactory/artifactory/commands/python" "github.com/jfrog/jfrog-cli-artifactory/artifactory/commands/setup" "github.com/jfrog/jfrog-cli-core/v2/artifactory/utils" @@ -1161,6 +1162,49 @@ func getCommandName(orgArgs []string) (string, []string) { return "", cmdArgs } +// getPrimarySourceFromToml reads pyproject.toml and returns the name of the primary source +// Returns (sourceName, isPrimary) where isPrimary indicates if source has priority='primary' +func getPrimarySourceFromToml() (string, bool) { + type PyProjectSource struct { + Name string `toml:"name"` + URL string `toml:"url"` + Priority string `toml:"priority"` + } + + type PyProject struct { + Tool struct { + Poetry struct { + Source []PyProjectSource `toml:"source"` + } `toml:"poetry"` + } `toml:"tool"` + } + + tomlPath := filepath.Join(".", "pyproject.toml") + tomlData, err := os.ReadFile(tomlPath) + if err != nil { + return "", false + } + + var pyproject PyProject + if err := toml.Unmarshal(tomlData, &pyproject); err != nil { + return "", false + } + + // Look for primary source + for _, source := range pyproject.Tool.Poetry.Source { + if source.Priority == "primary" { + return source.Name, true + } + } + + // If no primary, use first source + if len(pyproject.Tool.Poetry.Source) > 0 { + return pyproject.Tool.Poetry.Source[0].Name, false + } + + return "", false +} + func NpmInstallCmd(c *cli.Context) error { if show, err := cliutils.ShowGenericCmdHelpIfNeeded(c, c.Args(), "npminstallhelp"); show || err != nil { return err @@ -1353,6 +1397,34 @@ func pythonCmd(c *cli.Context, projectType project.ProjectType) error { } cmdName, poetryArgs := getCommandName(filteredArgs) + // Extract --repository flag for artifact collection (if publishing) + deployerRepo := "" + for i, arg := range poetryArgs { + if strings.HasPrefix(arg, "--repository=") { + deployerRepo = strings.TrimPrefix(arg, "--repository=") + } else if (arg == "--repository" || arg == "-r") && i+1 < len(poetryArgs) { + deployerRepo = poetryArgs[i+1] + } + } + + // Auto-add repository flag if not provided and we're publishing + if cmdName == "publish" && deployerRepo == "" { + // Try to get primary source from pyproject.toml (same as native Poetry behavior) + if repoName, isPrimary := getPrimarySourceFromToml(); repoName != "" { + if isPrimary { + log.Info(fmt.Sprintf("No --repository flag specified. Using '%s' from pyproject.toml (priority='primary')", repoName)) + } else { + log.Info(fmt.Sprintf("No --repository flag specified. Using '%s' from pyproject.toml (first source)", repoName)) + } + poetryArgs = append([]string{"-r", repoName}, poetryArgs...) + deployerRepo = repoName + } else { + log.Warn("No repository specified and no sources found in pyproject.toml. Poetry will attempt to publish to PyPI.") + } + } else if cmdName == "publish" && deployerRepo != "" { + log.Info(fmt.Sprintf("Publishing to repository: %s (from --repository flag)", deployerRepo)) + } + // Execute native poetry command directly (similar to Maven FlexPack) log.Info(fmt.Sprintf("Running Poetry %s.", cmdName)) poetryCmd := exec.Command("poetry", append([]string{cmdName}, poetryArgs...)...) @@ -1369,7 +1441,7 @@ func pythonCmd(c *cli.Context, projectType project.ProjectType) error { workingDir, err := os.Getwd() if err != nil { log.Warn("Failed to get working directory, skipping build info collection: " + err.Error()) - } else if err := buildinfo.GetPoetryBuildInfo(workingDir, buildConfiguration); err != nil { + } else if err := buildinfo.GetPoetryBuildInfo(workingDir, buildConfiguration, deployerRepo); err != nil { log.Warn("Failed to collect Poetry build info: " + err.Error()) } else { buildNumber, err := buildConfiguration.GetBuildNumber() diff --git a/go.mod b/go.mod index c750ae4f0..d5597be37 100644 --- a/go.mod +++ b/go.mod @@ -10,15 +10,16 @@ replace ( ) require ( + github.com/BurntSushi/toml v1.5.0 github.com/agnivade/levenshtein v1.2.1 github.com/buger/jsonparser v1.1.1 github.com/docker/docker v28.5.2+incompatible github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1 github.com/jfrog/archiver/v3 v3.6.1 - github.com/jfrog/build-info-go v1.12.5-0.20251209031413-f5f0e93dc8db + github.com/jfrog/build-info-go v1.12.5-0.20251209120002-025bda5ff78b github.com/jfrog/gofrog v1.7.6 github.com/jfrog/jfrog-cli-application v1.0.2-0.20251208114900-b3cc968c8e3d - github.com/jfrog/jfrog-cli-artifactory v0.8.1-0.20251209060234-c05dd4957894 + github.com/jfrog/jfrog-cli-artifactory v0.8.1-0.20251209121625-98f7b22a08c1 github.com/jfrog/jfrog-cli-core/v2 v2.60.1-0.20251125083543-e689762c4ff0 github.com/jfrog/jfrog-cli-evidence v0.8.3-0.20251204144808-73fa744851c0 github.com/jfrog/jfrog-cli-platform-services v1.10.1-0.20251205121610-171eb9b0000e @@ -26,6 +27,7 @@ require ( github.com/jfrog/jfrog-client-go v1.55.1-0.20251209032636-8966184db185 github.com/jszwec/csvutil v1.10.0 github.com/manifoldco/promptui v0.9.0 + github.com/spf13/viper v1.21.0 github.com/stretchr/testify v1.11.1 github.com/testcontainers/testcontainers-go v0.35.0 github.com/urfave/cli v1.22.17 @@ -48,7 +50,6 @@ require ( dario.cat/mergo v1.0.2 // indirect github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 // indirect github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect - github.com/BurntSushi/toml v1.5.0 // indirect github.com/CycloneDX/cyclonedx-go v0.9.3 // indirect github.com/GoogleCloudPlatform/grpc-gcp-go/grpcgcp v1.5.3 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 // indirect @@ -214,7 +215,6 @@ require ( github.com/spf13/afero v1.15.0 // indirect github.com/spf13/cast v1.10.0 // indirect github.com/spf13/pflag v1.0.10 // indirect - github.com/spf13/viper v1.21.0 // indirect github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect github.com/stretchr/objx v0.5.3 // indirect github.com/subosito/gotenv v1.6.0 // indirect diff --git a/go.sum b/go.sum index c5a4d46f0..a5d93bde7 100644 --- a/go.sum +++ b/go.sum @@ -1175,8 +1175,8 @@ github.com/jellydator/ttlcache/v3 v3.4.0 h1:YS4P125qQS0tNhtL6aeYkheEaB/m8HCqdMMP github.com/jellydator/ttlcache/v3 v3.4.0/go.mod h1:Hw9EgjymziQD3yGsQdf1FqFdpp7YjFMd4Srg5EJlgD4= github.com/jfrog/archiver/v3 v3.6.1 h1:LOxnkw9pOn45DzCbZNFV6K0+6dCsQ0L8mR3ZcujO5eI= github.com/jfrog/archiver/v3 v3.6.1/go.mod h1:VgR+3WZS4N+i9FaDwLZbq+jeU4B4zctXL+gL4EMzfLw= -github.com/jfrog/build-info-go v1.12.5-0.20251209031413-f5f0e93dc8db h1:5q4hUqZVl7Xt+R+ono5lDH1/lkvV1spnfDtp0VtJqlo= -github.com/jfrog/build-info-go v1.12.5-0.20251209031413-f5f0e93dc8db/go.mod h1:9W4U440fdTHwW1HiB/R0VQvz/5q8ZHsms9MWcq+JrdY= +github.com/jfrog/build-info-go v1.12.5-0.20251209120002-025bda5ff78b h1:wf8u5g84GW8ZYsM59UGk+1vvcUOCaP75NiGZeOatkC8= +github.com/jfrog/build-info-go v1.12.5-0.20251209120002-025bda5ff78b/go.mod h1:9W4U440fdTHwW1HiB/R0VQvz/5q8ZHsms9MWcq+JrdY= github.com/jfrog/froggit-go v1.20.6 h1:Xp7+LlEh0m1KGrQstb+u0aGfjRUtv1eh9xQBV3571jQ= github.com/jfrog/froggit-go v1.20.6/go.mod h1:obSG1SlsWjktkuqmKtpq7MNTTL63e0ot+ucTnlOMV88= github.com/jfrog/go-mockhttp v0.3.1 h1:/wac8v4GMZx62viZmv4wazB5GNKs+GxawuS1u3maJH8= @@ -1187,8 +1187,8 @@ github.com/jfrog/jfrog-apps-config v1.0.1 h1:mtv6k7g8A8BVhlHGlSveapqf4mJfonwvXYL github.com/jfrog/jfrog-apps-config v1.0.1/go.mod h1:8AIIr1oY9JuH5dylz2S6f8Ym2MaadPLR6noCBO4C22w= github.com/jfrog/jfrog-cli-application v1.0.2-0.20251208114900-b3cc968c8e3d h1:0o6tj4nPP9uCscyfPbKBUcCaIYof42irwii6XsBB8zM= github.com/jfrog/jfrog-cli-application v1.0.2-0.20251208114900-b3cc968c8e3d/go.mod h1:xum2HquWO5uExa/A7MQs3TgJJVEeoqTR+6Z4mfBr1Xw= -github.com/jfrog/jfrog-cli-artifactory v0.8.1-0.20251209060234-c05dd4957894 h1:PCtFq128LZCCrMtQ/tzZViCDxvzwEvnjHQDzAA8ZVAw= -github.com/jfrog/jfrog-cli-artifactory v0.8.1-0.20251209060234-c05dd4957894/go.mod h1:Jb2MTLteycwR+95MfcfLvguo/JcRqILdDpbdnMWveVA= +github.com/jfrog/jfrog-cli-artifactory v0.8.1-0.20251209121625-98f7b22a08c1 h1:vT9QWrwW6pJPcHewSVsDWWoCHq+Nt3UqnEaJjyMqFMU= +github.com/jfrog/jfrog-cli-artifactory v0.8.1-0.20251209121625-98f7b22a08c1/go.mod h1:b/Sf+FOjWwoQOZ00r+fXMbDqpts8L0q1lMNgS5cofAs= github.com/jfrog/jfrog-cli-core/v2 v2.60.1-0.20251125083543-e689762c4ff0 h1:EsasTBE5i2MyCESS/icZxKIlObpGiOyW9K67MAaEWco= github.com/jfrog/jfrog-cli-core/v2 v2.60.1-0.20251125083543-e689762c4ff0/go.mod h1:d9aADumiyjCBvZLffp8wldvP9XFHxcvk2PoOSUYms2g= github.com/jfrog/jfrog-cli-evidence v0.8.3-0.20251204144808-73fa744851c0 h1:8S1vE1PeVtrzWkKL0N39cX6XLLNV0It+f6xjRKjw7Ug= diff --git a/utils/buildinfo/buildinfo.go b/utils/buildinfo/buildinfo.go index 4d55d9e06..1dedee1fb 100644 --- a/utils/buildinfo/buildinfo.go +++ b/utils/buildinfo/buildinfo.go @@ -2,7 +2,9 @@ package buildinfo import ( "fmt" + "net/url" "os" + "os/exec" "path/filepath" "strings" @@ -19,6 +21,7 @@ import ( specutils "github.com/jfrog/jfrog-client-go/artifactory/services/utils" "github.com/jfrog/jfrog-client-go/utils/io/content" "github.com/jfrog/jfrog-client-go/utils/log" + "github.com/spf13/viper" ) const ( @@ -39,7 +42,7 @@ func GetBuildInfoForPackageManager(pkgManager, workingDir string, buildConfigura switch pkgManager { case "poetry": - return GetPoetryBuildInfo(workingDir, buildConfiguration) + return GetPoetryBuildInfo(workingDir, buildConfiguration, "") // Empty deployer repo - will use from pyproject.toml case "mvn", "maven": // Maven FlexPack is handled directly in jfrog-cli-artifactory Maven command return GetBuildInfoForUploadedArtifacts("", buildConfiguration) @@ -57,8 +60,9 @@ func GetBuildInfoForPackageManager(pkgManager, workingDir string, buildConfigura } // GetPoetryBuildInfo collects build info for Poetry projects -func GetPoetryBuildInfo(workingDir string, buildConfiguration *buildUtils.BuildConfiguration) error { +func GetPoetryBuildInfo(workingDir string, buildConfiguration *buildUtils.BuildConfiguration, deployerRepo string) error { log.Debug("Collecting Poetry build info from directory: " + workingDir) + log.Debug("Deployer repository: " + deployerRepo) buildName, err := buildConfiguration.GetBuildName() if err != nil { @@ -89,10 +93,17 @@ func GetPoetryBuildInfo(workingDir string, buildConfiguration *buildUtils.BuildC return fmt.Errorf("ServerDetails extraction failed: %w", err) } - err = collectPoetryBuildInfo(workingDir, buildName, buildNumber, serverDetails, repoConfig.TargetRepo(), buildConfiguration) + // Use deployerRepo if provided, otherwise use resolver repo from config + artifactRepo := deployerRepo + if artifactRepo == "" { + artifactRepo = repoConfig.TargetRepo() + } + log.Info(fmt.Sprintf("Using repository for artifacts: %s", artifactRepo)) + + err = collectPoetryBuildInfo(workingDir, buildName, buildNumber, serverDetails, repoConfig.TargetRepo(), artifactRepo, buildConfiguration) if err != nil { log.Warn("Enhanced Poetry collection failed, falling back to standard method: " + err.Error()) - err = saveBuildInfo(serverDetails, repoConfig.TargetRepo(), "", buildConfiguration) + err = saveBuildInfo(serverDetails, artifactRepo, "", buildConfiguration) if err != nil { log.Error("Failed to save Poetry build info: ", err) return fmt.Errorf("saveBuildInfo failed: %w", err) @@ -289,6 +300,13 @@ func extractRepositoryConfig() (*project.RepositoryConfig, error) { // extractRepositoryConfigForProject extracts repository configuration for a specific project type func extractRepositoryConfigForProject(projectType project.ProjectType) (*project.RepositoryConfig, error) { log.Debug("Extracting repository config for project type") + + // In native mode, completely ignore YAML and infer from pyproject.toml + if os.Getenv("JFROG_RUN_NATIVE") == "true" { + log.Info("Native mode enabled: inferring Poetry config from pyproject.toml") + return inferPoetryConfigFromToml(projectType) + } + prefix := project.ProjectConfigDeployerPrefix // Handle invalid project types gracefully @@ -298,8 +316,8 @@ func extractRepositoryConfigForProject(projectType project.ProjectType) (*projec configFilePath, exists, err := project.GetProjectConfFilePath(projectType) if !exists { - log.Warn("Project configuration file not found") - return nil, fmt.Errorf("project configuration file not found") + log.Warn("Project configuration file not found, attempting to infer from pyproject.toml...") + return inferPoetryConfigFromToml(projectType) // Fallback to TOML } if err != nil { log.Error("Failed to get project config file path: ", err) @@ -340,7 +358,7 @@ func CreateAqlQueryForSearch(repo, file string) string { } // collectPoetryBuildInfo collects Poetry dependencies and artifacts for build info -func collectPoetryBuildInfo(workingDir, buildName, buildNumber string, serverDetails *config.ServerDetails, targetRepo string, buildConfiguration *buildUtils.BuildConfiguration) error { +func collectPoetryBuildInfo(workingDir, buildName, buildNumber string, serverDetails *config.ServerDetails, _ string, artifactRepo string, buildConfiguration *buildUtils.BuildConfiguration) error { log.Debug("Initializing Poetry dependency collection...") // Create Poetry configuration @@ -369,14 +387,14 @@ func collectPoetryBuildInfo(workingDir, buildName, buildNumber string, serverDet } // Collect artifacts and add them to the build info with proper repository paths - err = addArtifactsToBuildInfo(buildInfo, serverDetails, targetRepo, workingDir) + err = addArtifactsToBuildInfo(buildInfo, serverDetails, artifactRepo, workingDir) if err != nil { log.Warn("Failed to add artifacts to build info: " + err.Error()) // Continue anyway - dependencies are more important than artifacts for now } // Then set build properties on uploaded artifacts (this tags them with build info) - err = setPoetryBuildProperties(serverDetails, targetRepo, buildName, buildNumber, buildConfiguration.GetProject(), buildInfo) + err = setPoetryBuildProperties(serverDetails, artifactRepo, buildName, buildNumber, buildConfiguration.GetProject(), buildInfo) if err != nil { log.Warn("Failed to set build properties on artifacts: " + err.Error()) } @@ -660,3 +678,130 @@ func setPoetryBuildProperties(serverDetails *config.ServerDetails, targetRepo, b log.Info(fmt.Sprintf("Successfully set build properties on %d Poetry artifacts", len(module.Artifacts))) return nil } + +// inferPoetryConfigFromToml reads pyproject.toml and infers repository configuration +func inferPoetryConfigFromToml(_ project.ProjectType) (*project.RepositoryConfig, error) { + workingDir, err := os.Getwd() + if err != nil { + return nil, fmt.Errorf("failed to get working directory: %w", err) + } + + tomlPath := filepath.Join(workingDir, "pyproject.toml") + viper.SetConfigType("toml") + viper.SetConfigFile(tomlPath) + + if err := viper.ReadInConfig(); err != nil { + return nil, fmt.Errorf("failed to read pyproject.toml: %w", err) + } + + sources := viper.Get("tool.poetry.source") + if sources == nil { + return nil, fmt.Errorf("no Poetry sources found in pyproject.toml") + } + + // Get list of configured servers from jf config + cmd := exec.Command("jf", "config", "show") + output, err := cmd.Output() + if err != nil { + return nil, fmt.Errorf("failed to get jf config: %w", err) + } + + sourcesList, ok := sources.([]interface{}) + if !ok { + return nil, fmt.Errorf("invalid Poetry sources format in pyproject.toml") + } + + for _, source := range sourcesList { + sourceMap, ok := source.(map[string]interface{}) + if !ok { + continue + } + + sourceName, _ := sourceMap["name"].(string) + sourceURL, _ := sourceMap["url"].(string) + + if sourceURL == "" { + continue + } + + // Parse the URL to extract host + parsedURL, err := url.Parse(sourceURL) + if err != nil { + continue + } + + // Check if this URL matches any configured server + serverID := "" + lines := strings.Split(string(output), "\n") + for _, line := range lines { + if strings.Contains(line, "Server ID:") { + parts := strings.SplitN(line, ":", 2) + if len(parts) > 1 { + serverID = strings.TrimSpace(parts[1]) + } + } + if strings.Contains(line, "JFrog Platform URL:") && serverID != "" { + // Extract URL after "JFrog Platform URL: " prefix + parts := strings.SplitN(line, "JFrog Platform URL:", 2) + if len(parts) < 2 { + continue + } + configURL := strings.TrimSpace(parts[1]) + if strings.Contains(sourceURL, configURL) || strings.Contains(configURL, parsedURL.Host) { + log.Info(fmt.Sprintf("Matched source '%s' to server '%s'", sourceName, serverID)) + + // Extract repository name from URL + repoName := extractRepoNameFromPypiURL(sourceURL) + if repoName == "" { + repoName = sourceName // Fallback to source name + } + + log.Info(fmt.Sprintf("Inferred repository: %s", repoName)) + + // Get server details from config + serverDetails, err := config.GetSpecificConfig(serverID, false, true) + if err != nil { + return nil, fmt.Errorf("failed to get server details for %s: %w", serverID, err) + } + + // Create and return repository config + repoConfig := &project.RepositoryConfig{} + repoConfig.SetServerDetails(serverDetails) + repoConfig.SetTargetRepo(repoName) + + log.Info("Successfully inferred Poetry config from pyproject.toml (native mode)") + return repoConfig, nil + } + } + } + } + + return nil, fmt.Errorf("no matching JFrog server found for Poetry sources in pyproject.toml") +} + +// extractRepoNameFromPypiURL extracts repository name from PyPI-format Artifactory URL +// Example: https://server.com/artifactory/api/pypi/poetry-local/simple -> poetry-local +func extractRepoNameFromPypiURL(urlStr string) string { + // Remove trailing /simple or /simple/ + urlStr = strings.TrimSuffix(urlStr, "/") + urlStr = strings.TrimSuffix(urlStr, "/simple") + urlStr = strings.TrimSuffix(urlStr, "/") + + // Split by / and find "api/pypi/" pattern + parts := strings.Split(urlStr, "/") + for i := 0; i < len(parts)-1; i++ { + if parts[i] == "api" && i+1 < len(parts) && parts[i+1] == "pypi" { + // Repo name is after "pypi" + if i+2 < len(parts) { + return parts[i+2] + } + } + } + + // Fallback: take the last meaningful part + if len(parts) > 0 { + return parts[len(parts)-1] + } + + return "" +}