Repository navigation
Addressed bugs/improvements in huggingface bughunts #3395
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
32b4f9e
074c2f8
0191e5d
36c6540
1ee6043
99f82e7
a2eccbc
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -4,6 +4,7 @@ import ( | |
| "errors" | ||
| "fmt" | ||
| conancommand "github.com/jfrog/jfrog-cli-artifactory/artifactory/commands/conan" | ||
| "io/fs" | ||
| "os" | ||
| "os/exec" | ||
| "path/filepath" | ||
|
|
@@ -60,6 +61,8 @@ import ( | |
| gradledoc "github.com/jfrog/jfrog-cli/docs/buildtools/gradle" | ||
| "github.com/jfrog/jfrog-cli/docs/buildtools/gradleconfig" | ||
| "github.com/jfrog/jfrog-cli/docs/buildtools/huggingface" | ||
| huggingfacedownloaddocs "github.com/jfrog/jfrog-cli/docs/buildtools/huggingfacedownload" | ||
| huggingfaceuploaddocs "github.com/jfrog/jfrog-cli/docs/buildtools/huggingfaceupload" | ||
| mvndoc "github.com/jfrog/jfrog-cli/docs/buildtools/mvn" | ||
| "github.com/jfrog/jfrog-cli/docs/buildtools/mvnconfig" | ||
| "github.com/jfrog/jfrog-cli/docs/buildtools/npmcommand" | ||
|
|
@@ -84,10 +87,12 @@ import ( | |
| ) | ||
|
|
||
| const ( | ||
| buildToolsCategory = "Package Managers:" | ||
| huggingfaceAPI = "api/huggingfaceml" | ||
| HF_ENDPOINT = "HF_ENDPOINT" | ||
| HF_TOKEN = "HF_TOKEN" | ||
| buildToolsCategory = "Package Managers:" | ||
| huggingfaceAPI = "api/huggingfaceml" | ||
| HF_ENDPOINT = "HF_ENDPOINT" | ||
| HF_TOKEN = "HF_TOKEN" | ||
| HF_HUB_ETAG_TIMEOUT = "HF_HUB_ETAG_TIMEOUT" | ||
| HF_HUB_DOWNLOAD_TIMEOUT = "HF_HUB_DOWNLOAD_TIMEOUT" | ||
| ) | ||
|
|
||
| func GetCommands() []cli.Command { | ||
|
|
@@ -483,13 +488,36 @@ func GetCommands() []cli.Command { | |
| { | ||
| Name: "hugging-face", | ||
| Aliases: []string{"hf"}, | ||
| Flags: cliutils.GetCommandFlags(cliutils.HuggingFace), | ||
| HelpName: corecommon.CreateUsage("hugging-face", huggingface.GetDescription(), huggingface.Usage), | ||
| Description: huggingface.GetDescription(), | ||
| UsageText: huggingface.GetArguments(), | ||
| Hidden: true, | ||
| Action: huggingFaceCmd, | ||
| Category: buildToolsCategory, | ||
| Action: func(c *cli.Context) error { | ||
| if c.Args().Present() { | ||
| return fmt.Errorf("'%s %s' is not a valid subcommand. Run 'jf hf --help' for usage", c.App.Name, c.Args().First()) | ||
| } | ||
| return cli.ShowSubcommandHelp(c) | ||
| }, | ||
| Subcommands: []cli.Command{ | ||
| { | ||
| Name: "upload", | ||
| Aliases: []string{"u"}, | ||
| Flags: cliutils.GetCommandFlags(cliutils.HuggingFaceUpload), | ||
| HelpName: corecommon.CreateUsage("hf upload", huggingfaceuploaddocs.GetDescription(), huggingfaceuploaddocs.Usage), | ||
| Usage: huggingfaceuploaddocs.GetDescription(), | ||
| UsageText: huggingfaceuploaddocs.GetArguments(), | ||
| Action: huggingFaceUploadCmd, | ||
| }, | ||
| { | ||
| Name: "download", | ||
| Aliases: []string{"d"}, | ||
| Flags: cliutils.GetCommandFlags(cliutils.HuggingFaceDownload), | ||
| HelpName: corecommon.CreateUsage("hf download", huggingfacedownloaddocs.GetDescription(), huggingfacedownloaddocs.Usage), | ||
| Usage: huggingfacedownloaddocs.GetDescription(), | ||
| UsageText: huggingfacedownloaddocs.GetArguments(), | ||
| Action: huggingFaceDownloadCmd, | ||
| }, | ||
| }, | ||
| }, | ||
| }) | ||
| return decorateWithFlagCapture(cmds) | ||
|
|
@@ -1131,39 +1159,27 @@ func loginCmd(c *cli.Context) error { | |
| return nil | ||
| } | ||
|
|
||
| func huggingFaceCmd(c *cli.Context) error { | ||
| if show, err := cliutils.ShowCmdHelpIfNeeded(c, c.Args()); show || err != nil { | ||
| return err | ||
| } | ||
| if c.NArg() < 1 { | ||
| return cliutils.WrongNumberOfArgumentsHandler(c) | ||
| } | ||
| args := cliutils.ExtractCommand(c) | ||
| cmdName, hfArgs := getCommandName(args) | ||
| switch cmdName { | ||
| case "u", "upload": | ||
| return huggingFaceUploadCmd(c, "upload", hfArgs) | ||
| case "d", "download": | ||
| return huggingFaceDownloadCmd(c, "download", hfArgs) | ||
| default: | ||
| return errorutils.CheckErrorf("unknown HuggingFace command: '%s'. Valid commands are: upload (u), download (d)", cmdName) | ||
| } | ||
| } | ||
|
|
||
| func huggingFaceUploadCmd(c *cli.Context, cmdName string, hfArgs []string) error { | ||
| // Upload requires folderPath and repoID | ||
| if len(hfArgs) < 2 { | ||
| func huggingFaceUploadCmd(c *cli.Context) error { | ||
| if c.NArg() < 2 { | ||
| return cliutils.PrintHelpAndReturnError("Folder path and repository ID are required.", c) | ||
| } | ||
| folderPath := hfArgs[0] | ||
| folderPath := c.Args().Get(0) | ||
| if folderPath == "" { | ||
| return cliutils.PrintHelpAndReturnError("Folder path cannot be empty.", c) | ||
| } | ||
| repoID := hfArgs[1] | ||
| absPath, err := filepath.Abs(folderPath) | ||
| if err != nil { | ||
| return fmt.Errorf("failed to resolve absolute path: %w", err) | ||
| } | ||
| folderPath = absPath | ||
| if err = validateFolderHasUploadableFiles(folderPath); err != nil { | ||
| return err | ||
| } | ||
| repoID := c.Args().Get(1) | ||
| if repoID == "" { | ||
| return cliutils.PrintHelpAndReturnError("Repository ID cannot be empty.", c) | ||
| } | ||
| serverDetails, err := getHuggingFaceServerDetails(hfArgs) | ||
| serverDetails, err := getHuggingFaceServerDetails(c) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
@@ -1183,28 +1199,30 @@ func huggingFaceUploadCmd(c *cli.Context, cmdName string, hfArgs []string) error | |
| if repoType == "" { | ||
| repoType = "model" | ||
| } | ||
| huggingFaceUploadCmd := huggingfaceCommands.NewHuggingFaceUpload(). | ||
| SetCommandName(cmdName). | ||
| if repoType != "model" && repoType != "dataset" { | ||
| return fmt.Errorf("wrong repo type provided, allowed repo-type are : model and dataset") | ||
| } | ||
| cmd := huggingfaceCommands.NewHuggingFaceUpload(). | ||
| SetCommandName("upload"). | ||
| SetFolderPath(folderPath). | ||
| SetRepoId(repoID). | ||
| SetRepoType(repoType). | ||
| SetRevision(revision). | ||
| SetServerDetails(serverDetails). | ||
| SetBuildConfiguration(buildConfiguration) | ||
| return commands.Exec(huggingFaceUploadCmd) | ||
| return commands.Exec(cmd) | ||
| } | ||
|
|
||
| func huggingFaceDownloadCmd(c *cli.Context, cmdName string, hfArgs []string) error { | ||
| // Download requires repoID | ||
| if len(hfArgs) < 1 { | ||
| func huggingFaceDownloadCmd(c *cli.Context) error { | ||
| if c.NArg() < 1 { | ||
| return cliutils.PrintHelpAndReturnError("Model/Dataset name is required.", c) | ||
| } | ||
| const defaultETagTimeout = 86400 | ||
| repoID := hfArgs[0] | ||
| repoID := c.Args().Get(0) | ||
| if repoID == "" { | ||
| return cliutils.PrintHelpAndReturnError("Model/Dataset name cannot be empty.", c) | ||
| } | ||
| serverDetails, err := getHuggingFaceServerDetails(hfArgs) | ||
| serverDetails, err := getHuggingFaceServerDetails(c) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
@@ -1227,26 +1245,59 @@ func huggingFaceDownloadCmd(c *cli.Context, cmdName string, hfArgs []string) err | |
| if repoType == "" { | ||
| repoType = "model" | ||
| } | ||
| if repoType != "model" && repoType != "dataset" { | ||
| return fmt.Errorf("wrong repo type provided, allowed repo-type are : model and dataset") | ||
| } | ||
| revision := c.String("revision") | ||
| if revision == "" { | ||
| revision = "main" | ||
| } | ||
| huggingFaceDownloadCmd := huggingfaceCommands.NewHuggingFaceDownload(). | ||
| SetCommandName(cmdName). | ||
| cmd := huggingfaceCommands.NewHuggingFaceDownload(). | ||
| SetCommandName("download"). | ||
| SetRepoId(repoID). | ||
| SetRepoType(repoType). | ||
| SetRevision(revision). | ||
| SetEtagTimeout(etagTimeout). | ||
| SetServerDetails(serverDetails). | ||
| SetBuildConfiguration(buildConfiguration) | ||
| return commands.Exec(huggingFaceDownloadCmd) | ||
| return commands.Exec(cmd) | ||
| } | ||
|
|
||
| func getHuggingFaceServerDetails(args []string) (*coreConfig.ServerDetails, error) { | ||
| _, serverID, err := coreutils.ExtractServerIdFromCommand(args) | ||
| // validateFolderHasUploadableFiles walks the folder recursively and returns an error | ||
| // if no visible (non-hidden) regular files are found. Hidden entries — anything whose | ||
| // name starts with '.' (e.g. .git, .DS_Store) — are skipped entirely. | ||
| func validateFolderHasUploadableFiles(folderPath string) error { | ||
| found := false | ||
| err := filepath.WalkDir(folderPath, func(path string, d fs.DirEntry, walkErr error) error { | ||
| if walkErr != nil { | ||
| return walkErr | ||
| } | ||
| name := d.Name() | ||
| // Skip hidden files and directories (e.g. .git, .DS_Store). | ||
| if len(name) > 0 && name[0] == '.' { | ||
| if d.IsDir() { | ||
| return filepath.SkipDir | ||
| } | ||
| return nil | ||
| } | ||
| // A visible regular file counts as uploadable content. | ||
| if !d.IsDir() { | ||
| found = true | ||
| return filepath.SkipAll | ||
| } | ||
| return nil | ||
| }) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to extract server ID: %w", err) | ||
| return fmt.Errorf("failed to read folder '%s': %w", folderPath, err) | ||
| } | ||
| if !found { | ||
| return fmt.Errorf("folder '%s' contains no uploadable files (only hidden files or empty directories were found)", folderPath) | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| func getHuggingFaceServerDetails(c *cli.Context) (*coreConfig.ServerDetails, error) { | ||
| serverID := c.String("server-id") | ||
| if serverID == "" { | ||
| serverDetails, err := coreConfig.GetDefaultServerConf() | ||
| if err != nil { | ||
|
|
@@ -1257,34 +1308,45 @@ func getHuggingFaceServerDetails(args []string) (*coreConfig.ServerDetails, erro | |
| } | ||
| return serverDetails, nil | ||
| } | ||
| serverDetails, err := coreConfig.GetSpecificConfig(serverID, true, true) | ||
| serverDetails, err := coreConfig.GetSpecificConfig(serverID, true, false) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to get server configuration for ID '%s': %w", serverID, err) | ||
| } | ||
| return serverDetails, nil | ||
| } | ||
|
|
||
| func updateHuggingFaceEnv(c *cli.Context, serverDetails *coreConfig.ServerDetails) error { | ||
| if os.Getenv(HF_ENDPOINT) == "" { | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Removed checks for existing
Collaborator
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. If user has provided --repo-key flag, then its mandatory to set HF_ENDPOINT env value. |
||
| repoKey := c.String("repo-key") | ||
| if repoKey == "" { | ||
| return cliutils.PrintHelpAndReturnError("Please specify a repository key.", c) | ||
| } | ||
| repoKey := c.String("repo-key") | ||
| if repoKey != "" { | ||
| hfEndpoint := serverDetails.GetArtifactoryUrl() + huggingfaceAPI + "/" + repoKey | ||
| err := os.Setenv(HF_ENDPOINT, hfEndpoint) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| } | ||
| if os.Getenv(HF_TOKEN) == "" { | ||
| accessToken := serverDetails.GetAccessToken() | ||
| if accessToken == "" { | ||
| return cliutils.PrintHelpAndReturnError("You need to specify an access token.", c) | ||
| } | ||
| err := os.Setenv(HF_TOKEN, accessToken) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| accessToken := serverDetails.GetAccessToken() | ||
| if accessToken == "" { | ||
| return cliutils.PrintHelpAndReturnError("Access token is expired or missing, please either use rt ping command or update access token.", c) | ||
| } | ||
| err := os.Setenv(HF_TOKEN, accessToken) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| etagTimeout := c.Int("hf-hub-etag-timeout") | ||
| if etagTimeout == 0 { | ||
| etagTimeout = 86400 | ||
| } | ||
| err = os.Setenv(HF_HUB_ETAG_TIMEOUT, strconv.Itoa(etagTimeout)) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| downloadTimeout := c.Int("hf-hub-download-timeout") | ||
| if downloadTimeout == 0 { | ||
| downloadTimeout = 86400 | ||
| } | ||
| err = os.Setenv(HF_HUB_DOWNLOAD_TIMEOUT, strconv.Itoa(downloadTimeout)) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,28 @@ | ||
| package huggingfacedownload | ||
|
|
||
| var Usage = []string{"hf download <model-name>"} | ||
|
|
||
| func GetDescription() string { | ||
| return `Download a model or dataset from HuggingFace Hub.` | ||
| } | ||
|
|
||
| func GetArguments() string { | ||
| return ` model-name | ||
| The HuggingFace model repository ID (e.g., 'bert-base-uncased' or 'username/model-name'). | ||
|
|
||
| Command options: | ||
| --repo-key | ||
| [Mandatory] The Artifactory repository key to route the download through. | ||
|
|
||
| --revision | ||
| [Optional] The revision (commit hash, branch name, or tag) to download. Default: 'main'. | ||
|
|
||
| --repo-type | ||
| [Optional] The repository type. Can be 'model' or 'dataset'. Default: 'model'. | ||
|
|
||
| --hf-hub-etag-timeout | ||
| [Optional] Timeout in seconds for ETag validation. Default: 86400 (24 hours). | ||
|
|
||
| --hf-hub-download-timeout | ||
| [Optional] Timeout in seconds for Download. Default: 86400 (24 hours).` | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,31 @@ | ||
| package huggingfaceupload | ||
|
|
||
| var Usage = []string{"hf upload <folder-path> <repo-id>"} | ||
|
|
||
| func GetDescription() string { | ||
| return `Upload a model or dataset folder to HuggingFace Hub.` | ||
| } | ||
|
|
||
| func GetArguments() string { | ||
| return ` folder-path | ||
| Path to the folder to upload. | ||
|
|
||
| repo-id | ||
| The HuggingFace repository ID (e.g., 'username/model-name' or 'username/dataset-name'). | ||
|
|
||
| Command options: | ||
| --repo-key | ||
| [Mandatory] The Artifactory repository key to route the upload through. | ||
|
|
||
| --revision | ||
| [Optional] The revision (branch name, tag, or commit hash) to upload to. Default: 'main'. | ||
|
|
||
| --repo-type | ||
| [Optional] The repository type. Can be 'model' or 'dataset'. Default: 'model'. | ||
|
|
||
| --hf-hub-etag-timeout | ||
| [Optional] Timeout in seconds for ETag validation. Default: 86400 (24 hours). | ||
|
|
||
| --hf-hub-download-timeout | ||
| [Optional] Timeout in seconds for Download. Default: 86400 (24 hours).` | ||
| } |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Repo type validation only in upload command, not download. Should both validate?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Addressed it.