From 0c0d44e12913c9c79998447dc27f18d436647bd6 Mon Sep 17 00:00:00 2001 From: amithk Date: Mon, 26 Oct 2026 14:16:36 +0530 Subject: [PATCH] JFMIG-88 - Replace data-transfer plugin with GET/PUT streaming in transfer-files transfer-files now reads files from the source Artifactory and writes them to the target directly from the CLI, so no plugin is installed on the source. - Add source and target clients and the GET/PUT streaming file transfer, wired into transfer-files - Remove the data-transfer plugin path, including the transferinstall package and the plugin installer (JFMIG-93) - Restore --proxy-key as a target-only HTTP proxy URL - Fix silent data loss and hangs in GET/PUT transfer-files (JFMIG-93) - Treat folder-info's missing size field as 0, fix the over-broad cocoapods/swift generated-file filter, and apply gofmt to proxy.go --- .../commands/transferfiles/api/types.go | 102 +- .../commands/transferfiles/api/types_test.go | 51 - .../transferfiles/delayedartifactshandler.go | 8 +- .../commands/transferfiles/errorshandler.go | 26 +- .../transferfiles/errorshandler_test.go | 146 ++ .../commands/transferfiles/filesdiff.go | 14 +- .../commands/transferfiles/fileserror.go | 12 +- .../commands/transferfiles/filetransfer.go | 247 ++ .../transferfiles/filetransfer_http_test.go | 786 +++++++ .../transferfiles/filetransfer_test.go | 649 ++++++ .../commands/transferfiles/fulltransfer.go | 115 +- .../transferfiles/fulltransfer_test.go | 125 ++ .../transferfiles/generatedpathfilter.go | 186 ++ .../transferfiles/generatedpathfilter_test.go | 108 + .../commands/transferfiles/legacyfilters.go | 81 + .../transferfiles/legacyfilters_test.go | 191 ++ .../transferfiles/localgeneratedfilter.go | 13 +- .../localgeneratedfilter_test.go | 6 +- artifactory/commands/transferfiles/manager.go | 279 +-- artifactory/commands/transferfiles/phase.go | 22 +- .../transferfiles/producerconsumer.go | 34 +- artifactory/commands/transferfiles/proxy.go | 271 +++ .../commands/transferfiles/proxy_test.go | 372 +++ .../commands/transferfiles/sourceclient.go | 837 +++++++ .../transferfiles/sourceclient_test.go | 1990 +++++++++++++++++ .../transferfiles/srcpluginservice.go | 162 -- .../commands/transferfiles/state/runstatus.go | 30 +- .../transferfiles/state/runstatus_test.go | 41 + .../commands/transferfiles/state/state.go | 17 +- .../transferfiles/state/state_test.go | 28 + .../transferfiles/state/statemanager.go | 125 +- .../transferfiles/state/statemanager_test.go | 24 + .../transferfiles/state/timeestimation.go | 81 +- .../state/timeestimation_test.go | 34 +- .../transferfiles/state/transfersnapshot.go | 9 +- .../commands/transferfiles/state/utils.go | 20 - artifactory/commands/transferfiles/status.go | 26 +- .../commands/transferfiles/status_test.go | 63 +- .../commands/transferfiles/streamtransfer.go | 219 ++ .../transferfiles/streamtransfer_test.go | 490 ++++ .../commands/transferfiles/targetclient.go | 705 ++++++ .../transferfiles/targetclient_test.go | 1627 ++++++++++++++ .../commands/transferfiles/transfer.go | 185 +- .../transferfiles/transfer_files_test.go | 315 +++ .../commands/transferfiles/transfer_test.go | 486 ++-- artifactory/commands/transferfiles/utils.go | 423 ++-- .../commands/transferfiles/utils_test.go | 273 +-- .../transferinstall/datatransferinstall.go | 364 --- .../datatransferinstall_test.go | 256 --- artifactory/utils/storageinfo.go | 7 +- artifactory/utils/storageinfo_test.go | 28 + artifactory/utils/utils.go | 12 + utils/reposnapshot/snapshotmanager.go | 2 +- utils/reposnapshot/snapshotmanager_test.go | 33 +- 54 files changed, 10401 insertions(+), 2355 deletions(-) delete mode 100644 artifactory/commands/transferfiles/api/types_test.go create mode 100644 artifactory/commands/transferfiles/filetransfer.go create mode 100644 artifactory/commands/transferfiles/filetransfer_http_test.go create mode 100644 artifactory/commands/transferfiles/filetransfer_test.go create mode 100644 artifactory/commands/transferfiles/generatedpathfilter.go create mode 100644 artifactory/commands/transferfiles/generatedpathfilter_test.go create mode 100644 artifactory/commands/transferfiles/legacyfilters.go create mode 100644 artifactory/commands/transferfiles/legacyfilters_test.go create mode 100644 artifactory/commands/transferfiles/proxy.go create mode 100644 artifactory/commands/transferfiles/proxy_test.go create mode 100644 artifactory/commands/transferfiles/sourceclient.go create mode 100644 artifactory/commands/transferfiles/sourceclient_test.go delete mode 100644 artifactory/commands/transferfiles/srcpluginservice.go create mode 100644 artifactory/commands/transferfiles/streamtransfer.go create mode 100644 artifactory/commands/transferfiles/streamtransfer_test.go create mode 100644 artifactory/commands/transferfiles/targetclient.go create mode 100644 artifactory/commands/transferfiles/targetclient_test.go create mode 100644 artifactory/commands/transferfiles/transfer_files_test.go delete mode 100644 artifactory/commands/transferinstall/datatransferinstall.go delete mode 100644 artifactory/commands/transferinstall/datatransferinstall_test.go diff --git a/artifactory/commands/transferfiles/api/types.go b/artifactory/commands/transferfiles/api/types.go index 708ee49f0..28fd99138 100644 --- a/artifactory/commands/transferfiles/api/types.go +++ b/artifactory/commands/transferfiles/api/types.go @@ -1,56 +1,20 @@ package api -import ( - "fmt" - - "github.com/jfrog/jfrog-client-go/utils/log" -) - -type ProcessStatusType string type ChunkFileStatusType string -type ChunkId string -type NodeId string const ( - Done ProcessStatusType = "DONE" - InProgress ProcessStatusType = "IN_PROGRESS" - - Success ChunkFileStatusType = "SUCCESS" - Fail ChunkFileStatusType = "FAIL" - SkippedLargeProps ChunkFileStatusType = "SKIPPED_LARGE_PROPS" - SkippedMetadataFile ChunkFileStatusType = "SKIPPED_METADATA_FILE" - SkippedNonEmptyDir ChunkFileStatusType = "SKIPPED_NON_EMPTY_DIR" + Success ChunkFileStatusType = "SUCCESS" + Fail ChunkFileStatusType = "FAIL" + SkippedLargeProps ChunkFileStatusType = "SKIPPED_LARGE_PROPS" + SkippedMetadataFile ChunkFileStatusType = "SKIPPED_METADATA_FILE" + SkippedNonEmptyDir ChunkFileStatusType = "SKIPPED_NON_EMPTY_DIR" + SkippedSourceItemGone ChunkFileStatusType = "SKIPPED_SOURCE_ITEM_GONE" Phase1 int = 0 Phase2 int = 1 Phase3 int = 2 - - maxFilesInChunk = 16 - // 1 GiB - maxBytesInChunk = 1 << 30 ) -type TargetAuth struct { - TargetArtifactoryUrl string `json:"target_artifactory_url,omitempty"` - TargetUsername string `json:"target_username,omitempty"` - TargetPassword string `json:"target_password,omitempty"` - TargetToken string `json:"target_token,omitempty"` - TargetProxyKey string `json:"target_proxy_key,omitempty"` -} - -type UploadChunk struct { - // Authentication details of the target server - TargetAuth - // Files and folders to transfer - UploadCandidates []FileRepresentation `json:"upload_candidates,omitempty"` - // True if should check for the existence of artifacts on the target filestore - CheckExistenceInFilestore bool `json:"check_existence_in_filestore,omitempty"` - // True if should skip file filtering in the Data Transfer plugin - SkipFileFiltering bool `json:"skip_file_filtering,omitempty"` - // Minimum file size in bytes for which JFrog CLI performs checksum deploy optimization - MinCheckSumDeploySize int64 `json:"min_checksum_deploy_size,omitempty"` -} - type FileRepresentation struct { Repo string `json:"repo,omitempty"` Path string `json:"path,omitempty"` @@ -59,26 +23,8 @@ type FileRepresentation struct { NonEmptyDir bool `json:"non_empty_dir,omitempty"` } -type UploadChunkResponse struct { - NodeIdResponse - UuidTokenResponse -} - -type UploadChunksStatusBody struct { - AwaitingStatusChunks []ChunkId `json:"awaiting_status_chunks,omitempty"` - ChunksToDelete []ChunkId `json:"chunks_to_delete,omitempty"` -} - -type UploadChunksStatusResponse struct { - NodeIdResponse - ChunksStatus []ChunkStatus `json:"chunks_status,omitempty"` - DeletedChunks []string `json:"deleted_chunks,omitempty"` -} - type ChunkStatus struct { - UuidTokenResponse - Status ProcessStatusType `json:"status,omitempty"` - Files []FileUploadStatusResponse `json:"files,omitempty"` + Files []FileUploadStatusResponse `json:"files,omitempty"` } type FileUploadStatusResponse struct { @@ -89,37 +35,3 @@ type FileUploadStatusResponse struct { StatusCode int `json:"status_code,omitempty"` Reason string `json:"reason,omitempty"` } - -type NodeIdResponse struct { - NodeId string `json:"node_id,omitempty"` -} - -type UuidTokenResponse struct { - UuidToken string `json:"uuid_token,omitempty"` -} - -// Append upload candidate to the list of upload candidates. Skip empty directories in build-info repositories. -// file - The upload candidate -// buildInfoRepo - True if this is a build-info repository -func (uc *UploadChunk) AppendUploadCandidateIfNeeded(file FileRepresentation, buildInfoRepo bool) { - if buildInfoRepo && file.Name == "" { - log.Debug(fmt.Sprintf("Skipping unneeded empty dir '%s' in the build-info repository '%s'", file.Path, file.Repo)) - return - } - uc.UploadCandidates = append(uc.UploadCandidates, file) -} - -// Return true if the chunk contains at least 16 files or at least 1GiB in total -func (uc *UploadChunk) IsChunkFull() bool { - if len(uc.UploadCandidates) >= maxFilesInChunk { - return true - } - var totalSize int64 = 0 - for _, uploadCandidate := range uc.UploadCandidates { - totalSize += uploadCandidate.Size - if totalSize > maxBytesInChunk { - return true - } - } - return false -} diff --git a/artifactory/commands/transferfiles/api/types_test.go b/artifactory/commands/transferfiles/api/types_test.go deleted file mode 100644 index c2ef3b207..000000000 --- a/artifactory/commands/transferfiles/api/types_test.go +++ /dev/null @@ -1,51 +0,0 @@ -package api - -import ( - "fmt" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestAppendUploadCandidateIfNeeded(t *testing.T) { - uploadChunk := &UploadChunk{} - - // Regular file - uploadChunk.AppendUploadCandidateIfNeeded(FileRepresentation{Name: "regular-file"}, false) - assert.Len(t, uploadChunk.UploadCandidates, 1) - - // Build info - uploadChunk.AppendUploadCandidateIfNeeded(FileRepresentation{Name: "build-info.json"}, true) - assert.Len(t, uploadChunk.UploadCandidates, 2) - - // Directory in build info - should be skipped - uploadChunk.AppendUploadCandidateIfNeeded(FileRepresentation{}, true) - assert.Len(t, uploadChunk.UploadCandidates, 2) -} - -var isChunkFullCases = []struct { - files []FileRepresentation - isFull bool -}{ - {[]FileRepresentation{}, false}, - {[]FileRepresentation{{Name: "slim-jim", Size: 10737418}}, false}, - {[]FileRepresentation{{Name: "fat-vinny", Size: 1073741825}}, true}, -} - -func TestIsChunkFull(t *testing.T) { - for _, testCase := range isChunkFullCases { - t.Run("", func(t *testing.T) { - uploadChunk := &UploadChunk{UploadCandidates: testCase.files} - assert.Equal(t, testCase.isFull, uploadChunk.IsChunkFull()) - }) - } -} - -func TestIsChunkFullNumberOfFiles(t *testing.T) { - uploadChunk := &UploadChunk{} - for i := 0; i < maxFilesInChunk; i++ { - assert.False(t, uploadChunk.IsChunkFull()) - uploadChunk.AppendUploadCandidateIfNeeded(FileRepresentation{Name: fmt.Sprintf("%d", i)}, false) - } - assert.True(t, uploadChunk.IsChunkFull()) -} diff --git a/artifactory/commands/transferfiles/delayedartifactshandler.go b/artifactory/commands/transferfiles/delayedartifactshandler.go index 46beddec8..fa1e07f48 100644 --- a/artifactory/commands/transferfiles/delayedartifactshandler.go +++ b/artifactory/commands/transferfiles/delayedartifactshandler.go @@ -150,11 +150,11 @@ func countDelayFilesContent(filePaths []string) (count int, storage int64, err e func handleDelayedArtifactsFiles(filesToConsume []string, base phaseBase, delayUploadComparisonFunctions []shouldDelayUpload) error { manager := newTransferManager(base, delayUploadComparisonFunctions) - action := func(pcWrapper *producerConsumerWrapper, uploadChunkChan chan UploadedChunk, delayHelper delayUploadHelper, errorsChannelMng *ErrorsChannelMng) error { + action := func(pcWrapper *producerConsumerWrapper, delayHelper delayUploadHelper, errorsChannelMng *ErrorsChannelMng) error { if ShouldStop(&base, &delayHelper, errorsChannelMng) { return nil } - return consumeDelayedArtifactsFiles(pcWrapper, filesToConsume, uploadChunkChan, base, delayHelper, errorsChannelMng) + return consumeDelayedArtifactsFiles(pcWrapper, filesToConsume, base, delayHelper, errorsChannelMng) } delayAction := func(pBase phaseBase, addedDelayFiles []string) error { // We call this method as a recursion in order to have inner order base on the comparison function list. @@ -171,7 +171,7 @@ func handleDelayedArtifactsFiles(filesToConsume []string, base phaseBase, delayU return manager.doTransferWithProducerConsumer(action, delayAction) } -func consumeDelayedArtifactsFiles(pcWrapper *producerConsumerWrapper, filesToConsume []string, uploadChunkChan chan UploadedChunk, base phaseBase, delayHelper delayUploadHelper, errorsChannelMng *ErrorsChannelMng) error { +func consumeDelayedArtifactsFiles(pcWrapper *producerConsumerWrapper, filesToConsume []string, base phaseBase, delayHelper delayUploadHelper, errorsChannelMng *ErrorsChannelMng) error { log.Debug(fmt.Sprintf("Starting to handle delayed artifacts files. Found %d files.", len(filesToConsume))) for _, filePath := range filesToConsume { log.Debug("Handling delayed artifacts file: '" + filePath + "'") @@ -180,7 +180,7 @@ func consumeDelayedArtifactsFiles(pcWrapper *producerConsumerWrapper, filesToCon return err } - shouldStop, err := uploadByChunks(delayedArtifactsFile.DelayedArtifacts, uploadChunkChan, base, delayHelper, errorsChannelMng, pcWrapper) + shouldStop, err := transferFiles(delayedArtifactsFile.DelayedArtifacts, base, delayHelper, errorsChannelMng, pcWrapper) if err != nil || shouldStop { return err } diff --git a/artifactory/commands/transferfiles/errorshandler.go b/artifactory/commands/transferfiles/errorshandler.go index b075d03b9..d9ed296aa 100644 --- a/artifactory/commands/transferfiles/errorshandler.go +++ b/artifactory/commands/transferfiles/errorshandler.go @@ -170,7 +170,7 @@ func getErrorsFileNamePrefix(repoKey string, phaseId int, phaseStartTime string) func (mng *TransferErrorsMng) writeErrorContent(e ExtendedFileUploadStatusResponse) error { var err error switch e.Status { - case api.SkippedLargeProps: + case api.SkippedLargeProps, api.SkippedSourceItemGone: err = mng.writeSkippedErrorContent(e) default: err = mng.writeRetryableErrorContent(e) @@ -276,8 +276,7 @@ func createErrorsCsvSummary(sourceRepos []string, timeStarted time.Time) (string return "", err } - errorsFiles = append(errorsFiles, skippedErrorsFiles...) - if len(errorsFiles) == 0 { + if len(errorsFiles)+len(skippedErrorsFiles) == 0 { return "", nil } // Collect all errors from the given log files @@ -285,9 +284,30 @@ func createErrorsCsvSummary(sourceRepos []string, timeStarted time.Time) (string if err != nil { return "", err } + skippedErrors, err := parseErrorsFromLogFiles(skippedErrorsFiles) + if err != nil { + return "", err + } + allErrors.Errors = append(allErrors.Errors, dedupeSkippedRows(skippedErrors.Errors)...) return cmdutils.CreateCSVFile("transfer-files-logs", allErrors.Errors, timeStarted) } +// dedupeSkippedRows keeps one row per repo/path/name/status. The same item can be skipped by more +// than one phase (or run), which would otherwise list it several times in the summary. +func dedupeSkippedRows(rows []ExtendedFileUploadStatusResponse) []ExtendedFileUploadStatusResponse { + seen := make(map[string]struct{}, len(rows)) + deduped := make([]ExtendedFileUploadStatusResponse, 0, len(rows)) + for _, row := range rows { + key := fmt.Sprintf("%s\x00%s\x00%s\x00%s", row.Repo, row.Path, row.Name, row.Status) + if _, exists := seen[key]; exists { + continue + } + seen[key] = struct{}{} + deduped = append(deduped, row) + } + return deduped +} + // Gets a list of all errors files from the CLI's cache. // Errors-files contain files that were failed to upload or actions that were skipped because of known limitations. func getErrorsFiles(repoKeys []string, isRetry bool) (filesPaths []string, err error) { diff --git a/artifactory/commands/transferfiles/errorshandler_test.go b/artifactory/commands/transferfiles/errorshandler_test.go index 3c3d70868..b17e73f06 100644 --- a/artifactory/commands/transferfiles/errorshandler_test.go +++ b/artifactory/commands/transferfiles/errorshandler_test.go @@ -11,6 +11,7 @@ import ( "math" "os" "path/filepath" + "strings" "sync" "testing" "time" @@ -68,6 +69,127 @@ func TestTransferErrorsMng(t *testing.T) { assert.Equal(t, errorsNumber, retryEntityCount) } +// TestTransferErrorsMng_sourceItemGoneGoesToSkippedNotRetryable verifies that items skipped as +// "source item gone" are auditable in the errors CSV via the skipped errors files, without being +// counted as failures or queued for retry. +func TestTransferErrorsMng_sourceItemGoneGoesToSkippedNotRetryable(t *testing.T) { + cleanUpJfrogHome, err := tests.SetJfrogHome() + assert.NoError(t, err) + defer cleanUpJfrogHome() + + errorsChannelMng := createErrorsChannelMng() + transferErrorsMng, err := newTransferErrorsToFile(testRepoKey, 0, state.ConvertTimeToEpochMilliseconds(time.Now()), &errorsChannelMng, nil, nil) + assert.NoError(t, err) + + var readWaitGroup sync.WaitGroup + var writingErrorsErr error + readWaitGroup.Add(1) + go func() { + defer readWaitGroup.Done() + writingErrorsErr = transferErrorsMng.start() + }() + + const goneNumber = 3 + for i := 0; i < goneNumber; i++ { + errorsChannelMng.add(api.FileUploadStatusResponse{ + FileRepresentation: api.FileRepresentation{Repo: testRepoKey, Path: "path", Name: fmt.Sprintf("gone%d", i)}, + Status: api.SkippedSourceItemGone, + Reason: ErrSourceItemGone.Error(), + }) + } + errorsChannelMng.close() + readWaitGroup.Wait() + assert.NoError(t, writingErrorsErr) + + skippedFiles, err := getErrorsFiles([]string{testRepoKey}, false) + assert.NoError(t, err) + var skippedEntities int + for _, file := range skippedFiles { + errs, readErr := readErrorFile(file) + assert.NoError(t, readErr) + for _, e := range errs.Errors { + assert.Equal(t, api.SkippedSourceItemGone, e.Status) + assert.Equal(t, ErrSourceItemGone.Error(), e.Reason) + } + skippedEntities += len(errs.Errors) + } + assert.Equal(t, goneNumber, skippedEntities) + + retryCount, err := getRetryErrorCount([]string{testRepoKey}) + assert.NoError(t, err) + assert.Zero(t, retryCount, "gone items are not retryable errors") +} + +// writeErrorRows writes the given rows through a TransferErrorsMng of the given phase, the way a +// transfer phase does (retryable and skipped statuses land in their own directories). +func writeErrorRows(t *testing.T, repoKey string, phaseId int, rows ...api.FileUploadStatusResponse) { + errorsChannelMng := createErrorsChannelMng() + mng, err := newTransferErrorsToFile(repoKey, phaseId, state.ConvertTimeToEpochMilliseconds(time.Now()), &errorsChannelMng, nil, nil) + assert.NoError(t, err) + var wg sync.WaitGroup + var writeErr error + wg.Add(1) + go func() { + defer wg.Done() + writeErr = mng.start() + }() + for _, row := range rows { + errorsChannelMng.add(row) + } + errorsChannelMng.close() + wg.Wait() + assert.NoError(t, writeErr) +} + +func goneRow(repoKey, name string) api.FileUploadStatusResponse { + return api.FileUploadStatusResponse{ + FileRepresentation: api.FileRepresentation{Repo: repoKey, Path: "path", Name: name}, + Status: api.SkippedSourceItemGone, + Reason: ErrSourceItemGone.Error(), + } +} + +// TestCreateErrorsCsvSummary_dedupesSkippedRowsByPath: the same path skipped in two phases (or runs) +// must appear once in the errors CSV. +func TestCreateErrorsCsvSummary_dedupesSkippedRowsByPath(t *testing.T) { + cleanUpJfrogHome, err := tests.SetJfrogHome() + assert.NoError(t, err) + defer cleanUpJfrogHome() + + writeErrorRows(t, testRepoKey, 1, goneRow(testRepoKey, "a.bin"), goneRow(testRepoKey, "b.bin")) + writeErrorRows(t, testRepoKey, 2, goneRow(testRepoKey, "a.bin")) + + csvPath, err := createErrorsCsvSummary([]string{testRepoKey}, time.Now()) + assert.NoError(t, err) + assert.NotEmpty(t, csvPath) + csvContent, err := os.ReadFile(csvPath) + assert.NoError(t, err) + assert.Equal(t, 1, strings.Count(string(csvContent), "a.bin"), "duplicate skipped rows for one path must be collapsed") + assert.Equal(t, 1, strings.Count(string(csvContent), "b.bin")) +} + +// TestErrorsRetryPhase_goneRowsAreNotRetried documents that items skipped as "source item gone" +// are audit-only: the retry phase reads the retryable directory only, so a rerun does not retry them. +func TestErrorsRetryPhase_goneRowsAreNotRetried(t *testing.T) { + cleanUpJfrogHome, err := tests.SetJfrogHome() + assert.NoError(t, err) + defer cleanUpJfrogHome() + + writeErrorRows(t, testRepoKey, 1, goneRow(testRepoKey, "a.bin")) + retryPhase := &errorsRetryPhase{phaseBase: phaseBase{repoKey: testRepoKey}} + skip, err := retryPhase.shouldSkipPhase() + assert.NoError(t, err) + assert.True(t, skip, "only gone rows exist: nothing to retry") + assert.Empty(t, retryPhase.errorsFilesToHandle) + + writeErrorRows(t, testRepoKey, 2, api.FileUploadStatusResponse{ + FileRepresentation: api.FileRepresentation{Repo: testRepoKey, Path: "path", Name: "failed.bin"}, Status: api.Fail, Reason: "boom"}) + skip, err = retryPhase.shouldSkipPhase() + assert.NoError(t, err) + assert.False(t, skip, "a real failure is still retried") + assert.Len(t, retryPhase.errorsFilesToHandle, 1) +} + func addErrorsToChannel(writeWaitGroup *sync.WaitGroup, errorsNumber int, errorsChannelMng ErrorsChannelMng, status api.ChunkFileStatusType) { writeWaitGroup.Add(1) go func() { @@ -167,3 +289,27 @@ func writeEmptyErrorsFile(t *testing.T, repoKey string, retryable bool, phase, c fileName := fmt.Sprintf("%s-%d.json", getErrorsFileNamePrefix(repoKey, phase, state.ConvertTimeToEpochMilliseconds(time.Now())), counter) assert.NoError(t, os.WriteFile(filepath.Join(errorsDirPath, fileName), nil, 0644)) } + +func TestDedupeSkippedRows_keyIncludesRepoPathNameAndStatus(t *testing.T) { + row := func(repo, dir, name string, status api.ChunkFileStatusType) ExtendedFileUploadStatusResponse { + return ExtendedFileUploadStatusResponse{FileUploadStatusResponse: api.FileUploadStatusResponse{ + FileRepresentation: api.FileRepresentation{Repo: repo, Path: dir, Name: name}, + Status: status, + }} + } + rows := []ExtendedFileUploadStatusResponse{ + row("repo-a", "dir1", "same.txt", api.SkippedSourceItemGone), + row("repo-a", "dir1", "same.txt", api.SkippedSourceItemGone), // exact duplicate: collapses + row("repo-a", "dir2", "same.txt", api.SkippedSourceItemGone), // other directory: kept + row("repo-b", "dir1", "same.txt", api.SkippedSourceItemGone), // other repo: kept + row("repo-a", "dir1", "other.txt", api.SkippedSourceItemGone), // other name: kept + row("repo-a", "dir1", "same.txt", api.SkippedLargeProps), // other status: kept + } + deduped := dedupeSkippedRows(rows) + if !assert.Len(t, deduped, 5) { + return + } + assert.Equal(t, rows[0], deduped[0]) + assert.Equal(t, rows[2], deduped[1]) + assert.Equal(t, rows[3], deduped[2]) +} diff --git a/artifactory/commands/transferfiles/filesdiff.go b/artifactory/commands/transferfiles/filesdiff.go index ef32aa8a4..c7c324000 100644 --- a/artifactory/commands/transferfiles/filesdiff.go +++ b/artifactory/commands/transferfiles/filesdiff.go @@ -70,12 +70,12 @@ func (f *filesDiffPhase) handleDiffTimeFrames() error { } f.transferManager = newTransferManager(f.phaseBase, getDelayUploadComparisonFunctions(f.repoSummary.PackageType)) - action := func(pcWrapper *producerConsumerWrapper, uploadChunkChan chan UploadedChunk, delayHelper delayUploadHelper, errorsChannelMng *ErrorsChannelMng) error { + action := func(pcWrapper *producerConsumerWrapper, delayHelper delayUploadHelper, errorsChannelMng *ErrorsChannelMng) error { // Create tasks to handle files diffs in time frames of searchTimeFramesMinutes. // In case an error occurred while handling errors/delayed artifacts files - stop transferring. curDiffTimeFrame := diffRangeStart for diffRangeEnd.Sub(curDiffTimeFrame) > 0 && !ShouldStop(&f.phaseBase, &delayHelper, errorsChannelMng) { - diffTimeFrameHandler := f.createDiffTimeFrameHandlerFunc(pcWrapper, uploadChunkChan, delayHelper, errorsChannelMng) + diffTimeFrameHandler := f.createDiffTimeFrameHandlerFunc(pcWrapper, delayHelper, errorsChannelMng) _, err = pcWrapper.chunkBuilderProducerConsumer.AddTaskWithError(diffTimeFrameHandler(timeFrameParams{fromTime: curDiffTimeFrame}), pcWrapper.errorsQueue.AddError) if err != nil { return err @@ -98,16 +98,16 @@ type timeFrameParams struct { fromTime time.Time } -func (f *filesDiffPhase) createDiffTimeFrameHandlerFunc(pcWrapper *producerConsumerWrapper, uploadChunkChan chan UploadedChunk, delayHelper delayUploadHelper, errorsChannelMng *ErrorsChannelMng) diffTimeFrameHandlerFunc { +func (f *filesDiffPhase) createDiffTimeFrameHandlerFunc(pcWrapper *producerConsumerWrapper, delayHelper delayUploadHelper, errorsChannelMng *ErrorsChannelMng) diffTimeFrameHandlerFunc { return func(params timeFrameParams) parallel.TaskFunc { return func(threadId int) error { logMsgPrefix := clientUtils.GetLogMsgPrefix(threadId, false) - return f.handleTimeFrameFilesDiff(pcWrapper, params, logMsgPrefix, uploadChunkChan, delayHelper, errorsChannelMng) + return f.handleTimeFrameFilesDiff(pcWrapper, params, logMsgPrefix, delayHelper, errorsChannelMng) } } } -func (f *filesDiffPhase) handleTimeFrameFilesDiff(pcWrapper *producerConsumerWrapper, params timeFrameParams, logMsgPrefix string, uploadChunkChan chan UploadedChunk, delayHelper delayUploadHelper, errorsChannelMng *ErrorsChannelMng) error { +func (f *filesDiffPhase) handleTimeFrameFilesDiff(pcWrapper *producerConsumerWrapper, params timeFrameParams, logMsgPrefix string, delayHelper delayUploadHelper, errorsChannelMng *ErrorsChannelMng) error { fromTimestamp := params.fromTime.Format(time.RFC3339) toTimestamp := params.fromTime.Add(searchTimeFramesMinutes * time.Minute).Format(time.RFC3339) log.Debug(logMsgPrefix + "Searching time frame: '" + fromTimestamp + "' to '" + toTimestamp + "'") @@ -142,7 +142,7 @@ func (f *filesDiffPhase) handleTimeFrameFilesDiff(pcWrapper *producerConsumerWra if f.progressBar != nil { f.progressBar.phases[f.phaseId].GetTasksProgressBar().SetGeneralProgressTotal(*storage) } - shouldStop, err := uploadByChunks(files, uploadChunkChan, f.phaseBase, delayHelper, errorsChannelMng, pcWrapper) + shouldStop, err := transferFiles(files, f.phaseBase, delayHelper, errorsChannelMng, pcWrapper) if err != nil || shouldStop { return err } @@ -210,7 +210,7 @@ func (f *filesDiffPhase) getTimeFrameFilesDiff(fromTimestamp, toTimestamp string return []servicesUtils.ResultItem{}, true, err } lastPage = len(timeFrameFilesDiff.Results) < AqlPaginationLimit - result, err = f.locallyGeneratedFilter.FilterLocallyGenerated(timeFrameFilesDiff.Results) + result, err = f.locallyGeneratedFilter.FilterLocallyGenerated(timeFrameFilesDiff.Results, f.packageType) return } diff --git a/artifactory/commands/transferfiles/fileserror.go b/artifactory/commands/transferfiles/fileserror.go index faa1aad52..e352bb0b8 100644 --- a/artifactory/commands/transferfiles/fileserror.go +++ b/artifactory/commands/transferfiles/fileserror.go @@ -37,8 +37,8 @@ func (e *errorsRetryPhase) handlePreviousUploadFailures() error { } log.Info("Starting to handle previous upload failures...") e.transferManager = newTransferManager(e.phaseBase, getDelayUploadComparisonFunctions(e.repoSummary.PackageType)) - action := func(pcWrapper *producerConsumerWrapper, uploadChunkChan chan UploadedChunk, delayHelper delayUploadHelper, errorsChannelMng *ErrorsChannelMng) error { - errFileHandler := e.createErrorFilesHandleFunc(pcWrapper, uploadChunkChan, delayHelper, errorsChannelMng) + action := func(pcWrapper *producerConsumerWrapper, delayHelper delayUploadHelper, errorsChannelMng *ErrorsChannelMng) error { + errFileHandler := e.createErrorFilesHandleFunc(pcWrapper, delayHelper, errorsChannelMng) _, err := pcWrapper.chunkBuilderProducerConsumer.AddTaskWithError(errFileHandler(), pcWrapper.errorsQueue.AddError) return err } @@ -61,7 +61,7 @@ func convertUploadStatusToFileRepresentation(statuses []ExtendedFileUploadStatus return } -func (e *errorsRetryPhase) handleErrorsFile(errFilePath string, pcWrapper *producerConsumerWrapper, uploadChunkChan chan UploadedChunk, delayHelper delayUploadHelper, errorsChannelMng *ErrorsChannelMng) error { +func (e *errorsRetryPhase) handleErrorsFile(errFilePath string, pcWrapper *producerConsumerWrapper, delayHelper delayUploadHelper, errorsChannelMng *ErrorsChannelMng) error { if ShouldStop(&e.phaseBase, &delayHelper, errorsChannelMng) { return nil } @@ -84,17 +84,17 @@ func (e *errorsRetryPhase) handleErrorsFile(errFilePath string, pcWrapper *produ } // Upload - _, err = uploadByChunks(convertUploadStatusToFileRepresentation(failedFiles.Errors), uploadChunkChan, e.phaseBase, delayHelper, errorsChannelMng, pcWrapper) + _, err = transferFiles(convertUploadStatusToFileRepresentation(failedFiles.Errors), e.phaseBase, delayHelper, errorsChannelMng, pcWrapper) return err } -func (e *errorsRetryPhase) createErrorFilesHandleFunc(pcWrapper *producerConsumerWrapper, uploadChunkChan chan UploadedChunk, delayHelper delayUploadHelper, errorsChannelMng *ErrorsChannelMng) errorFileHandlerFunc { +func (e *errorsRetryPhase) createErrorFilesHandleFunc(pcWrapper *producerConsumerWrapper, delayHelper delayUploadHelper, errorsChannelMng *ErrorsChannelMng) errorFileHandlerFunc { return func() parallel.TaskFunc { return func(int) error { var errList []string var err error for _, errFile := range e.errorsFilesToHandle { - err = e.handleErrorsFile(errFile, pcWrapper, uploadChunkChan, delayHelper, errorsChannelMng) + err = e.handleErrorsFile(errFile, pcWrapper, delayHelper, errorsChannelMng) if err != nil { errList = append(errList, fmt.Sprintf("handleErrorsFile for %s failed with error: \n%s", errFile, err.Error())) } diff --git a/artifactory/commands/transferfiles/filetransfer.go b/artifactory/commands/transferfiles/filetransfer.go new file mode 100644 index 000000000..030e55e03 --- /dev/null +++ b/artifactory/commands/transferfiles/filetransfer.go @@ -0,0 +1,247 @@ +package transferfiles + +import ( + "context" + "errors" + "io" + "sync" + "sync/atomic" + "time" + + "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/api" + "github.com/jfrog/jfrog-client-go/utils/log" +) + +type fileTransferSource interface { + GetFileMetadata(ctx context.Context, file api.FileRepresentation) (*SourceFileMetadata, error) + GetFileReader(ctx context.Context, file api.FileRepresentation) (io.ReadCloser, error) +} + +type fileTransferTarget interface { + TryChecksumDeploy(ctx context.Context, metadata *SourceFileMetadata, options TargetDeployOptions) (ChecksumDeployOutcome, error) + Put(ctx context.Context, metadata *SourceFileMetadata, reader io.Reader, options TargetDeployOptions) error + CreateFolder(ctx context.Context, metadata *SourceFileMetadata, options TargetDeployOptions) error + ApplyProperties(ctx context.Context, metadata *SourceFileMetadata, options TargetDeployOptions) (skippedLargeProps bool, err error) + ApplyStatistics(ctx context.Context, metadata *SourceFileMetadata, options TargetDeployOptions) error + ReleaseEligibleProperties(metadata *SourceFileMetadata, options TargetDeployOptions) + HasEligibleProperties(metadata *SourceFileMetadata, options TargetDeployOptions) bool +} + +type FileTransferOptions struct { + TargetDeployOptions TargetDeployOptions + StreamBufferSize int +} + +type TransferResult struct { + Candidate api.FileRepresentation + Status api.ChunkFileStatusType + Err error + ChecksumDeployed bool + BytesTransferred int64 + FileSize int64 + DurationMillis int64 + SkippedLargeProps bool + SourceItemGone bool +} + +func (r TransferResult) ToFileUploadStatus() api.FileUploadStatusResponse { + status := api.FileUploadStatusResponse{ + FileRepresentation: r.Candidate, + SizeBytes: r.FileSize, + ChecksumDeployed: r.ChecksumDeployed, + Status: r.Status, + } + if r.Err != nil { + status.Reason = r.Err.Error() + } else if r.SourceItemGone { + status.Reason = ErrSourceItemGone.Error() + } + return status +} + +type FileTransfer struct { + source fileTransferSource + target fileTransferTarget + options FileTransferOptions + optionsLock sync.RWMutex +} + +func NewFileTransfer(source fileTransferSource, target fileTransferTarget, options FileTransferOptions) *FileTransfer { + return &FileTransfer{ + source: source, + target: target, + options: options, + } +} + +// ConfigureOptions updates deploy options for subsequent transfers. +func (ft *FileTransfer) ConfigureOptions(options FileTransferOptions) { + ft.optionsLock.Lock() + defer ft.optionsLock.Unlock() + ft.options = options +} + +// sourceItemGoneSkipCount counts items skipped as "source item gone" during this run, so the +// end-of-run summary can tell the user how many were skipped. +var sourceItemGoneSkipCount atomic.Int64 + +// takeSourceItemGoneSkipCount returns the number of items skipped as "source item gone" since +// the last call, and resets the counter. +func takeSourceItemGoneSkipCount() int64 { + return sourceItemGoneSkipCount.Swap(0) +} + +// logSourceItemGone records every source-item-gone skip so it's auditable post-run instead +// of vanishing silently into a "success" summary. +func logSourceItemGone(candidate api.FileRepresentation, stage string) { + sourceItemGoneSkipCount.Add(1) + log.Warn("Source item no longer exists, skipping:", fileRelativePath(candidate)+". Detected while:", stage) +} + +func (ft *FileTransfer) TransferFile(ctx context.Context, candidate api.FileRepresentation) TransferResult { + startTime := time.Now() + result := TransferResult{Candidate: candidate, Status: api.Success} + options := ft.getOptions() + if err := ctx.Err(); err != nil { + return ft.finalizeResult(ft.failResult(result, err), startTime, nil, candidate) + } + + metadata, err := ft.source.GetFileMetadata(ctx, candidate) + if ctxErr := ctx.Err(); ctxErr != nil { + return ft.finalizeResult(ft.failResult(result, errors.Join(err, ctxErr)), startTime, nil, candidate) + } + if err != nil { + if IsSourceItemGone(err) { + logSourceItemGone(candidate, "fetching metadata") + result.SourceItemGone = true + result.Status = api.SkippedSourceItemGone + return ft.finalizeResult(result, startTime, nil, candidate) + } + return ft.finalizeResult(ft.failResult(result, err), startTime, nil, candidate) + } + defer ft.target.ReleaseEligibleProperties(metadata, options.TargetDeployOptions) + + if metadata.Name == "" { + return ft.transferFolder(ctx, result, metadata, startTime, candidate, options) + } + + outcome, err := ft.target.TryChecksumDeploy(ctx, metadata, options.TargetDeployOptions) + if ctxErr := ctx.Err(); ctxErr != nil { + return ft.finalizeResult(ft.failResult(result, errors.Join(err, ctxErr)), startTime, metadata, candidate) + } + if err != nil { + return ft.finalizeResult(ft.failResult(result, err), startTime, metadata, candidate) + } + if outcome == ChecksumDeployHit { + result.ChecksumDeployed = true + return ft.finalizeResult(ft.applyPropertiesAndStats(ctx, result, metadata, options), startTime, metadata, candidate) + } + + reader, err := ft.source.GetFileReader(ctx, candidate) + if ctxErr := ctx.Err(); ctxErr != nil { + closeReadCloser(reader) + return ft.finalizeResult(ft.failResult(result, errors.Join(err, ctxErr)), startTime, metadata, candidate) + } + if err != nil { + closeReadCloser(reader) + if IsSourceItemGone(err) { + logSourceItemGone(candidate, "reading file content") + result.SourceItemGone = true + result.Status = api.SkippedSourceItemGone + return ft.finalizeResult(result, startTime, metadata, candidate) + } + return ft.finalizeResult(ft.failResult(result, err), startTime, metadata, candidate) + } + + bufferSize := options.StreamBufferSize + if bufferSize <= 0 { + bufferSize = defaultStreamBufferSize + } + bytesTransferred, err := StreamGetToPut(ctx, metadata.Size, reader, func(streamCtx context.Context, streamReader io.Reader) error { + return ft.target.Put(streamCtx, metadata, streamReader, options.TargetDeployOptions) + }, bufferSize) + if err != nil { + return ft.finalizeResult(ft.failResult(result, err), startTime, metadata, candidate) + } + result.BytesTransferred = bytesTransferred + + return ft.finalizeResult(ft.applyPropertiesAndStats(ctx, result, metadata, options), startTime, metadata, candidate) +} + +func (ft *FileTransfer) transferFolder(ctx context.Context, result TransferResult, metadata *SourceFileMetadata, startTime time.Time, candidate api.FileRepresentation, options FileTransferOptions) TransferResult { + if candidate.NonEmptyDir && !ft.target.HasEligibleProperties(metadata, options.TargetDeployOptions) { + result.Status = api.SkippedNonEmptyDir + return ft.finalizeResult(result, startTime, metadata, candidate) + } + err := ft.target.CreateFolder(ctx, metadata, options.TargetDeployOptions) + if ctxErr := ctx.Err(); ctxErr != nil { + return ft.finalizeResult(ft.failResult(result, errors.Join(err, ctxErr)), startTime, metadata, candidate) + } + if err != nil { + return ft.finalizeResult(ft.failResult(result, err), startTime, metadata, candidate) + } + return ft.finalizeResult(ft.applyPropertiesAndStats(ctx, result, metadata, options), startTime, metadata, candidate) +} + +func (ft *FileTransfer) applyPropertiesAndStats(ctx context.Context, result TransferResult, metadata *SourceFileMetadata, options FileTransferOptions) TransferResult { + skippedLargeProps, err := ft.target.ApplyProperties(ctx, metadata, options.TargetDeployOptions) + if ctxErr := ctx.Err(); ctxErr != nil { + return failAfterContentTransfer(result, errors.Join(err, ctxErr)) + } + if err != nil { + return failAfterContentTransfer(result, err) + } + result.SkippedLargeProps = skippedLargeProps + if skippedLargeProps { + result.Status = api.SkippedLargeProps + } + err = ft.target.ApplyStatistics(ctx, metadata, options.TargetDeployOptions) + if ctxErr := ctx.Err(); ctxErr != nil { + return failAfterContentTransfer(result, errors.Join(err, ctxErr)) + } + if err != nil { + return failAfterContentTransfer(result, err) + } + return result +} + +func (ft *FileTransfer) getOptions() FileTransferOptions { + ft.optionsLock.RLock() + defer ft.optionsLock.RUnlock() + return ft.options +} + +func (ft *FileTransfer) failResult(result TransferResult, err error) TransferResult { + result.Status = api.Fail + result.Err = err + result.BytesTransferred = 0 + return result +} + +// failAfterContentTransfer marks the item failed without clearing BytesTransferred. +// Content may already have landed on the target (checksum-deploy hit or a completed stream) +// before properties/stats failed. +func failAfterContentTransfer(result TransferResult, err error) TransferResult { + result.Status = api.Fail + result.Err = err + return result +} + +func (ft *FileTransfer) finalizeResult(result TransferResult, startTime time.Time, metadata *SourceFileMetadata, candidate api.FileRepresentation) TransferResult { + result.DurationMillis = time.Since(startTime).Milliseconds() + if result.SourceItemGone { + result.FileSize = 0 + return result + } + if metadata != nil { + result.FileSize = metadata.Size + } else { + result.FileSize = candidate.Size + } + return result +} + +var ( + _ fileTransferSource = (*SourceClient)(nil) + _ fileTransferTarget = (*TargetClient)(nil) +) diff --git a/artifactory/commands/transferfiles/filetransfer_http_test.go b/artifactory/commands/transferfiles/filetransfer_http_test.go new file mode 100644 index 000000000..14c7e8f2c --- /dev/null +++ b/artifactory/commands/transferfiles/filetransfer_http_test.go @@ -0,0 +1,786 @@ +package transferfiles + +import ( + "bytes" + "context" + "crypto/md5" // #nosec G501 -- MD5 is an Artifactory checksum type, not used for security + "crypto/sha1" // #nosec G505 -- SHA-1 is an Artifactory checksum type, not used for security + "crypto/sha256" + "encoding/hex" + "encoding/json" + "io" + "maps" + "net/http" + "net/http/httptest" + "net/url" + "runtime" + "strconv" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/api" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type harnessBlob struct { + sha1 string + sha256 string + md5 string + size int64 + body []byte + generatedByte byte + generated bool + properties map[string][]string +} + +type transferHTTPHarness struct { + mu sync.Mutex + + sourceStore map[string]*harnessBlob + targetStore map[string]*harnessBlob + targetBySha1 map[string]*harnessBlob + targetStaged map[string][]byte + + sourceDownloadGETs int + sourceReaderClosed atomic.Bool + sourceReaderClosedCh chan struct{} + sourceReaderCloseOnce sync.Once + sourceMaxReadLen atomic.Int64 + checksumPUTs int + checksumPutBodyBytes int64 + fullPUTs int + failFirstFullPutAfter int + fullPutAttempts int + + afterFullPutBytes func(n int) + blockFullPutAfter int + fullPutBlocked chan struct{} + releaseFullPut chan struct{} + blockFullPutOnce sync.Once + fullPutDone chan struct{} + fullPutDoneOnce sync.Once + discardFullPutBody bool + + sourceServer *httptest.Server + targetServer *httptest.Server +} + +func newTransferHTTPHarness(t *testing.T) *transferHTTPHarness { + t.Helper() + h := &transferHTTPHarness{ + sourceStore: make(map[string]*harnessBlob), + targetStore: make(map[string]*harnessBlob), + targetBySha1: make(map[string]*harnessBlob), + targetStaged: make(map[string][]byte), + sourceReaderClosedCh: make(chan struct{}), + } + h.sourceServer = httptest.NewServer(http.HandlerFunc(h.serveSource)) + h.targetServer = httptest.NewServer(http.HandlerFunc(h.serveTarget)) + t.Cleanup(func() { + h.sourceServer.Close() + h.targetServer.Close() + }) + return h +} + +func (h *transferHTTPHarness) seedGeneratedSource(relativePath string, size int64, b byte) *harnessBlob { + sha1Sum, sha256Sum, md5Sum := checksumsForRepeatingBytes(size, b) + blob := &harnessBlob{ + sha1: sha1Sum, + sha256: sha256Sum, + md5: md5Sum, + size: size, + generatedByte: b, + generated: true, + } + h.mu.Lock() + defer h.mu.Unlock() + h.sourceStore[relativePath] = blob + return blob +} + +func (h *transferHTTPHarness) seedSource(relativePath string, body []byte, properties map[string][]string) *harnessBlob { + blob := newHarnessBlob(body, properties) + h.mu.Lock() + defer h.mu.Unlock() + h.sourceStore[relativePath] = blob + return blob +} + +func (h *transferHTTPHarness) seedTarget(relativePath string, body []byte, properties map[string][]string) *harnessBlob { + blob := newHarnessBlob(body, properties) + h.mu.Lock() + defer h.mu.Unlock() + h.targetStore[relativePath] = blob + h.targetBySha1[blob.sha1] = cloneHarnessBlob(blob) + return blob +} + +func (h *transferHTTPHarness) seedTargetBySha1(body []byte) *harnessBlob { + blob := newHarnessBlob(body, nil) + h.mu.Lock() + defer h.mu.Unlock() + h.targetBySha1[blob.sha1] = cloneHarnessBlob(blob) + return blob +} + +func (h *transferHTTPHarness) targetBlob(relativePath string) *harnessBlob { + h.mu.Lock() + defer h.mu.Unlock() + return cloneHarnessBlob(h.targetStore[relativePath]) +} + +func (h *transferHTTPHarness) stagedTargetContent(relativePath string) []byte { + h.mu.Lock() + defer h.mu.Unlock() + return append([]byte(nil), h.targetStaged[relativePath]...) +} + +type closeTrackingTransferSource struct { + fileTransferSource + onClose func() + maxReadLen *atomic.Int64 +} + +func (s *closeTrackingTransferSource) GetFileReader(ctx context.Context, file api.FileRepresentation) (io.ReadCloser, error) { + reader, err := s.fileTransferSource.GetFileReader(ctx, file) + if err != nil { + return nil, err + } + if s.maxReadLen != nil { + reader = &readSizeTrackingReadCloser{ReadCloser: reader, maxReadLen: s.maxReadLen} + } + return &closeTrackingReadCloser{ + ReadCloser: reader, + onClose: s.onClose, + }, nil +} + +// readSizeTrackingReadCloser records the largest single Read() buffer requested by the caller. +// Unlike a heap-growth sample, this is a deterministic proof that the source is consumed in +// bounded chunks (<= defaultStreamBufferSize) rather than read whole into memory at once. +type readSizeTrackingReadCloser struct { + io.ReadCloser + maxReadLen *atomic.Int64 +} + +func (r *readSizeTrackingReadCloser) Read(p []byte) (int, error) { + requested := int64(len(p)) + for { + old := r.maxReadLen.Load() + if requested <= old || r.maxReadLen.CompareAndSwap(old, requested) { + break + } + } + return r.ReadCloser.Read(p) +} + +func (h *transferHTTPHarness) newFileTransfer(t *testing.T, options ...FileTransferOptions) *FileTransfer { + t.Helper() + opts := FileTransferOptions{TargetDeployOptions: defaultTargetDeployOptions()} + if len(options) > 0 { + opts = options[0] + } + sourceClient, err := NewSourceClient(context.Background(), newTestSourceServerDetails(h.sourceServer.URL)) + require.NoError(t, err) + source := &closeTrackingTransferSource{ + fileTransferSource: sourceClient, + onClose: func() { + h.sourceReaderClosed.Store(true) + h.sourceReaderCloseOnce.Do(func() { close(h.sourceReaderClosedCh) }) + }, + maxReadLen: &h.sourceMaxReadLen, + } + targetClient, err := NewTargetClient(context.Background(), newTestTargetServerDetails(h.targetServer.URL), nil) + require.NoError(t, err) + return NewFileTransfer(source, targetClient, opts) +} + +func (h *transferHTTPHarness) serveSource(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.NotFound(w, r) + return + } + if strings.HasPrefix(r.URL.Path, "/api/storage/") { + h.serveSourceStorage(w, r) + return + } + relativePath := strings.TrimPrefix(r.URL.Path, "/") + h.mu.Lock() + blob, ok := h.sourceStore[relativePath] + if ok { + h.sourceDownloadGETs++ + } + h.mu.Unlock() + if !ok { + http.NotFound(w, r) + return + } + w.WriteHeader(http.StatusOK) + if blob.generated { + _, _ = io.Copy(w, io.LimitReader(repeatingByteReader{b: blob.generatedByte}, blob.size)) + return + } + _, _ = w.Write(blob.body) +} + +func (h *transferHTTPHarness) serveSourceStorage(w http.ResponseWriter, r *http.Request) { + relativePath := strings.TrimPrefix(r.URL.Path, "/api/storage/") + h.mu.Lock() + blob, ok := h.sourceStore[relativePath] + h.mu.Unlock() + if !ok { + http.NotFound(w, r) + return + } + switch r.URL.RawQuery { + case "properties": + props := blob.properties + if props == nil { + props = map[string][]string{} + } + w.WriteHeader(http.StatusOK) + writeTestJSON(w, map[string]any{"properties": props}) + case "stats": + w.WriteHeader(http.StatusOK) + writeTestJSON(w, map[string]any{ + "downloadCount": 0, + "lastDownloaded": 0, + "lastDownloadedBy": "", + }) + case "": + w.WriteHeader(http.StatusOK) + writeTestJSON(w, map[string]any{ + "repo": testSourceRepo, + "path": testSourcePath + "/" + testSourceName, + "created": "2020-01-01T00:00:00.000Z", + "createdBy": "admin", + "lastModified": "2020-01-02T00:00:00.000Z", + "modifiedBy": "deployer", + "size": strconv.FormatInt(blob.size, 10), + "checksums": map[string]string{ + "sha1": blob.sha1, + "sha256": blob.sha256, + "md5": blob.md5, + }, + }) + default: + http.NotFound(w, r) + } +} + +func (h *transferHTTPHarness) serveTarget(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/system/ping" { + _, _ = w.Write([]byte("OK")) + return + } + if r.Method == http.MethodPatch && strings.HasPrefix(r.URL.Path, "/api/metadata/") { + h.serveTargetPropertiesPatch(w, r) + return + } + if r.Method != http.MethodPut { + http.NotFound(w, r) + return + } + itemPath := strings.TrimPrefix(stripMatrixParams(r.URL.Path), "/") + if strings.HasSuffix(itemPath, itemStatisticsSuffix) { + w.WriteHeader(http.StatusCreated) + return + } + matrixProps := parseMatrixProperties(r.URL.Path) + if r.Header.Get("X-Checksum-Deploy") == "true" { + h.serveChecksumDeploy(w, r, itemPath, matrixProps) + return + } + h.serveFullPut(w, r, itemPath, matrixProps) +} + +func (h *transferHTTPHarness) serveChecksumDeploy(w http.ResponseWriter, r *http.Request, itemPath string, matrixProps map[string][]string) { + body, _ := io.ReadAll(r.Body) + sha1Header := r.Header.Get("X-Checksum-Sha1") + h.mu.Lock() + h.checksumPUTs++ + h.checksumPutBodyBytes += int64(len(body)) + existing, ok := h.targetBySha1[sha1Header] + if ok { + stored := cloneHarnessBlob(existing) + stored.properties = mergeProperties(stored.properties, matrixProps) + h.targetStore[itemPath] = stored + } + h.mu.Unlock() + if !ok { + w.WriteHeader(http.StatusNotFound) + return + } + w.WriteHeader(http.StatusOK) +} + +func (h *transferHTTPHarness) serveFullPut(w http.ResponseWriter, r *http.Request, itemPath string, matrixProps map[string][]string) { + if h.fullPutDone != nil { + defer h.fullPutDoneOnce.Do(func() { close(h.fullPutDone) }) + } + h.mu.Lock() + h.fullPutAttempts++ + attempt := h.fullPutAttempts + failAfter := h.failFirstFullPutAfter + afterBytes := h.afterFullPutBytes + discardBody := h.discardFullPutBody + h.mu.Unlock() + + var buf bytes.Buffer + sha1Hash := sha1.New() // #nosec G401 -- Artifactory checksum type, not used for security + sha256Hash := sha256.New() + md5Hash := md5.New() // #nosec G401 -- Artifactory checksum type, not used for security + var received int64 + defer func() { + h.mu.Lock() + delete(h.targetStaged, itemPath) + h.mu.Unlock() + }() + chunk := make([]byte, 32*1024) + for { + n, err := r.Body.Read(chunk) + if n > 0 { + received += int64(n) + if discardBody { + _, _ = sha1Hash.Write(chunk[:n]) + _, _ = sha256Hash.Write(chunk[:n]) + _, _ = md5Hash.Write(chunk[:n]) + } else { + _, _ = buf.Write(chunk[:n]) + h.mu.Lock() + h.targetStaged[itemPath] = append([]byte(nil), buf.Bytes()...) + h.mu.Unlock() + } + if afterBytes != nil { + afterBytes(int(received)) + } + if h.blockFullPutAfter > 0 && received >= int64(h.blockFullPutAfter) { + h.blockFullPutOnce.Do(func() { close(h.fullPutBlocked) }) + <-h.releaseFullPut + } + if failAfter > 0 && attempt == 1 && received >= int64(failAfter) { + if hj, ok := w.(http.Hijacker); ok { + conn, _, hijackErr := hj.Hijack() + if hijackErr == nil { + _ = conn.Close() + } + } + return + } + } + if err == io.EOF { + break + } + if err != nil { + return + } + } + + if contentLength := r.Header.Get("Content-Length"); contentLength != "" { + want, convErr := strconv.Atoi(contentLength) + if convErr == nil && int64(want) != received { + w.WriteHeader(http.StatusBadRequest) + return + } + } + + var blob *harnessBlob + if discardBody { + blob = &harnessBlob{ + sha1: hex.EncodeToString(sha1Hash.Sum(nil)), + sha256: hex.EncodeToString(sha256Hash.Sum(nil)), + md5: hex.EncodeToString(md5Hash.Sum(nil)), + size: received, + properties: cloneProperties(matrixProps), + } + } else { + blob = newHarnessBlob(buf.Bytes(), matrixProps) + } + + h.mu.Lock() + h.fullPUTs++ + h.targetStore[itemPath] = blob + h.targetBySha1[blob.sha1] = cloneHarnessBlob(blob) + h.mu.Unlock() + w.WriteHeader(http.StatusCreated) +} + +func (h *transferHTTPHarness) serveTargetPropertiesPatch(w http.ResponseWriter, r *http.Request) { + itemPath := strings.TrimPrefix(r.URL.Path, "/api/metadata/") + var payload updateItemPropertiesBody + if err := json.NewDecoder(r.Body).Decode(&payload); err != nil { + w.WriteHeader(http.StatusBadRequest) + return + } + h.mu.Lock() + defer h.mu.Unlock() + existing := h.targetStore[itemPath] + if existing == nil { + existing = &harnessBlob{properties: map[string][]string{}} + h.targetStore[itemPath] = existing + } + existing.properties = mergeProperties(existing.properties, payload.Props) + w.WriteHeader(http.StatusNoContent) +} + +func newHarnessBlob(body []byte, properties map[string][]string) *harnessBlob { + sha1Sum := sha1.Sum(body) // #nosec G401 -- Artifactory checksum type, not used for security + sha256Sum := sha256.Sum256(body) + md5Sum := md5.Sum(body) // #nosec G401 -- Artifactory checksum type, not used for security + return &harnessBlob{ + sha1: hex.EncodeToString(sha1Sum[:]), + sha256: hex.EncodeToString(sha256Sum[:]), + md5: hex.EncodeToString(md5Sum[:]), + size: int64(len(body)), + body: append([]byte(nil), body...), + properties: cloneProperties(properties), + } +} + +func cloneHarnessBlob(blob *harnessBlob) *harnessBlob { + if blob == nil { + return nil + } + return &harnessBlob{ + sha1: blob.sha1, + sha256: blob.sha256, + md5: blob.md5, + size: blob.size, + body: append([]byte(nil), blob.body...), + generatedByte: blob.generatedByte, + generated: blob.generated, + properties: cloneProperties(blob.properties), + } +} + +type repeatingByteReader struct { + b byte +} + +func (r repeatingByteReader) Read(p []byte) (int, error) { + for i := range p { + p[i] = r.b + } + return len(p), nil +} + +func checksumsForRepeatingBytes(size int64, b byte) (sha1Sum, sha256Sum, md5Sum string) { + sha1Hash := sha1.New() // #nosec G401 -- Artifactory checksum type, not used for security + sha256Hash := sha256.New() + md5Hash := md5.New() // #nosec G401 -- Artifactory checksum type, not used for security + _, _ = io.Copy(io.MultiWriter(sha1Hash, sha256Hash, md5Hash), io.LimitReader(repeatingByteReader{b: b}, size)) + return hex.EncodeToString(sha1Hash.Sum(nil)), hex.EncodeToString(sha256Hash.Sum(nil)), hex.EncodeToString(md5Hash.Sum(nil)) +} + +func cloneProperties(properties map[string][]string) map[string][]string { + if properties == nil { + return nil + } + cloned := maps.Clone(properties) + for key, values := range cloned { + cloned[key] = append([]string(nil), values...) + } + return cloned +} + +func mergeProperties(existing, extra map[string][]string) map[string][]string { + merged := cloneProperties(existing) + if merged == nil { + merged = map[string][]string{} + } + for key, values := range extra { + merged[key] = append([]string(nil), values...) + } + return merged +} + +func stripMatrixParams(path string) string { + if i := strings.Index(path, ";"); i >= 0 { + return path[:i] + } + return path +} + +func parseMatrixProperties(path string) map[string][]string { + props := map[string][]string{} + semicolon := strings.Index(path, ";") + if semicolon < 0 { + return props + } + for _, segment := range strings.Split(path[semicolon+1:], ";") { + if segment == "" { + continue + } + key, value, ok := strings.Cut(segment, "=") + if !ok { + continue + } + decodedKey, err := url.QueryUnescape(key) + if err != nil { + decodedKey = key + } + decodedValue, err := url.QueryUnescape(value) + if err != nil { + decodedValue = value + } + props[decodedKey] = append(props[decodedKey], decodedValue) + } + return props +} + +func TestFileTransferHTTP_largeStream_matchesSourceSha1AndSize(t *testing.T) { + h := newTransferHTTPHarness(t) + payload := bytes.Repeat([]byte("a"), defaultStreamBufferSize*4) + sourceBlob := h.seedSource(testSourceRelativePath(), payload, nil) + + ft := h.newFileTransfer(t) + candidate := testFileCandidate() + candidate.Size = sourceBlob.size + result := ft.TransferFile(context.Background(), candidate) + + require.NoError(t, result.Err) + assert.Equal(t, api.Success, result.Status) + stored := h.targetBlob(testSourceRelativePath()) + require.NotNil(t, stored) + assert.Equal(t, sourceBlob.sha1, stored.sha1) + assert.Equal(t, sourceBlob.size, stored.size) + assert.Equal(t, payload, stored.body) + h.mu.Lock() + downloadGETs := h.sourceDownloadGETs + fullPUTs := h.fullPUTs + h.mu.Unlock() + assert.GreaterOrEqual(t, downloadGETs, 1) + assert.GreaterOrEqual(t, fullPUTs, 1) +} + +func TestFileTransferHTTP_largeStream_peakHeapStaysBelowFullBuffering(t *testing.T) { + h := newTransferHTTPHarness(t) + h.discardFullPutBody = true + const payloadSize int64 = 8 << 20 + sourceBlob := h.seedGeneratedSource(testSourceRelativePath(), payloadSize, 'm') + + var peak atomic.Uint64 + stop := make(chan struct{}) + var sampler sync.WaitGroup + sampler.Add(1) + go func() { + defer sampler.Done() + var ms runtime.MemStats + for { + runtime.ReadMemStats(&ms) + for { + old := peak.Load() + if ms.HeapAlloc <= old || peak.CompareAndSwap(old, ms.HeapAlloc) { + break + } + } + select { + case <-stop: + return + case <-time.After(2 * time.Millisecond): + } + } + }() + + runtime.GC() + var before runtime.MemStats + runtime.ReadMemStats(&before) + peak.Store(before.HeapAlloc) + + ft := h.newFileTransfer(t) + candidate := testFileCandidate() + candidate.Size = payloadSize + result := ft.TransferFile(context.Background(), candidate) + close(stop) + sampler.Wait() + + require.NoError(t, result.Err) + assert.Equal(t, api.Success, result.Status) + stored := h.targetBlob(testSourceRelativePath()) + require.NotNil(t, stored) + assert.Equal(t, sourceBlob.sha1, stored.sha1) + assert.Equal(t, payloadSize, stored.size) + assert.Empty(t, stored.body) + + growth := int64(peak.Load()) - int64(before.HeapAlloc) // #nosec G115 -- test heap sizes fit in int64 + if growth < 0 { + growth = 0 + } + assert.Less(t, growth, payloadSize, "peak heap grew by %d bytes; full-buffering the payload would retain ~%d", growth, payloadSize) + + // Deterministic complement to the heap-growth sample above: heap sampling can be noisy + // (GC timing, unrelated allocations), so also prove directly that the source is never asked + // to fill a buffer larger than the stream's fixed chunk size, i.e. it is never read whole. + assert.LessOrEqual(t, h.sourceMaxReadLen.Load(), int64(defaultStreamBufferSize), + "source Read() was asked to fill a buffer of %d bytes; expected reads bounded by the %d-byte stream buffer", + h.sourceMaxReadLen.Load(), defaultStreamBufferSize) +} + +func TestFileTransferHTTP_midPutFail_retryHasNoPartialAndFreshGet(t *testing.T) { + h := newTransferHTTPHarness(t) + payload := bytes.Repeat([]byte("b"), defaultStreamBufferSize*3) + sourceBlob := h.seedSource(testSourceRelativePath(), payload, nil) + previous := h.seedTarget(testSourceRelativePath(), []byte("previous-complete-blob"), nil) + h.failFirstFullPutAfter = defaultStreamBufferSize + + ft := h.newFileTransfer(t) + candidate := testFileCandidate() + candidate.Size = sourceBlob.size + + failed := ft.TransferFile(context.Background(), candidate) + require.Error(t, failed.Err) + assert.Equal(t, api.Fail, failed.Status) + afterFail := h.targetBlob(testSourceRelativePath()) + require.NotNil(t, afterFail) + assert.Equal(t, previous.body, afterFail.body) + assert.Equal(t, previous.sha1, afterFail.sha1) + assert.True(t, h.sourceReaderClosed.Load(), "failed attempt must close its source response body") + assert.Empty(t, h.stagedTargetContent(testSourceRelativePath()), "failed attempt must remove staged target bytes") + + h.sourceReaderClosed.Store(false) + retried := ft.TransferFile(context.Background(), candidate) + require.NoError(t, retried.Err) + assert.Equal(t, api.Success, retried.Status) + stored := h.targetBlob(testSourceRelativePath()) + require.NotNil(t, stored) + assert.Equal(t, sourceBlob.sha1, stored.sha1) + assert.Equal(t, sourceBlob.size, stored.size) + assert.Equal(t, payload, stored.body) + h.mu.Lock() + downloadGETs := h.sourceDownloadGETs + h.mu.Unlock() + assert.GreaterOrEqual(t, downloadGETs, 2) +} + +func TestFileTransferHTTP_checksumHit_streamsZeroBytes(t *testing.T) { + h := newTransferHTTPHarness(t) + payload := []byte("hello world") + sourceBlob := h.seedSource(testSourceRelativePath(), payload, map[string][]string{ + "build.name": {"app"}, + }) + h.seedTargetBySha1(payload) + + ft := h.newFileTransfer(t) + result := ft.TransferFile(context.Background(), testFileCandidate()) + + require.NoError(t, result.Err) + assert.Equal(t, api.Success, result.Status) + assert.True(t, result.ChecksumDeployed) + assert.Zero(t, result.BytesTransferred) + h.mu.Lock() + downloadGETs := h.sourceDownloadGETs + checksumPUTs := h.checksumPUTs + checksumBodyBytes := h.checksumPutBodyBytes + fullPUTs := h.fullPUTs + h.mu.Unlock() + assert.Zero(t, downloadGETs) + assert.GreaterOrEqual(t, checksumPUTs, 1) + assert.Zero(t, checksumBodyBytes) + assert.Zero(t, fullPUTs) + stored := h.targetBlob(testSourceRelativePath()) + require.NotNil(t, stored) + assert.Equal(t, sourceBlob.sha1, stored.sha1) +} + +func TestFileTransferHTTP_interrupt_leavesNoStagedContent(t *testing.T) { + h := newTransferHTTPHarness(t) + payload := bytes.Repeat([]byte("c"), defaultStreamBufferSize*256) + sourceBlob := h.seedSource(testSourceRelativePath(), payload, nil) + h.blockFullPutAfter = defaultStreamBufferSize + h.fullPutBlocked = make(chan struct{}) + h.releaseFullPut = make(chan struct{}) + h.fullPutDone = make(chan struct{}) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + ft := h.newFileTransfer(t) + candidate := testFileCandidate() + candidate.Size = sourceBlob.size + resultCh := make(chan TransferResult, 1) + go func() { + resultCh <- ft.TransferFile(ctx, candidate) + }() + + var midStreamStaged []byte + select { + case <-h.fullPutBlocked: + // Captured before cancel(), while the target handler is parked mid-write: this proves + // a genuine partial write reached the target (not zero bytes, not the full payload), + // i.e. the interruption is exercised mid-stream rather than before or after any data flowed. + midStreamStaged = h.stagedTargetContent(testSourceRelativePath()) + cancel() + case <-time.After(time.Second): + close(h.releaseFullPut) + t.Fatal("target PUT did not reach the synchronized interruption point") + } + assert.NotEmpty(t, midStreamStaged, "target handler should have staged partial bytes before the interruption point") + assert.Less(t, int64(len(midStreamStaged)), sourceBlob.size, "target handler should not have received the full payload before the interruption point") + + select { + case <-h.sourceReaderClosedCh: + case <-time.After(time.Second): + close(h.releaseFullPut) + t.Fatal("cancellation did not close the source response") + } + close(h.releaseFullPut) + + var result TransferResult + select { + case result = <-resultCh: + case <-time.After(time.Second): + t.Fatal("interrupted transfer did not return") + } + select { + case <-h.fullPutDone: + case <-time.After(time.Second): + t.Fatal("target handler did not discard staged content") + } + + require.Error(t, result.Err) + assert.Equal(t, api.Fail, result.Status) + assert.True(t, h.sourceReaderClosed.Load()) + assert.Nil(t, h.targetBlob(testSourceRelativePath())) + // stagedTargetContent is not re-checked here: serveFullPut unconditionally deletes it via + // defer on handler return, success or failure alike, so an empty read at this point would be + // guaranteed by the harness regardless of production behavior. The meaningful assertion - that + // real partial content existed before the interruption - is made above, while still in-flight. +} + +func TestFileTransferHTTP_propertiesLandOnTarget(t *testing.T) { + h := newTransferHTTPHarness(t) + payload := []byte("hello world") + h.seedSource(testSourceRelativePath(), payload, map[string][]string{ + "build.name": {"app"}, + "env": {"prod"}, + "npm.name": {"generated-should-drop"}, + }) + + ft := h.newFileTransfer(t, FileTransferOptions{TargetDeployOptions: TargetDeployOptions{ + MinChecksumDeploySize: 10, + PackageType: "npm", + }}) + result := ft.TransferFile(context.Background(), testFileCandidate()) + + require.NoError(t, result.Err) + assert.Equal(t, api.Success, result.Status) + stored := h.targetBlob(testSourceRelativePath()) + require.NotNil(t, stored) + assert.Equal(t, map[string][]string{ + "build.name": {"app"}, + "env": {"prod"}, + }, stored.properties) + assert.NotContains(t, stored.properties, "npm.name") +} + +// writeTestJSON encodes v to w, reporting an encoding failure to the client instead of ignoring it. +func writeTestJSON(w http.ResponseWriter, v map[string]any) { + if err := json.NewEncoder(w).Encode(v); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } +} diff --git a/artifactory/commands/transferfiles/filetransfer_test.go b/artifactory/commands/transferfiles/filetransfer_test.go new file mode 100644 index 000000000..21fc603b8 --- /dev/null +++ b/artifactory/commands/transferfiles/filetransfer_test.go @@ -0,0 +1,649 @@ +package transferfiles + +import ( + "context" + "errors" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/api" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type mockTransferSource struct { + metadata *SourceFileMetadata + metadataErr error + reader io.ReadCloser + readerFactory func() io.ReadCloser + readerErr error + getReaderCalls int +} + +func (m *mockTransferSource) GetFileMetadata(_ context.Context, _ api.FileRepresentation) (*SourceFileMetadata, error) { + if m.metadataErr != nil { + return nil, m.metadataErr + } + return m.metadata, nil +} + +func (m *mockTransferSource) GetFileReader(_ context.Context, _ api.FileRepresentation) (io.ReadCloser, error) { + m.getReaderCalls++ + if m.readerErr != nil { + return nil, m.readerErr + } + if m.readerFactory != nil { + return m.readerFactory(), nil + } + return m.reader, nil +} + +type mockTransferTarget struct { + checksumOutcome ChecksumDeployOutcome + checksumErr error + checksumCalls int + putErr error + putFailAfter int64 + putCalls int + putContent []byte + propsSkipped bool + propsErr error + propsCalls int + propsOptions []TargetDeployOptions + onApplyProps func() + statsErr error + statsCalls int + statsOptions []TargetDeployOptions + folderErr error + folderCalls int + releaseCalls int +} + +func (m *mockTransferTarget) TryChecksumDeploy(_ context.Context, _ *SourceFileMetadata, _ TargetDeployOptions) (ChecksumDeployOutcome, error) { + m.checksumCalls++ + if m.checksumErr != nil { + return ChecksumDeployMiss, m.checksumErr + } + return m.checksumOutcome, nil +} + +func (m *mockTransferTarget) Put(_ context.Context, _ *SourceFileMetadata, reader io.Reader, _ TargetDeployOptions) error { + m.putCalls++ + if m.putErr != nil { + if m.putFailAfter > 0 { + _, _ = io.CopyN(io.Discard, reader, m.putFailAfter) + } + return m.putErr + } + content, err := io.ReadAll(reader) + if err == nil { + m.putContent = content + } + return err +} + +func (m *mockTransferTarget) CreateFolder(_ context.Context, _ *SourceFileMetadata, _ TargetDeployOptions) error { + m.folderCalls++ + return m.folderErr +} + +func (m *mockTransferTarget) ApplyProperties(_ context.Context, _ *SourceFileMetadata, options TargetDeployOptions) (bool, error) { + m.propsCalls++ + m.propsOptions = append(m.propsOptions, options) + if m.onApplyProps != nil { + m.onApplyProps() + } + return m.propsSkipped, m.propsErr +} + +func (m *mockTransferTarget) ApplyStatistics(_ context.Context, _ *SourceFileMetadata, options TargetDeployOptions) error { + m.statsCalls++ + m.statsOptions = append(m.statsOptions, options) + return m.statsErr +} + +func (m *mockTransferTarget) ReleaseEligibleProperties(_ *SourceFileMetadata, _ TargetDeployOptions) { + m.releaseCalls++ +} + +func (m *mockTransferTarget) HasEligibleProperties(metadata *SourceFileMetadata, options TargetDeployOptions) bool { + eligible, _ := filterEligibleProperties(metadata.Properties, options) + return eligible.KeysLen() > 0 +} + +func testFileCandidate() api.FileRepresentation { + return api.FileRepresentation{ + Repo: testSourceRepo, + Path: testSourcePath, + Name: testSourceName, + Size: 11, + } +} + +func testFileMetadata() *SourceFileMetadata { + return &SourceFileMetadata{ + Repo: testSourceRepo, + Path: testSourcePath, + Name: testSourceName, + Size: 11, + Sha1: "sha1-value", + Sha256: "sha256-value", + Md5: "md5-value", + Created: "2020-01-01T00:00:00.000Z", + CreatedBy: "admin", + LastModified: "2020-01-02T00:00:00.000Z", + ModifiedBy: "deployer", + Properties: map[string][]string{ + "build.name": {"app"}, + }, + } +} + +func TestFileTransfer_checksumHit_skipsGet(t *testing.T) { + source := &mockTransferSource{metadata: testFileMetadata()} + target := &mockTransferTarget{checksumOutcome: ChecksumDeployHit} + + ft := NewFileTransfer(source, target, FileTransferOptions{TargetDeployOptions: defaultTargetDeployOptions()}) + result := ft.TransferFile(context.Background(), testFileCandidate()) + + require.NoError(t, result.Err) + assert.Equal(t, api.Success, result.Status) + assert.True(t, result.ChecksumDeployed) + assert.Zero(t, result.BytesTransferred) + assert.Equal(t, int64(11), result.FileSize) + assert.GreaterOrEqual(t, result.DurationMillis, int64(0)) + assert.Equal(t, 1, target.checksumCalls) + assert.Zero(t, source.getReaderCalls) + assert.Equal(t, 1, target.propsCalls) + assert.Equal(t, 1, target.statsCalls) + + status := result.ToFileUploadStatus() + assert.Equal(t, int64(11), result.FileSize) + assert.Equal(t, int64(11), status.SizeBytes) +} + +func TestFileTransfer_checksumMiss_streamsThenPropertiesAndStats(t *testing.T) { + payload := "hello world" + source := &mockTransferSource{ + metadata: testFileMetadata(), + reader: io.NopCloser(strings.NewReader(payload)), + } + target := &mockTransferTarget{checksumOutcome: ChecksumDeployMiss} + + ft := NewFileTransfer(source, target, FileTransferOptions{TargetDeployOptions: defaultTargetDeployOptions()}) + result := ft.TransferFile(context.Background(), testFileCandidate()) + + require.NoError(t, result.Err) + assert.Equal(t, api.Success, result.Status) + assert.False(t, result.ChecksumDeployed) + assert.Equal(t, int64(len(payload)), result.BytesTransferred) + assert.Equal(t, 1, target.checksumCalls) + assert.Equal(t, 1, source.getReaderCalls) + assert.Equal(t, 1, target.putCalls) + assert.Equal(t, 1, target.propsCalls) + assert.Equal(t, 1, target.statsCalls) +} + +func TestFileTransfer_checksumBeforeGet(t *testing.T) { + order := make([]string, 0, 2) + source := &orderTrackingSource{ + mockTransferSource: mockTransferSource{ + metadata: testFileMetadata(), + reader: io.NopCloser(strings.NewReader("hello world")), + }, + onGetReader: func() { order = append(order, "get") }, + } + target := &orderTrackingTarget{ + mockTransferTarget: mockTransferTarget{checksumOutcome: ChecksumDeployMiss}, + onChecksum: func() { order = append(order, "checksum") }, + } + + ft := NewFileTransfer(source, target, FileTransferOptions{TargetDeployOptions: defaultTargetDeployOptions()}) + _ = ft.TransferFile(context.Background(), testFileCandidate()) + assert.Equal(t, []string{"checksum", "get"}, order) +} + +func TestFileTransfer_source404_isSkippedWithZeroSize(t *testing.T) { + source := &mockTransferSource{metadataErr: ErrSourceItemGone} + target := &mockTransferTarget{} + + ft := NewFileTransfer(source, target, FileTransferOptions{TargetDeployOptions: defaultTargetDeployOptions()}) + result := ft.TransferFile(context.Background(), testFileCandidate()) + + require.NoError(t, result.Err) + assert.Equal(t, api.SkippedSourceItemGone, result.Status) + assert.True(t, result.SourceItemGone) + assert.Zero(t, result.FileSize) + assert.Zero(t, result.BytesTransferred) + assert.Zero(t, target.checksumCalls) + assert.Zero(t, source.getReaderCalls) +} + +// TestFileTransfer_sourceItemGone_isCountedForSummary verifies that every gone-skip is counted so +// the end-of-run log line can report how many items were skipped as deleted. +func TestFileTransfer_sourceItemGone_isCountedForSummary(t *testing.T) { + takeSourceItemGoneSkipCount() // reset + ft := NewFileTransfer(&mockTransferSource{metadataErr: ErrSourceItemGone}, &mockTransferTarget{}, FileTransferOptions{TargetDeployOptions: defaultTargetDeployOptions()}) + _ = ft.TransferFile(context.Background(), testFileCandidate()) + + ft = NewFileTransfer(&mockTransferSource{metadata: testFileMetadata(), readerErr: ErrSourceItemGone}, &mockTransferTarget{checksumOutcome: ChecksumDeployMiss}, FileTransferOptions{TargetDeployOptions: defaultTargetDeployOptions()}) + _ = ft.TransferFile(context.Background(), testFileCandidate()) + + assert.Equal(t, int64(2), takeSourceItemGoneSkipCount()) + assert.Zero(t, takeSourceItemGoneSkipCount(), "taking the count resets it") +} + +func TestTransferResult_ToFileUploadStatus_sourceItemGoneHasReason(t *testing.T) { + result := TransferResult{Candidate: testFileCandidate(), Status: api.SkippedSourceItemGone, SourceItemGone: true} + status := result.ToFileUploadStatus() + assert.Equal(t, api.SkippedSourceItemGone, status.Status) + assert.Equal(t, ErrSourceItemGone.Error(), status.Reason) +} + +func TestFileTransfer_contentGet404_isSkippedWithZeroSize(t *testing.T) { + source := &mockTransferSource{ + metadata: testFileMetadata(), + readerErr: ErrSourceItemGone, + } + target := &mockTransferTarget{checksumOutcome: ChecksumDeployMiss} + + ft := NewFileTransfer(source, target, FileTransferOptions{TargetDeployOptions: defaultTargetDeployOptions()}) + result := ft.TransferFile(context.Background(), testFileCandidate()) + + require.NoError(t, result.Err) + assert.Equal(t, api.SkippedSourceItemGone, result.Status) + assert.True(t, result.SourceItemGone) + assert.Zero(t, result.FileSize) + assert.Zero(t, result.BytesTransferred) + assert.Equal(t, 1, source.getReaderCalls) + assert.Zero(t, target.putCalls) +} + +func TestFileTransfer_putFailure_releasesEligibleProperties(t *testing.T) { + putErr := errors.New("target put failed") + source := &mockTransferSource{ + metadata: testFileMetadata(), + reader: io.NopCloser(strings.NewReader("hello world")), + } + target := &mockTransferTarget{checksumOutcome: ChecksumDeployMiss, putErr: putErr, putFailAfter: 3} + + ft := NewFileTransfer(source, target, FileTransferOptions{TargetDeployOptions: defaultTargetDeployOptions()}) + result := ft.TransferFile(context.Background(), testFileCandidate()) + + require.ErrorIs(t, result.Err, putErr) + assert.Equal(t, api.Fail, result.Status) + assert.Zero(t, target.propsCalls) + assert.Equal(t, 1, target.releaseCalls) +} + +func TestFileTransfer_checksumDeployError_releasesEligibleProperties(t *testing.T) { + checksumErr := errors.New("checksum deploy failed") + source := &mockTransferSource{metadata: testFileMetadata()} + target := &mockTransferTarget{checksumErr: checksumErr} + + ft := NewFileTransfer(source, target, FileTransferOptions{TargetDeployOptions: defaultTargetDeployOptions()}) + result := ft.TransferFile(context.Background(), testFileCandidate()) + + require.ErrorIs(t, result.Err, checksumErr) + assert.Equal(t, api.Fail, result.Status) + assert.Zero(t, source.getReaderCalls) + assert.Zero(t, target.propsCalls) + assert.Equal(t, 1, target.releaseCalls) +} + +func TestFileTransfer_createFolderError_releasesEligibleProperties(t *testing.T) { + folderErr := errors.New("create folder failed") + source := &mockTransferSource{metadata: testFolderMetadata()} + target := &mockTransferTarget{folderErr: folderErr} + + ft := NewFileTransfer(source, target, FileTransferOptions{TargetDeployOptions: defaultTargetDeployOptions()}) + result := ft.TransferFile(context.Background(), testFolderCandidate(false)) + + require.ErrorIs(t, result.Err, folderErr) + assert.Equal(t, api.Fail, result.Status) + assert.Zero(t, target.propsCalls) + assert.Equal(t, 1, target.releaseCalls) +} + +func TestFileTransfer_contentGet404_releasesTargetEligibleCache(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"errors":[{"message":"not found"}]}`)) + })) + defer server.Close() + + target, err := NewTargetClient(context.Background(), newTestTargetServerDetails(server.URL), nil) + require.NoError(t, err) + + metadata := testFileMetadata() + source := &mockTransferSource{metadata: metadata, readerErr: ErrSourceItemGone} + options := defaultTargetDeployOptions() + ft := NewFileTransfer(source, target, FileTransferOptions{TargetDeployOptions: options}) + result := ft.TransferFile(context.Background(), testFileCandidate()) + + require.NoError(t, result.Err) + assert.Equal(t, api.SkippedSourceItemGone, result.Status) + assert.False(t, eligibleCacheHas(target, metadata, options), + "TransferFile must release eligibleCache when content GET 404 skips ApplyProperties") +} + +func TestFileTransfer_putFailure_isRetryable(t *testing.T) { + putErr := errors.New("target put failed") + source := &mockTransferSource{ + metadata: testFileMetadata(), + reader: io.NopCloser(strings.NewReader("hello world")), + } + target := &mockTransferTarget{checksumOutcome: ChecksumDeployMiss, putErr: putErr, putFailAfter: 3} + + ft := NewFileTransfer(source, target, FileTransferOptions{TargetDeployOptions: defaultTargetDeployOptions()}) + result := ft.TransferFile(context.Background(), testFileCandidate()) + + require.ErrorIs(t, result.Err, putErr) + assert.Equal(t, api.Fail, result.Status) + assert.Zero(t, result.BytesTransferred) + assert.Empty(t, target.putContent) +} + +func TestFileTransfer_putSuccess_propertiesFailure_keepsTransferredBytes(t *testing.T) { + payload := "hello world" + propsErr := errors.New("apply properties failed") + source := &mockTransferSource{ + metadata: testFileMetadata(), + reader: io.NopCloser(strings.NewReader(payload)), + } + target := &mockTransferTarget{checksumOutcome: ChecksumDeployMiss, propsErr: propsErr} + + ft := NewFileTransfer(source, target, FileTransferOptions{TargetDeployOptions: defaultTargetDeployOptions()}) + result := ft.TransferFile(context.Background(), testFileCandidate()) + + require.ErrorIs(t, result.Err, propsErr) + assert.Equal(t, api.Fail, result.Status) + assert.Equal(t, payload, string(target.putContent)) + assert.Equal(t, 1, target.putCalls) + assert.Equal(t, 1, target.propsCalls) + assert.Zero(t, target.statsCalls) + assert.Equal(t, int64(len(payload)), result.BytesTransferred) +} + +func TestFileTransfer_putSuccess_statisticsFailure_keepsTransferredBytes(t *testing.T) { + payload := "hello world" + statsErr := errors.New("apply statistics failed") + source := &mockTransferSource{ + metadata: testFileMetadata(), + reader: io.NopCloser(strings.NewReader(payload)), + } + target := &mockTransferTarget{checksumOutcome: ChecksumDeployMiss, statsErr: statsErr} + + ft := NewFileTransfer(source, target, FileTransferOptions{TargetDeployOptions: defaultTargetDeployOptions()}) + result := ft.TransferFile(context.Background(), testFileCandidate()) + + require.ErrorIs(t, result.Err, statsErr) + assert.Equal(t, api.Fail, result.Status) + assert.Equal(t, payload, string(target.putContent)) + assert.Equal(t, 1, target.putCalls) + assert.Equal(t, 1, target.propsCalls) + assert.Equal(t, 1, target.statsCalls) + assert.Equal(t, int64(len(payload)), result.BytesTransferred) +} + +func TestFileTransfer_retryOpensFreshGet(t *testing.T) { + payload := "hello world" + var sources []*closeTrackingReadCloser + source := &mockTransferSource{ + metadata: testFileMetadata(), + readerFactory: func() io.ReadCloser { + reader := &closeTrackingReadCloser{ReadCloser: io.NopCloser(strings.NewReader(payload))} + sources = append(sources, reader) + return reader + }, + } + putErr := errors.New("mid-stream PUT failure") + target := &mockTransferTarget{checksumOutcome: ChecksumDeployMiss, putErr: putErr, putFailAfter: 3} + + ft := NewFileTransfer(source, target, FileTransferOptions{TargetDeployOptions: defaultTargetDeployOptions()}) + + result1 := ft.TransferFile(context.Background(), testFileCandidate()) + require.ErrorIs(t, result1.Err, putErr) + assert.Zero(t, result1.BytesTransferred) + assert.Empty(t, target.putContent) + require.Len(t, sources, 1) + assert.True(t, sources[0].closed.Load()) + + target.putErr = nil + result2 := ft.TransferFile(context.Background(), testFileCandidate()) + require.NoError(t, result2.Err) + assert.Equal(t, 2, source.getReaderCalls) + require.Len(t, sources, 2) + assert.NotSame(t, sources[0], sources[1]) + assert.Equal(t, payload, string(target.putContent)) +} + +func TestFileTransfer_configureOptions_isConcurrentSafe(t *testing.T) { + ft := NewFileTransfer(nil, nil, FileTransferOptions{}) + var wg sync.WaitGroup + + for i := 0; i < 10; i++ { + wg.Add(2) + go func(bufferSize int) { + defer wg.Done() + ft.ConfigureOptions(FileTransferOptions{StreamBufferSize: bufferSize}) + }(i + 1) + go func() { + defer wg.Done() + _ = ft.getOptions() + }() + } + + wg.Wait() +} + +func TestFileTransfer_statisticsUsesTransferOptionsSnapshot(t *testing.T) { + initialOptions := FileTransferOptions{ + TargetDeployOptions: TargetDeployOptions{PackageType: "generic"}, + } + updatedOptions := FileTransferOptions{ + TargetDeployOptions: TargetDeployOptions{PackageType: "buildinfo", BuildInfoRepo: true}, + } + source := &mockTransferSource{metadata: testFileMetadata()} + target := &mockTransferTarget{checksumOutcome: ChecksumDeployHit} + ft := NewFileTransfer(source, target, initialOptions) + target.onApplyProps = func() { + ft.ConfigureOptions(updatedOptions) + } + + result := ft.TransferFile(context.Background(), testFileCandidate()) + + require.NoError(t, result.Err) + require.Equal(t, []TargetDeployOptions{initialOptions.TargetDeployOptions}, target.propsOptions) + require.Equal(t, []TargetDeployOptions{initialOptions.TargetDeployOptions}, target.statsOptions) +} + +func TestFileTransfer_canceledContextAfterGetReader_closesReader(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + reader := &closeTrackingReadCloser{ReadCloser: io.NopCloser(strings.NewReader("hello world"))} + source := &mockTransferSource{ + metadata: testFileMetadata(), + readerFactory: func() io.ReadCloser { + cancel() + return reader + }, + } + target := &mockTransferTarget{checksumOutcome: ChecksumDeployMiss} + + ft := NewFileTransfer(source, target, FileTransferOptions{TargetDeployOptions: defaultTargetDeployOptions()}) + result := ft.TransferFile(ctx, testFileCandidate()) + + require.ErrorIs(t, result.Err, context.Canceled) + assert.Equal(t, api.Fail, result.Status) + assert.Zero(t, target.putCalls) + assert.Equal(t, 1, target.releaseCalls) + assert.True(t, reader.closed.Load()) + assert.Equal(t, int32(1), reader.closeCalls.Load()) +} + +func TestFileTransfer_inFlightCancellationReturnsContextError(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + source := &cancelAwareTransferSource{started: make(chan struct{})} + ft := NewFileTransfer(source, &mockTransferTarget{}, FileTransferOptions{}) + result := make(chan TransferResult, 1) + + go func() { + result <- ft.TransferFile(ctx, testFileCandidate()) + }() + <-source.started + cancel() + + select { + case transferResult := <-result: + require.ErrorIs(t, transferResult.Err, context.Canceled) + assert.Equal(t, api.Fail, transferResult.Status) + case <-time.After(time.Second): + t.Fatal("in-flight transfer did not observe context cancellation") + } +} + +func TestFileTransfer_toFileUploadStatus_success(t *testing.T) { + result := TransferResult{ + Candidate: testFileCandidate(), + Status: api.Success, + ChecksumDeployed: true, + BytesTransferred: 0, + FileSize: testFileCandidate().Size, + } + status := result.ToFileUploadStatus() + assert.Equal(t, api.Success, status.Status) + assert.True(t, status.ChecksumDeployed) + assert.Equal(t, testFileCandidate().Repo, status.Repo) + assert.Equal(t, testFileCandidate().Size, status.SizeBytes) +} + +func TestFileTransfer_checksumHit_toFileUploadStatus_usesLogicalFileSize(t *testing.T) { + candidate := api.FileRepresentation{ + Repo: testSourceRepo, + Path: testSourcePath, + Name: testSourceName, + Size: 42, + } + source := &mockTransferSource{metadata: &SourceFileMetadata{ + Repo: testSourceRepo, + Path: testSourcePath, + Name: testSourceName, + Size: 42, + }} + target := &mockTransferTarget{checksumOutcome: ChecksumDeployHit} + + ft := NewFileTransfer(source, target, FileTransferOptions{TargetDeployOptions: defaultTargetDeployOptions()}) + result := ft.TransferFile(context.Background(), candidate) + + require.NoError(t, result.Err) + assert.Zero(t, result.BytesTransferred) + assert.Equal(t, int64(42), result.FileSize) + assert.Equal(t, int64(42), result.ToFileUploadStatus().SizeBytes) +} + +func testFolderCandidate(nonEmpty bool) api.FileRepresentation { + return api.FileRepresentation{ + Repo: testSourceRepo, + Path: "empty-dir", + NonEmptyDir: nonEmpty, + } +} + +func testFolderMetadata() *SourceFileMetadata { + return &SourceFileMetadata{ + Repo: testSourceRepo, + Path: "empty-dir", + Name: "", + Size: 0, + } +} + +func TestFileTransfer_folderNonEmptyDirNoProperties_skipsCreateFolder(t *testing.T) { + source := &mockTransferSource{metadata: testFolderMetadata()} + target := &mockTransferTarget{} + + ft := NewFileTransfer(source, target, FileTransferOptions{TargetDeployOptions: defaultTargetDeployOptions()}) + result := ft.TransferFile(context.Background(), testFolderCandidate(true)) + + require.NoError(t, result.Err) + assert.Equal(t, api.SkippedNonEmptyDir, result.Status) + assert.Zero(t, target.folderCalls) + assert.Zero(t, target.propsCalls) + assert.Zero(t, target.statsCalls) + assert.GreaterOrEqual(t, result.DurationMillis, int64(0)) +} + +func TestFileTransfer_folderNonEmptyDirWithProperties_createsFolder(t *testing.T) { + metadata := testFolderMetadata() + metadata.Properties = map[string][]string{ + "build.name": {"app"}, + } + source := &mockTransferSource{metadata: metadata} + target := &mockTransferTarget{} + + ft := NewFileTransfer(source, target, FileTransferOptions{TargetDeployOptions: defaultTargetDeployOptions()}) + result := ft.TransferFile(context.Background(), testFolderCandidate(true)) + + require.NoError(t, result.Err) + assert.Equal(t, api.Success, result.Status) + assert.Equal(t, 1, target.folderCalls) + assert.Equal(t, 1, target.propsCalls) + assert.Equal(t, 1, target.statsCalls) +} + +func TestFileTransfer_emptyFolder_createsFolder(t *testing.T) { + source := &mockTransferSource{metadata: testFolderMetadata()} + target := &mockTransferTarget{} + + ft := NewFileTransfer(source, target, FileTransferOptions{TargetDeployOptions: defaultTargetDeployOptions()}) + result := ft.TransferFile(context.Background(), testFolderCandidate(false)) + + require.NoError(t, result.Err) + assert.Equal(t, api.Success, result.Status) + assert.Equal(t, 1, target.folderCalls) + assert.Equal(t, 1, target.propsCalls) + assert.Equal(t, 1, target.statsCalls) +} + +type orderTrackingSource struct { + mockTransferSource + onGetReader func() +} + +func (s *orderTrackingSource) GetFileReader(ctx context.Context, file api.FileRepresentation) (io.ReadCloser, error) { + if s.onGetReader != nil { + s.onGetReader() + } + return s.mockTransferSource.GetFileReader(ctx, file) +} + +type orderTrackingTarget struct { + mockTransferTarget + onChecksum func() +} + +type cancelAwareTransferSource struct { + started chan struct{} +} + +func (s *cancelAwareTransferSource) GetFileMetadata(ctx context.Context, _ api.FileRepresentation) (*SourceFileMetadata, error) { + close(s.started) + <-ctx.Done() + return nil, nil +} + +func (s *cancelAwareTransferSource) GetFileReader(_ context.Context, _ api.FileRepresentation) (io.ReadCloser, error) { + return nil, errors.New("unexpected GetFileReader call") +} + +func (t *orderTrackingTarget) TryChecksumDeploy(ctx context.Context, metadata *SourceFileMetadata, options TargetDeployOptions) (ChecksumDeployOutcome, error) { + if t.onChecksum != nil { + t.onChecksum() + } + return t.mockTransferTarget.TryChecksumDeploy(ctx, metadata, options) +} diff --git a/artifactory/commands/transferfiles/fulltransfer.go b/artifactory/commands/transferfiles/fulltransfer.go index 62291fade..cd194f144 100644 --- a/artifactory/commands/transferfiles/fulltransfer.go +++ b/artifactory/commands/transferfiles/fulltransfer.go @@ -11,6 +11,7 @@ import ( "github.com/jfrog/gofrog/parallel" "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/api" "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/state" + coreutils "github.com/jfrog/jfrog-cli-core/v2/artifactory/utils" "github.com/jfrog/jfrog-cli-core/v2/utils/reposnapshot" servicesUtils "github.com/jfrog/jfrog-client-go/artifactory/services/utils" clientUtils "github.com/jfrog/jfrog-client-go/utils" @@ -21,7 +22,7 @@ import ( // Manages the phase of performing a full transfer of the repository. // This phase is only executed once per repository if its completed. // Transfer is performed by treating every folder as a task, and searching for it's content in a flat AQL. -// New folders found are handled as a separate task, and files are uploaded in chunks and polled on for status. +// New folders found are handled as a separate task, and files are transferred directly per file. type fullTransferPhase struct { phaseBase transferManager *transferManager @@ -109,7 +110,7 @@ func (m *fullTransferPhase) run() error { // runWithFolderTraversal uses the traditional folder-by-folder traversal approach. // This is the default behavior when no include patterns are specified. func (m *fullTransferPhase) runWithFolderTraversal() error { - action := func(pcWrapper *producerConsumerWrapper, uploadChunkChan chan UploadedChunk, delayHelper delayUploadHelper, errorsChannelMng *ErrorsChannelMng) error { + action := func(pcWrapper *producerConsumerWrapper, delayHelper delayUploadHelper, errorsChannelMng *ErrorsChannelMng) error { if ShouldStop(&m.phaseBase, &delayHelper, errorsChannelMng) { return nil } @@ -120,7 +121,7 @@ func (m *fullTransferPhase) runWithFolderTraversal() error { return err } - folderHandler := m.createFolderFullTransferHandlerFunc(node, pcWrapper, uploadChunkChan, delayHelper, errorsChannelMng) + folderHandler := m.createFolderFullTransferHandlerFunc(node, pcWrapper, delayHelper, errorsChannelMng) _, err = pcWrapper.chunkBuilderProducerConsumer.AddTaskWithError(folderHandler(folderParams{relativePath: "."}), pcWrapper.errorsQueue.AddError) return err } @@ -140,7 +141,7 @@ func (m *fullTransferPhase) runWithFolderTraversal() error { func (m *fullTransferPhase) runWithAqlPatternFiltering() error { log.Info("Using AQL-based pattern filtering for include patterns:", m.includeFilesPatterns) - action := func(pcWrapper *producerConsumerWrapper, uploadChunkChan chan UploadedChunk, delayHelper delayUploadHelper, errorsChannelMng *ErrorsChannelMng) error { + action := func(pcWrapper *producerConsumerWrapper, delayHelper delayUploadHelper, errorsChannelMng *ErrorsChannelMng) error { if ShouldStop(&m.phaseBase, &delayHelper, errorsChannelMng) { return nil } @@ -160,13 +161,14 @@ func (m *fullTransferPhase) runWithAqlPatternFiltering() error { if len(result) == 0 { if paginationOffset == 0 { log.Info("No files found matching the include patterns") + m.warnIfRepoAppearsBlackedOut("the include-pattern AQL query") } break } // Convert results to file representations and upload files := convertResultsToFileRepresentation(result) - shouldStop, err := uploadByChunks(files, uploadChunkChan, m.phaseBase, delayHelper, errorsChannelMng, pcWrapper) + shouldStop, err := transferFiles(files, m.phaseBase, delayHelper, errorsChannelMng, pcWrapper) if err != nil || shouldStop { return err } @@ -200,28 +202,51 @@ func (m *fullTransferPhase) getPatternMatchingFiles(paginationOffset int) (resul } lastPage = len(aqlResults.Results) < AqlPaginationLimit - result, err = m.locallyGeneratedFilter.FilterLocallyGenerated(aqlResults.Results) + result, err = m.locallyGeneratedFilter.FilterLocallyGenerated(aqlResults.Results, m.packageType) return } +// warnIfRepoAppearsBlackedOut logs a warning when the repository's own summary (fetched once, +// upfront, via GetRepoSummary at repo-transfer start) reports it holds files, but a query that +// should have covered the whole repository (queryDescription) returned zero results. Artifactory's +// AQL endpoint silently filters out items the querying identity can't read, returning an empty +// result set (HTTP 200) rather than an error - which looks identical to a genuinely empty +// repository. This does not fail or retry the transfer (an empty repo, or a repo whose files were +// all legitimately deleted since the summary was fetched, are both real possibilities too); it +// only surfaces the ambiguity so operators can audit a suspicious zero-file result instead of +// silently trusting it. +func (m *fullTransferPhase) warnIfRepoAppearsBlackedOut(queryDescription string) { + filesCount, err := coreutils.GetFilesCountFromRepositorySummary(&m.repoSummary) + if err != nil || filesCount <= 0 { + return + } + log.Warn(fmt.Sprintf( + "Repository '%s' reports %d file(s) in its summary, but %s returned zero results. "+ + "Artifactory's AQL endpoint returns an empty result set (not an error) when the transferring "+ + "identity lacks read permission on a repository or path, which is indistinguishable from a "+ + "genuinely empty repository. If this repository unexpectedly transfers 0 files, verify the "+ + "transferring user's read permissions on '%s' before trusting this run's result.", + m.repoKey, filesCount, queryDescription, m.repoKey)) +} + type folderFullTransferHandlerFunc func(params folderParams) parallel.TaskFunc type folderParams struct { relativePath string } -func (m *fullTransferPhase) createFolderFullTransferHandlerFunc(node *reposnapshot.Node, pcWrapper *producerConsumerWrapper, uploadChunkChan chan UploadedChunk, +func (m *fullTransferPhase) createFolderFullTransferHandlerFunc(node *reposnapshot.Node, pcWrapper *producerConsumerWrapper, delayHelper delayUploadHelper, errorsChannelMng *ErrorsChannelMng) folderFullTransferHandlerFunc { return func(params folderParams) parallel.TaskFunc { return func(threadId int) error { logMsgPrefix := clientUtils.GetLogMsgPrefix(threadId, false) - return m.transferFolder(node, params, logMsgPrefix, pcWrapper, uploadChunkChan, delayHelper, errorsChannelMng) + return m.transferFolder(node, params, logMsgPrefix, pcWrapper, delayHelper, errorsChannelMng) } } } func (m *fullTransferPhase) transferFolder(node *reposnapshot.Node, params folderParams, logMsgPrefix string, pcWrapper *producerConsumerWrapper, - uploadChunkChan chan UploadedChunk, delayHelper delayUploadHelper, errorsChannelMng *ErrorsChannelMng) (err error) { + delayHelper delayUploadHelper, errorsChannelMng *ErrorsChannelMng) (err error) { log.Debug(logMsgPrefix+"Handling folder:", path.Join(m.repoKey, params.relativePath)) // Increment progress number of folders @@ -232,8 +257,7 @@ func (m *fullTransferPhase) transferFolder(node *reposnapshot.Node, params folde return } - curUploadChunk, err := m.searchAndHandleFolderContents(params, pcWrapper, - uploadChunkChan, delayHelper, errorsChannelMng, node) + err = m.searchAndHandleFolderContents(params, pcWrapper, delayHelper, errorsChannelMng, node) if err != nil { return } @@ -243,28 +267,13 @@ func (m *fullTransferPhase) transferFolder(node *reposnapshot.Node, params folde return err } - // Chunk didn't reach full size. Upload the remaining files. - if len(curUploadChunk.UploadCandidates) > 0 { - if _, err = pcWrapper.chunkUploaderProducerConsumer.AddTaskWithError(uploadChunkWhenPossibleHandler(pcWrapper, &m.phaseBase, curUploadChunk, uploadChunkChan, errorsChannelMng), pcWrapper.errorsQueue.AddError); err != nil { - return - } - } log.Debug(logMsgPrefix+"Done transferring folder:", path.Join(m.repoKey, params.relativePath)) return } func (m *fullTransferPhase) searchAndHandleFolderContents(params folderParams, pcWrapper *producerConsumerWrapper, - uploadChunkChan chan UploadedChunk, delayHelper delayUploadHelper, errorsChannelMng *ErrorsChannelMng, - node *reposnapshot.Node) (curUploadChunk api.UploadChunk, err error) { - curUploadChunk = api.UploadChunk{ - TargetAuth: createTargetAuth(m.targetRtDetails, m.proxyKey), - CheckExistenceInFilestore: m.checkExistenceInFilestore, - // Skip file filtering in the Data Transfer plugin if it is already enabled in the JFrog CLI. - // The local generated filter is enabled in the JFrog CLI for target Artifactory servers >= 7.55. - SkipFileFiltering: m.locallyGeneratedFilter.IsEnabled(), - MinCheckSumDeploySize: m.minCheckSumDeploySize, - } - + delayHelper delayUploadHelper, errorsChannelMng *ErrorsChannelMng, + node *reposnapshot.Node) (err error) { var result []servicesUtils.ResultItem var lastPage bool paginationI := 0 @@ -274,13 +283,21 @@ func (m *fullTransferPhase) searchAndHandleFolderContents(params folderParams, p return } - // Add the folder as a candidate to transfer. The reason is that we'd like to transfer only folders with properties or empty folders. - if params.relativePath != "." { - curUploadChunk.AppendUploadCandidateIfNeeded(api.FileRepresentation{Repo: m.repoKey, Path: params.relativePath, NonEmptyDir: len(result) > 0}, m.buildInfoRepo) + // Transfer folders with properties or empty folders once per directory scan. + if params.relativePath != "." && paginationI == 0 { + folder := api.FileRepresentation{Repo: m.repoKey, Path: params.relativePath, NonEmptyDir: len(result) > 0} + var shouldStop bool + shouldStop, err = m.transferFolderCandidateIfNeeded(folder, delayHelper, errorsChannelMng, pcWrapper) + if err != nil || shouldStop { + return err + } } // Empty folder if paginationI == 0 && len(result) == 0 { + if params.relativePath == "." { + m.warnIfRepoAppearsBlackedOut("the repository root folder listing") + } return } @@ -293,12 +310,9 @@ func (m *fullTransferPhase) searchAndHandleFolderContents(params folderParams, p } switch item.Type { case "folder": - err = m.handleFoundChildFolder(params, pcWrapper, - uploadChunkChan, delayHelper, errorsChannelMng, item) + err = m.handleFoundChildFolder(params, pcWrapper, delayHelper, errorsChannelMng, item) case "file": - err = m.handleFoundFile(pcWrapper, - uploadChunkChan, delayHelper, errorsChannelMng, - node, item, &curUploadChunk) + err = m.handleFoundFile(pcWrapper, delayHelper, errorsChannelMng, node, item) } if err != nil { return @@ -310,8 +324,17 @@ func (m *fullTransferPhase) searchAndHandleFolderContents(params folderParams, p return } +func (m *fullTransferPhase) transferFolderCandidateIfNeeded(folder api.FileRepresentation, delayHelper delayUploadHelper, + errorsChannelMng *ErrorsChannelMng, pcWrapper *producerConsumerWrapper) (shouldStop bool, err error) { + if m.buildInfoRepo && folder.Name == "" { + log.Debug(fmt.Sprintf("Skipping unneeded empty dir '%s' in the build-info repository '%s'", folder.Path, folder.Repo)) + return false, nil + } + return transferFiles([]api.FileRepresentation{folder}, m.phaseBase, delayHelper, errorsChannelMng, pcWrapper) +} + func (m *fullTransferPhase) handleFoundChildFolder(params folderParams, pcWrapper *producerConsumerWrapper, - uploadChunkChan chan UploadedChunk, delayHelper delayUploadHelper, errorsChannelMng *ErrorsChannelMng, + delayHelper delayUploadHelper, errorsChannelMng *ErrorsChannelMng, item servicesUtils.ResultItem) (err error) { newRelativePath := getFolderRelativePath(item.Name, params.relativePath) @@ -321,7 +344,7 @@ func (m *fullTransferPhase) handleFoundChildFolder(params folderParams, pcWrappe return err } - folderHandler := m.createFolderFullTransferHandlerFunc(node, pcWrapper, uploadChunkChan, delayHelper, errorsChannelMng) + folderHandler := m.createFolderFullTransferHandlerFunc(node, pcWrapper, delayHelper, errorsChannelMng) _, err = pcWrapper.chunkBuilderProducerConsumer.AddTaskWithError(folderHandler(folderParams{relativePath: newRelativePath}), pcWrapper.errorsQueue.AddError) return } @@ -329,8 +352,8 @@ func (m *fullTransferPhase) handleFoundChildFolder(params folderParams, pcWrappe // Note: Pattern filtering is handled at AQL level when --include-files is provided. // This function is only called during folder traversal (when no patterns are specified). func (m *fullTransferPhase) handleFoundFile(pcWrapper *producerConsumerWrapper, - uploadChunkChan chan UploadedChunk, delayHelper delayUploadHelper, errorsChannelMng *ErrorsChannelMng, - node *reposnapshot.Node, item servicesUtils.ResultItem, curUploadChunk *api.UploadChunk) (err error) { + delayHelper delayUploadHelper, errorsChannelMng *ErrorsChannelMng, + node *reposnapshot.Node, item servicesUtils.ResultItem) (err error) { file := api.FileRepresentation{Repo: item.Repo, Path: item.Path, Name: item.Name, Size: item.Size} delayed, stopped := delayHelper.delayUploadIfNecessary(m.phaseBase, file) if delayed || stopped { @@ -346,15 +369,7 @@ func (m *fullTransferPhase) handleFoundFile(pcWrapper *producerConsumerWrapper, if err != nil { return } - curUploadChunk.AppendUploadCandidateIfNeeded(file, m.buildInfoRepo) - if curUploadChunk.IsChunkFull() { - _, err = pcWrapper.chunkUploaderProducerConsumer.AddTaskWithError(uploadChunkWhenPossibleHandler(pcWrapper, &m.phaseBase, *curUploadChunk, uploadChunkChan, errorsChannelMng), pcWrapper.errorsQueue.AddError) - if err != nil { - return - } - // Empty the uploaded chunk. - curUploadChunk.UploadCandidates = []api.FileRepresentation{} - } + _, err = pcWrapper.chunkUploaderProducerConsumer.AddTaskWithError(transferFileHandler(&m.phaseBase, file, errorsChannelMng), pcWrapper.errorsQueue.AddError) return } @@ -373,7 +388,7 @@ func (m *fullTransferPhase) getDirectoryContentAql(relativePath string, paginati } lastPage = len(aqlResults.Results) < AqlPaginationLimit - result, err = m.locallyGeneratedFilter.FilterLocallyGenerated(aqlResults.Results) + result, err = m.locallyGeneratedFilter.FilterLocallyGenerated(aqlResults.Results, m.packageType) return } diff --git a/artifactory/commands/transferfiles/fulltransfer_test.go b/artifactory/commands/transferfiles/fulltransfer_test.go index e3fbf4ee5..1ba99d4e6 100644 --- a/artifactory/commands/transferfiles/fulltransfer_test.go +++ b/artifactory/commands/transferfiles/fulltransfer_test.go @@ -3,20 +3,95 @@ package transferfiles import ( "context" "encoding/json" + "errors" "net/http" + "net/http/httptest" "strings" "sync" "testing" "time" + "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/api" "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/state" commonTests "github.com/jfrog/jfrog-cli-core/v2/common/tests" + coreConfig "github.com/jfrog/jfrog-cli-core/v2/utils/config" "github.com/jfrog/jfrog-cli-core/v2/utils/tests" servicesUtils "github.com/jfrog/jfrog-client-go/artifactory/services/utils" "github.com/jfrog/jfrog-client-go/utils/log" "github.com/stretchr/testify/assert" ) +type fakeFileTransferExecutor struct { + callCount int + mu sync.Mutex +} + +func (f *fakeFileTransferExecutor) TransferFile(_ context.Context, candidate api.FileRepresentation) TransferResult { + f.mu.Lock() + defer f.mu.Unlock() + f.callCount++ + if candidate.Name == "fail.jar" { + return TransferResult{ + Candidate: candidate, + Status: api.Fail, + Err: errors.New("transfer failed"), + } + } + return TransferResult{Candidate: candidate, Status: api.Success} +} + +func TestFolderTraversal_schedulesFileTransfer(t *testing.T) { + stateManager, cleanUp := state.InitStateTest(t) + defer cleanUp() + + mockAqlResults := servicesUtils.AqlSearchResult{ + Results: []servicesUtils.ResultItem{ + {Repo: "test-repo", Path: ".", Name: "file.jar", Size: 100, Type: "file"}, + }, + } + + testServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.RequestURI == "/api/search/aql" { + w.WriteHeader(http.StatusOK) + response, _ := json.Marshal(mockAqlResults) + _, _ = w.Write(response) + } + })) + defer testServer.Close() + + serverDetails := &coreConfig.ServerDetails{ArtifactoryUrl: testServer.URL + "/"} + + assert.NoError(t, stateManager.SetRepoState("test-repo", 0, 0, false, true)) + node, err := stateManager.LookUpNode(".") + assert.NoError(t, err) + + executor := &fakeFileTransferExecutor{} + pcWrapper := newProducerConsumerWrapper() + errorsChannelMng := createErrorsChannelMng() + + phase := &fullTransferPhase{ + phaseBase: phaseBase{ + context: context.Background(), + stateManager: stateManager, + repoKey: "test-repo", + srcRtDetails: serverDetails, + fileTransfer: executor, + pcDetails: &pcWrapper, + locallyGeneratedFilter: &locallyGeneratedFilter{enabled: false}, + }, + } + + delayedArtifactsChannelMng := createdDelayedArtifactsChannelMng() + delayHelper := delayUploadHelper{delayedArtifactsChannelMng: &delayedArtifactsChannelMng} + + err = phase.transferFolder(node, folderParams{relativePath: "."}, "", &pcWrapper, delayHelper, &errorsChannelMng) + assert.NoError(t, err) + + assert.NoError(t, runProducerConsumers(&pcWrapper)) + + assert.Equal(t, 1, executor.callCount, "folder traversal should schedule FileTransfer.TransferFile per file") +} + // TestGetPatternMatchingFilesWithResults tests getPatternMatchingFiles with files returned func TestGetPatternMatchingFilesWithResults(t *testing.T) { stateManager, cleanUp := state.InitStateTest(t) @@ -192,6 +267,56 @@ func TestRunWithAqlPatternFilteringPagination(t *testing.T) { assert.Equal(t, 2, aqlCallCount, "AQL should be called twice for two pages") } +// TestWarnIfRepoAppearsBlackedOut covers the B-38 fix: an AQL query returning zero results is +// corroborated against the repo's own GetRepoSummary file count (fetched once at repo-transfer +// start) before being trusted at face value, since Artifactory's AQL endpoint silently filters +// out items the querying identity can't read (HTTP 200, empty result set - not an error), +// which looks identical to a genuinely empty repository. +func TestWarnIfRepoAppearsBlackedOut(t *testing.T) { + t.Run("warns when repo summary reports files but the query found none", func(t *testing.T) { + buffer, stderrBuffer, previousLog := tests.RedirectLogOutputToBuffer() + defer log.SetLogger(previousLog) + + phase := &fullTransferPhase{ + phaseBase: phaseBase{ + repoKey: "blacked-out-repo", + repoSummary: servicesUtils.RepositorySummary{FilesCount: json.Number("42")}, + }, + } + phase.warnIfRepoAppearsBlackedOut("the include-pattern AQL query") + + output := buffer.String() + stderrBuffer.String() + assert.Contains(t, output, "blacked-out-repo") + assert.Contains(t, output, "42") + assert.Contains(t, output, "read permission") + }) + + t.Run("no warning when repo summary reports zero files", func(t *testing.T) { + buffer, stderrBuffer, previousLog := tests.RedirectLogOutputToBuffer() + defer log.SetLogger(previousLog) + + phase := &fullTransferPhase{ + phaseBase: phaseBase{ + repoKey: "genuinely-empty-repo", + repoSummary: servicesUtils.RepositorySummary{FilesCount: json.Number("0")}, + }, + } + phase.warnIfRepoAppearsBlackedOut("the include-pattern AQL query") + + assert.NotContains(t, buffer.String()+stderrBuffer.String(), "read permission") + }) + + t.Run("no warning when repo summary's file count is unparseable", func(t *testing.T) { + buffer, stderrBuffer, previousLog := tests.RedirectLogOutputToBuffer() + defer log.SetLogger(previousLog) + + phase := &fullTransferPhase{phaseBase: phaseBase{repoKey: "no-summary-repo"}} + phase.warnIfRepoAppearsBlackedOut("the include-pattern AQL query") + + assert.NotContains(t, buffer.String()+stderrBuffer.String(), "read permission") + }) +} + func TestMaybeWarnCompletedFolderSkippedWithFilter(t *testing.T) { t.Run("no warning without filter", func(t *testing.T) { buffer, stderrBuffer, previousLog := tests.RedirectLogOutputToBuffer() diff --git a/artifactory/commands/transferfiles/generatedpathfilter.go b/artifactory/commands/transferfiles/generatedpathfilter.go new file mode 100644 index 000000000..972043710 --- /dev/null +++ b/artifactory/commands/transferfiles/generatedpathfilter.go @@ -0,0 +1,186 @@ +package transferfiles + +import ( + "path" + "regexp" + "strings" + + "github.com/jfrog/jfrog-client-go/artifactory/services/utils" + "github.com/jfrog/jfrog-client-go/utils/log" +) + +const ( + // cocoapodsSpecsDir is the hidden index directory Artifactory writes generated podspec + // copies to (CocoaPodsConstants.SPECS_PATH in artifactory-service). Real user-uploaded + // podspecs live under the repo's visible "Specs/" or "pods/" layout instead. + cocoapodsSpecsDir = ".specs" + // swiftCacheDir is the hidden cache directory Artifactory writes generated Swift registry + // metadata to (SwiftConstants.CACHE_DIRECTORY in artifactory-service). Real uploaded source + // archives and package metadata live outside it. + swiftCacheDir = ".swift" +) + +var ( + yumRepodataPattern = regexp.MustCompile(`(^|.*/)repodata($|/.*)`) + yumTmpPattern = regexp.MustCompile(`(^|.*/)_tmp_\d{13,}($|/.*)`) + condaTmpPattern = regexp.MustCompile(`(^|.*/)_tmp-repodata-\d{13,}-.*`) + swiftManifestFile = regexp.MustCompile(`\APackage(@swift-(\d+)(?:\.(\d+))?(?:\.(\d+))?)?\.swift\z`) + swiftManifestURL = regexp.MustCompile(`\APackage\.swift\?swift-version=(\d+)(?:\.(\d+))?(?:\.(\d+))?\z`) +) + +// shouldExcludeGeneratedFile mirrors Artifactory isLocalGenerated checkers (and the older +// data-transfer PathPropsFilter table). Used when the target is older than 7.55 and +// POST /api/localgenerated/filter/paths is unavailable. +func shouldExcludeGeneratedFile(packageType, pathInRepo string) bool { + if pathInRepo == "" || pathInRepo == "." { + return false + } + // InternalMetadataChecker: .jfrog/ is local-generated for every package type. + if isInternalMetadataPath(pathInRepo) { + return true + } + switch strings.ToLower(packageType) { + case "p2", "ivy", "sbt", "maven", "gradle": + return isMavenGeneratedPath(pathInRepo) + case "npm": + return pathInRepo == ".npm" || strings.HasPrefix(pathInRepo, ".npm/") + case "pub": + return pathInRepo == ".pub" || strings.HasPrefix(pathInRepo, ".pub/") + case "rpm", "yum": + return yumRepodataPattern.MatchString(pathInRepo) || yumTmpPattern.MatchString(pathInRepo) + case "gems": + return isGemsGeneratedPath(pathInRepo) + case "helm": + return strings.HasSuffix(pathInRepo, "index.yaml") + case "opkg": + return isOpkgGeneratedPath(pathInRepo) + case "pypi": + return pathInRepo == ".pypi/simple.html" + case "cargo": + return strings.HasPrefix(pathInRepo, ".git/") || + pathInRepo == "config.json" || + pathInRepo == "config.original.json" || + strings.HasPrefix(pathInRepo, "index/") + case "conan": + return strings.HasSuffix(pathInRepo, "index.json") || + strings.HasSuffix(pathInRepo, ".conan/packages.ref.json") + case "conda": + return strings.HasSuffix(pathInRepo, "current_repodata.json") || + strings.HasSuffix(pathInRepo, "repodata.json") || + condaTmpPattern.MatchString(pathInRepo) + case "nuget": + return strings.HasPrefix(pathInRepo, ".nuget") || strings.HasPrefix(pathInRepo, ".nuGetV3/") + case "swift": + return isUnderDir(pathInRepo, swiftCacheDir) && + (strings.HasSuffix(pathInRepo, "releases.json") || + strings.HasSuffix(pathInRepo, "release_info.json") || + isSwiftGeneratedManifest(pathInRepo)) + case "alpine": + return strings.HasSuffix(pathInRepo, "APKINDEX.tar.gz") + case "debian": + return isDebianGeneratedPath(pathInRepo) + case "docker", "oci", "helmoci": + return isDockerGeneratedPath(pathInRepo) + case "puppet": + return strings.HasPrefix(pathInRepo, ".puppet") + case "composer": + return isComposerGeneratedPath(pathInRepo) + case "cocoapods": + return isUnderDir(pathInRepo, cocoapodsSpecsDir) && + (strings.HasSuffix(pathInRepo, ".podspec.json") || strings.HasSuffix(pathInRepo, ".podspec")) + case "terraform": + return strings.HasSuffix(pathInRepo, "module.json") + case "releasebundles": + return strings.HasSuffix(pathInRepo, ".json.draft") + default: + return false + } +} + +func isInternalMetadataPath(pathInRepo string) bool { + return pathInRepo == ".jfrog" || strings.HasPrefix(pathInRepo, ".jfrog/") +} + +// isUnderDir reports whether dir appears as an actual path segment of pathInRepo, rather than +// merely as a substring, so a hidden generated-file directory (e.g. ".specs", ".swift") never +// matches a real file that merely happens to contain that text in its name. +func isUnderDir(pathInRepo, dir string) bool { + return pathInRepo == dir || strings.HasPrefix(pathInRepo, dir+"/") || strings.Contains(pathInRepo, "/"+dir+"/") +} + +func isDockerGeneratedPath(pathInRepo string) bool { + return strings.HasSuffix(pathInRepo, "repository.catalog") || + strings.HasSuffix(pathInRepo, "repository_v2.catalog") || + pathInRepo == "_uploads" || + strings.HasPrefix(pathInRepo, "_uploads/") || + strings.Contains(pathInRepo, "/_uploads/") +} + +func isMavenGeneratedPath(pathInRepo string) bool { + fileName := path.Base(pathInRepo) + return strings.HasPrefix(fileName, "nexus-maven-repository-index") || + fileName == "maven-metadata.xml" || + strings.HasSuffix(fileName, ":maven-metadata.xml") +} + +func isGemsGeneratedPath(pathInRepo string) bool { + switch pathInRepo { + case "specs.4.8.gz", "latest_specs.4.8.gz", "prerelease_specs.4.8.gz", "versions": + return true + } + return strings.HasPrefix(pathInRepo, "info/") || strings.HasSuffix(pathInRepo, ".gemspec.rz") +} + +func isOpkgGeneratedPath(pathInRepo string) bool { + switch strings.ToLower(path.Base(pathInRepo)) { + case "packages", "packages.gz", "packages.bz2", "packages.xz", "packages.lzma", + "packages.stamps", "packages.sig", "packages.gz.sig": + return true + default: + return false + } +} + +func isComposerGeneratedPath(pathInRepo string) bool { + if pathInRepo == ".composer/packages.json" { + return true + } + if strings.HasPrefix(pathInRepo, ".composer/packages/") && strings.HasSuffix(pathInRepo, "list.json") { + return true + } + return (strings.HasPrefix(pathInRepo, ".composer/p/") || strings.HasPrefix(pathInRepo, ".composer/p2/")) && + strings.HasSuffix(pathInRepo, ".json") +} + +func isDebianGeneratedPath(pathInRepo string) bool { + if !isDebianDistPath(pathInRepo) { + return false + } + lower := strings.ToLower(pathInRepo) + return !strings.HasSuffix(lower, ".deb") && !strings.HasSuffix(lower, ".ddeb") +} + +func isDebianDistPath(pathInRepo string) bool { + return pathInRepo == "dists" || strings.HasPrefix(pathInRepo, "dists/") || strings.Contains(pathInRepo, "/dists/") +} + +func isSwiftGeneratedManifest(pathInRepo string) bool { + fileName := path.Base(pathInRepo) + return swiftManifestFile.MatchString(fileName) || swiftManifestURL.MatchString(fileName) +} + +func filterGeneratedPathsByPackageType(aqlResultItems []utils.ResultItem, packageType string) []utils.ResultItem { + if packageType == "" { + return aqlResultItems + } + kept := make([]utils.ResultItem, 0, len(aqlResultItems)) + for i := range aqlResultItems { + pathInRepo := getPathInRepo(&aqlResultItems[i]) + if shouldExcludeGeneratedFile(packageType, pathInRepo) { + log.Debug("Excluding locally generated item from being transferred:", pathInRepo) + continue + } + kept = append(kept, aqlResultItems[i]) + } + return kept +} diff --git a/artifactory/commands/transferfiles/generatedpathfilter_test.go b/artifactory/commands/transferfiles/generatedpathfilter_test.go new file mode 100644 index 000000000..5c17d8e16 --- /dev/null +++ b/artifactory/commands/transferfiles/generatedpathfilter_test.go @@ -0,0 +1,108 @@ +package transferfiles + +import ( + "context" + "net/http" + "testing" + + "github.com/jfrog/jfrog-cli-core/v2/common/tests" + "github.com/jfrog/jfrog-client-go/artifactory/services/utils" + "github.com/stretchr/testify/assert" +) + +func Test_shouldExcludeGeneratedFile(t *testing.T) { + cases := []struct { + packageType string + path string + exclude bool + }{ + {packageType: "npm", path: ".npm/pkg", exclude: true}, + {packageType: "npm", path: ".npm", exclude: true}, + {packageType: "npm", path: ".npmbackup.tgz", exclude: false}, + {packageType: "NPM", path: "pkg/-/pkg-1.0.0.tgz", exclude: false}, + {packageType: "docker", path: "repo/repository.catalog", exclude: true}, + {packageType: "docker", path: ".jfrog/repository_v2.catalog", exclude: true}, + {packageType: "docker", path: "busybox/_uploads/layer", exclude: true}, + {packageType: "docker", path: "_uploads/layer", exclude: true}, + {packageType: "docker", path: "busybox/manifest.json", exclude: false}, + {packageType: "oci", path: "img/repository_v2.catalog", exclude: true}, + {packageType: "helmoci", path: "chart/_uploads/blob", exclude: true}, + {packageType: "generic", path: ".jfrog/internal", exclude: true}, + {packageType: "debian", path: "dists/stable/Release", exclude: true}, + {packageType: "debian", path: "dists/stable/foo.deb", exclude: false}, + {packageType: "debian", path: "pool/main/foo.deb", exclude: false}, + {packageType: "debian", path: "foosdists/stable/Release", exclude: false}, + {packageType: "gems", path: "latest_specs.4.8.gz", exclude: true}, + {packageType: "gems", path: "info/foo", exclude: true}, + {packageType: "gems", path: "foo.gemspec.rz", exclude: true}, + {packageType: "cargo", path: "config.original.json", exclude: true}, + {packageType: "maven", path: "org/foo/1.0/maven-metadata.xml", exclude: true}, + {packageType: "gradle", path: "nexus-maven-repository-index.gz", exclude: true}, + {packageType: "maven", path: "org/foo/1.0/foo-1.0.jar", exclude: false}, + {packageType: "helm", path: "index.yaml", exclude: true}, + {packageType: "helm", path: "charts/mychart-1.0.0.tgz", exclude: false}, + {packageType: "alpine", path: "x86_64/APKINDEX.tar.gz", exclude: true}, + {packageType: "conan", path: "pkg/1.0/index.json", exclude: true}, + {packageType: "conan", path: ".conan/packages.ref.json", exclude: true}, + {packageType: "nuget", path: ".nuget/packages", exclude: true}, + {packageType: "pypi", path: ".pypi/simple.html", exclude: true}, + {packageType: "cocoapods", path: ".specs/a/1.0/a.podspec.json", exclude: true}, + {packageType: "cocoapods", path: ".specs/a/1.0/a.podspec", exclude: true}, + // Real user-uploaded podspecs under the repo's visible layout must not be excluded (JFMIG-88). + {packageType: "cocoapods", path: "Specs/a.podspec.json", exclude: false}, + {packageType: "cocoapods", path: "Specs/a.podspec", exclude: false}, + {packageType: "cocoapods", path: "Specs/XferValid/1.0.1/XferValid.podspec.json", exclude: false}, + {packageType: "cocoapods", path: "pods/XferValid/1.0.1/XferValid.podspec.json", exclude: false}, + {packageType: "cocoapods", path: "pods/xfer-pad/1.0.0/xfer-pad.podspec.json", exclude: false}, + {packageType: "terraform", path: "mod/module.json", exclude: true}, + {packageType: "cargo", path: "config.json", exclude: true}, + {packageType: "cargo", path: ".git/HEAD", exclude: true}, + {packageType: "cargo", path: "index/ab/crate", exclude: true}, + {packageType: "cargo", path: "crates/foo.crate", exclude: false}, + {packageType: "gems", path: "specs.4.8.gz", exclude: true}, + {packageType: "gems", path: "pkg-1.0.0.gem", exclude: false}, + {packageType: "composer", path: ".composer/packages.json", exclude: true}, + {packageType: "opkg", path: "arm/Packages.gz", exclude: true}, + {packageType: "conda", path: "linux-64/repodata.json", exclude: true}, + {packageType: "rpm", path: "repodata/repomd.xml", exclude: true}, + {packageType: "yum", path: "repodata/repomd.xml", exclude: true}, + {packageType: "swift", path: ".swift/scope/pkg/releases.json", exclude: true}, + {packageType: "swift", path: ".swift/scope/pkg/1.0.0/release_info.json", exclude: true}, + {packageType: "swift", path: ".swift/scope/pkg/1.0.0/Package.swift?swift-version=5", exclude: true}, + {packageType: "swift", path: ".swift/scope/pkg/1.0.0/Package.swift", exclude: true}, + {packageType: "swift", path: ".swift/scope/pkg/1.0.0/Package@swift-5.7.swift", exclude: true}, + {packageType: "swift", path: "Sources/foo.swift", exclude: false}, + // Real package release metadata outside the hidden .swift/ cache dir must not be excluded (JFMIG-88). + {packageType: "swift", path: "pkg/releases.json", exclude: false}, + {packageType: "swift", path: "xfer-pad/1.0.0/releases.json", exclude: false}, + {packageType: "swift", path: "Package.swift?swift-version=5", exclude: false}, + {packageType: "releasebundles", path: "bundle.json.draft", exclude: true}, + {packageType: "releasebundles", path: "bundle.json", exclude: false}, + {packageType: "go", path: "github.com/foo/@v/list", exclude: false}, + {packageType: "generic", path: ".npm/pkg", exclude: false}, + {packageType: "pub", path: ".pub/cache", exclude: true}, + {packageType: "pub", path: "pkg/config.json", exclude: false}, + {packageType: "chef", path: "any/path", exclude: false}, + } + + for _, tc := range cases { + t.Run(tc.packageType+"/"+tc.path, func(t *testing.T) { + assert.Equal(t, tc.exclude, shouldExcludeGeneratedFile(tc.packageType, tc.path)) + }) + } +} + +func TestFilterLocallyGenerated_disabledUsesPluginPathTable(t *testing.T) { + testServer, _, servicesManager := tests.CreateRtRestsMockServer(t, func(http.ResponseWriter, *http.Request) {}) + defer testServer.Close() + disabled := NewLocallyGenerated(context.Background(), servicesManager, "7.54.5") + + items := []utils.ResultItem{ + {Repo: "npm-local", Path: ".", Name: "pkg-1.0.0.tgz"}, + {Repo: "npm-local", Path: ".npm", Name: "pkg"}, + } + results, err := disabled.FilterLocallyGenerated(items, "npm") + assert.NoError(t, err) + assert.Len(t, results, 1) + assert.Equal(t, "pkg-1.0.0.tgz", results[0].Name) +} diff --git a/artifactory/commands/transferfiles/legacyfilters.go b/artifactory/commands/transferfiles/legacyfilters.go new file mode 100644 index 000000000..b86fffb0b --- /dev/null +++ b/artifactory/commands/transferfiles/legacyfilters.go @@ -0,0 +1,81 @@ +package transferfiles + +import ( + "strings" + + "github.com/jfrog/gofrog/datastructures" +) + +// packageTypeGeneratedProperties mirrors the exact property keys excluded by jfrog/data-transfer 1.7.6: +// src/main/java/com/jfrog/filters/PathPropsFilter.java and its package-specific subclasses at +// commit dbeeeaa95fd791fc929f579082ea1054e1cd5e0d. PathPropsFilter uses exact set membership; +// only subclass overrides such as ConanFilter's conan.settings prefix are matched by prefix. +// +// Verified directly against that commit's source (not just this comment's say-so): PathPropsFilter +// itself has no global/cross-package exclusion - shouldExcludeProperty is a plain per-instance +// excludedProps.contains(key) - so there is no plugin-side equivalent of a package-type-agnostic +// deny list. The 5 keys previously treated as globally generated by this file each come from exactly +// one subclass's shouldExcludeProperty override, matching the assignment below: +// - artifactory.licenses -> ChefFilter, CondaFilter, DebianFilter, ComposerFilter +// - artifactory.metadata.exclude -> NpmFilter +// - package.lowercase -> GoFilter +// - ruby -> GemsFilter +// - baseUrl -> PubFilter +// - conan.settings. prefix -> ConanFilter (startsWith, ConanFilter only) +// +// All other package-specific filters (AlpineFilter, OpkgFilter, BowerFilter, TerraformFilter, +// CargoFilter, NuGetFilter, PuppetFilter, CocoaPodsFilter, HelmFilter, SwiftFilter, PypiFilter, +// YumFilter, MavenFilter, DockerFilter, EmptyFilter) do not override shouldExcludeProperty at all, +// so none of the 5 keys apply outside the package types listed above. +var packageTypeGeneratedProperties = map[string]*datastructures.Set[string]{ + "go": datastructures.MakeSetFromElements("go.name", "package.lowercase"), + "npm": datastructures.MakeSetFromElements("npm.name", "npm.version", "npm.description", "npm.keywords", "npm.deprecated", + "npm.detachedtags", "artifactory.metadata.exclude"), + "pub": datastructures.MakeSetFromElements("pub.name", "pub.version", "pub.description", "pub.homepage", "pub.repository", + "pub.dependencies", "pub.devdependencies", "pub.environment", "baseurl"), + "rpm": datastructures.MakeSetFromElements("rpm.metadata.name", "rpm.metadata.arch", "rpm.metadata.version", "rpm.metadata.license", + "rpm.metadata.release", "rpm.metadata.epoch", "rpm.metadata.group", "rpm.metadata.vendor", "rpm.metadata.summary"), + "chef": datastructures.MakeSetFromElements("chef.name", "chef.version", "chef.maintainer", "chef.description", "chef.external", + "chef.issues", "chef.dependencies", "chef.platforms", "artifactory.licenses"), + "gems": datastructures.MakeSetFromElements("gem.name", "gem.version", "gem.platform", "gem.runtime.dependencies", "ruby"), + "helm": datastructures.MakeSetFromElements("chart.name", "chart.version", "chart.appversion", "chart.description", "chart.home", + "chart.created", "chart.type", "chart.annotations", "chart.apiversion", "chart.isdeprecated", + "chart.sources", "chart.maintainers", "chart.dependencies"), + "opkg": datastructures.MakeSetFromElements("opkg.architecture", "opkg.name", "opkg.version", "opkg.maintainer", "opkg.priority", + "opkg.section", "opkg.website"), + "pypi": datastructures.MakeSetFromElements("pypi.name", "pypi.normalized.name", "pypi.version", "pypi.summary", "pypi.requires.python"), + "bower": datastructures.MakeSetFromElements("bower.name", "bower.version", "bower.pkg"), + "cargo": datastructures.MakeSetFromElements("crate.name", "crate.version", "crate.description", "crate.keywords", "crate.categories", + "crate.dependencies", "crate.features"), + "conan": datastructures.MakeSetFromElements("conan.recipe_hash", "conan.requires", "conan.packages.author", "conan.packages.license", + "conan.packages.url"), + "conda": datastructures.MakeSetFromElements("conda.name", "conda.version", "conda.arch", "conda.platform", "artifactory.licenses"), + "nuget": datastructures.MakeSetFromElements("nuget.id", "nuget.version", "nuget.title", "nuget.authors", "nuget.summary", + "nuget.copyright", "nuget.releasenotes", "nuget.owners", "nuget.description", "nuget.requirelicenseacceptance", + "nuget.projecturl", "nuget.iconurl", "nuget.licenseurl", "nuget.tags", "nuget.language", "nuget.digest", + "nuget.dependency", "nuget.reference", "nuget.frameworks"), + "swift": datastructures.MakeSetFromElements("swift.name", "swift.version"), + "alpine": datastructures.MakeSetFromElements("alpine.name", "alpine.version", "alpine.branch", "alpine.repository", "alpine.architecture"), + "debian": datastructures.MakeSetFromElements("deb.name", "deb.version", "deb.maintainer", "deb.priority", "deb.section", "deb.website", + "artifactory.licenses"), + "puppet": datastructures.MakeSetFromElements("puppet.name", "puppet.version", "puppet.description", "puppet.module_groups", + "puppet.supported", "puppet.license", "puppet.author", "puppet.tags", "puppet.dependencies"), + "composer": datastructures.MakeSetFromElements("composer.name", "composer.version", "composer.description", "composer.author", + "composer.type", "composer.dependencies", "composer.keywords", "composer.full.reindex", "artifactory.licenses"), + "cocoapods": datastructures.MakeSetFromElements("pods.name", "pods.version", "pods.git.org", "pods.git.repo", "pods.git.version"), + "terraform": datastructures.MakeSetFromElements("terraform.id", "terraform.namespace", "terraform.version", "terraform.name", + "terraform.provider", "terraform.flavor", "terraform.type"), +} + +// isGeneratedPropertyKey reports whether a property key is package-generated and should be +// skipped on deploy. The production caller (filterEligibleProperties) lands in the target-client slice. +func isGeneratedPropertyKey(key, packageType string) bool { + normalizedKey := strings.ToLower(key) + normalizedPackageType := strings.ToLower(packageType) + if normalizedPackageType == "conan" && strings.HasPrefix(normalizedKey, "conan.settings.") { + return true + } + + generatedKeys := packageTypeGeneratedProperties[normalizedPackageType] + return generatedKeys != nil && generatedKeys.Exists(normalizedKey) +} diff --git a/artifactory/commands/transferfiles/legacyfilters_test.go b/artifactory/commands/transferfiles/legacyfilters_test.go new file mode 100644 index 000000000..6ad8b47f8 --- /dev/null +++ b/artifactory/commands/transferfiles/legacyfilters_test.go @@ -0,0 +1,191 @@ +package transferfiles + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_isGeneratedPropertyKey_matchesPluginExactSets(t *testing.T) { + cases := []struct { + name string + packageType string + generated []string + eligible []string + }{ + { + name: "go", + packageType: "Go", + generated: []string{"go.name", "package.lowercase"}, + eligible: []string{"go.version", "build.name", "npm.name"}, + }, + { + name: "npm", + packageType: "npm", + generated: []string{"npm.name", "npm.version", "npm.description", "npm.keywords", "npm.deprecated", + "npm.detachedtags", "artifactory.metadata.exclude"}, + eligible: []string{"npm.custom", "build.name", "go.name"}, + }, + { + name: "pub", + packageType: "Pub", + generated: []string{"pub.name", "pub.version", "pub.description", "pub.homepage", "pub.repository", + "pub.dependencies", "pub.devDependencies", "pub.environment", "baseUrl"}, + eligible: []string{"pub.custom", "build.name", "npm.name"}, + }, + { + name: "rpm", + packageType: "rpm", + generated: []string{"rpm.metadata.name", "rpm.metadata.arch", "rpm.metadata.version", "rpm.metadata.license", + "rpm.metadata.release", "rpm.metadata.epoch", "rpm.metadata.group", "rpm.metadata.vendor", + "rpm.metadata.summary"}, + eligible: []string{"rpm.metadata.custom", "build.name", "rpm.custom"}, + }, + { + name: "chef", + packageType: "chef", + generated: []string{"chef.name", "chef.version", "chef.maintainer", "chef.description", "chef.external", + "chef.issues", "chef.dependencies", "chef.platforms", "artifactory.licenses"}, + eligible: []string{"chef.custom", "build.name"}, + }, + { + name: "gems", + packageType: "Gems", + generated: []string{"gem.name", "gem.version", "gem.platform", "gem.runtime.dependencies", "ruby"}, + eligible: []string{"gem.custom", "build.name", "gems.custom"}, + }, + { + name: "helm", + packageType: "helm", + generated: []string{"chart.name", "chart.version", "chart.appVersion", "chart.description", "chart.home", + "chart.created", "chart.type", "chart.annotations", "chart.apiVersion", "chart.isDeprecated", + "chart.sources", "chart.maintainers", "chart.dependencies"}, + eligible: []string{"chart.custom", "build.name", "helm.custom"}, + }, + { + name: "opkg", + packageType: "opkg", + generated: []string{"opkg.architecture", "opkg.name", "opkg.version", "opkg.maintainer", "opkg.priority", + "opkg.section", "opkg.website"}, + eligible: []string{"opkg.custom", "build.name"}, + }, + { + name: "pypi", + packageType: "pypi", + generated: []string{"pypi.name", "pypi.normalized.name", "pypi.version", "pypi.summary", "pypi.requires.python"}, + eligible: []string{"pypi.custom", "build.name"}, + }, + { + name: "bower", + packageType: "bower", + generated: []string{"bower.name", "bower.version", "bower.pkg"}, + eligible: []string{"bower.custom", "build.name"}, + }, + { + name: "cargo", + packageType: "cargo", + generated: []string{"crate.name", "crate.version", "crate.description", "crate.keywords", "crate.categories", + "crate.dependencies", "crate.features"}, + eligible: []string{"crate.custom", "build.name"}, + }, + { + name: "conan", + packageType: "conan", + generated: []string{"conan.recipe_hash", "conan.requires", "conan.packages.author", "conan.packages.license", + "conan.packages.url", "conan.settings.os", "conan.settings.compiler.version", "conan.settings.options"}, + eligible: []string{"conan.name", "conan.author", "build.name"}, + }, + { + name: "conda", + packageType: "conda", + generated: []string{"conda.name", "conda.version", "conda.arch", "conda.platform", "artifactory.licenses"}, + eligible: []string{"conda.custom", "build.name"}, + }, + { + name: "nuget", + packageType: "NuGet", + generated: []string{"nuget.id", "nuget.version", "nuget.title", "nuget.authors", "nuget.summary", + "nuget.copyright", "nuget.releaseNotes", "nuget.owners", "nuget.description", + "nuget.requireLicenseAcceptance", "nuget.projectUrl", "nuget.iconUrl", "nuget.licenseUrl", + "nuget.tags", "nuget.language", "nuget.digest", "nuget.dependency", "nuget.reference", + "nuget.frameworks"}, + eligible: []string{"nuget.custom", "build.name"}, + }, + { + name: "swift", + packageType: "swift", + generated: []string{"swift.name", "swift.version"}, + eligible: []string{"swift.custom", "build.name"}, + }, + { + name: "alpine", + packageType: "alpine", + generated: []string{"alpine.name", "alpine.version", "alpine.branch", "alpine.repository", "alpine.architecture"}, + eligible: []string{"alpine.origin", "alpine.maintainer", "build.name"}, + }, + { + name: "debian", + packageType: "debian", + generated: []string{"deb.name", "deb.version", "deb.maintainer", "deb.priority", "deb.section", + "deb.website", "artifactory.licenses"}, + eligible: []string{"deb.distribution", "deb.component", "deb.architecture", "build.name", "debian.custom"}, + }, + { + name: "puppet", + packageType: "puppet", + generated: []string{"puppet.name", "puppet.version", "puppet.description", "puppet.module_groups", + "puppet.supported", "puppet.license", "puppet.author", "puppet.tags", "puppet.dependencies"}, + eligible: []string{"puppet.custom", "build.name"}, + }, + { + name: "composer", + packageType: "composer", + generated: []string{"composer.name", "composer.version", "composer.description", "composer.author", + "composer.type", "composer.dependencies", "composer.keywords", "composer.full.reindex", + "artifactory.licenses"}, + eligible: []string{"composer.custom", "build.name"}, + }, + { + name: "cocoapods", + packageType: "cocoapods", + generated: []string{"pods.name", "pods.version", "pods.git.org", "pods.git.repo", "pods.git.version"}, + eligible: []string{"pods.custom", "build.name"}, + }, + { + name: "terraform", + packageType: "terraform", + generated: []string{"terraform.id", "terraform.namespace", "terraform.version", "terraform.name", + "terraform.provider", "terraform.flavor", "terraform.type"}, + eligible: []string{"terraform.custom", "build.name"}, + }, + { + name: "maven-no-prefix", + packageType: "maven", + eligible: []string{"artifactory.licenses", "artifactory.metadata.exclude", "package.lowercase", "ruby", + "baseUrl", "conan.settings.os", "build.name", "maven.name"}, + }, + { + name: "docker-no-prefix", + packageType: "docker", + eligible: []string{"artifactory.licenses", "artifactory.metadata.exclude", "package.lowercase", "ruby", + "baseUrl", "conan.settings.os", "build.name", "docker.manifest"}, + }, + { + name: "generic-no-prefix", + packageType: "generic", + eligible: []string{"artifactory.licenses", "artifactory.metadata.exclude", "package.lowercase", "ruby", + "baseUrl", "conan.settings.os", "build.name", "npm.name", "go.name"}, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + for _, key := range tc.generated { + assert.True(t, isGeneratedPropertyKey(key, tc.packageType), "expected generated key %q for %q", key, tc.packageType) + } + for _, key := range tc.eligible { + assert.False(t, isGeneratedPropertyKey(key, tc.packageType), "expected eligible key %q for %q", key, tc.packageType) + } + }) + } +} diff --git a/artifactory/commands/transferfiles/localgeneratedfilter.go b/artifactory/commands/transferfiles/localgeneratedfilter.go index 768b14c63..a23421507 100644 --- a/artifactory/commands/transferfiles/localgeneratedfilter.go +++ b/artifactory/commands/transferfiles/localgeneratedfilter.go @@ -57,10 +57,15 @@ func NewLocallyGenerated(context context.Context, serviceManager artifactory.Art // Filters out locally generated files. // Files that are generated automatically by Artifactory on the target instance (also known as "locally generated files") should not be transferred. // aqlResults - Directory content in phase 1 or 15 minutes interval results in phase 2. -func (lg *locallyGeneratedFilter) FilterLocallyGenerated(aqlResultItems []utils.ResultItem) ([]utils.ResultItem, error) { - if !lg.enabled || len(aqlResultItems) == 0 { +// packageType is the source repository package type (used when the target is older than 7.55). +// An empty value means no legacy package-type filter is applied. +func (lg *locallyGeneratedFilter) FilterLocallyGenerated(aqlResultItems []utils.ResultItem, packageType string) ([]utils.ResultItem, error) { + if len(aqlResultItems) == 0 { return aqlResultItems, nil } + if !lg.enabled { + return filterGeneratedPathsByPackageType(aqlResultItems, packageType), nil + } content, err := lg.createPayload(aqlResultItems) if err != nil || len(content) == 0 { return []utils.ResultItem{}, err @@ -90,8 +95,8 @@ func (lg *locallyGeneratedFilter) doFilterLocallyGenerated(content []byte) (resp return lg.targetServiceDetails.GetClient().SendPost(lg.targetServiceDetails.GetUrl()+locallyGeneratedApi, content, lg.httpDetails) } -// Return true if should filter Artifactory locally generated files in the JFrog CLI -// False if should filter Artifactory locally generated files in the Data Transfer plugin +// Return true if locally generated paths are filtered via POST /api/localgenerated/filter/paths (target >= 7.55). +// False if the CLI applies the plugin PathPropsFilter path table instead (target < 7.55). func (lg *locallyGeneratedFilter) IsEnabled() bool { return lg.enabled } diff --git a/artifactory/commands/transferfiles/localgeneratedfilter_test.go b/artifactory/commands/transferfiles/localgeneratedfilter_test.go index 3b6e46a24..c43247822 100644 --- a/artifactory/commands/transferfiles/localgeneratedfilter_test.go +++ b/artifactory/commands/transferfiles/localgeneratedfilter_test.go @@ -45,14 +45,14 @@ func TestFilterLocallyGenerated(t *testing.T) { returnedLocallyGenerated, err = json.Marshal(returnedPayload) assert.NoError(t, err) - results, err := locallyGeneratedEnabled.FilterLocallyGenerated(testCase.paths) + results, err := locallyGeneratedEnabled.FilterLocallyGenerated(testCase.paths, "generic") assert.NoError(t, err) assert.Len(t, results, len(testCase.returnedPath)) for i := range results { assert.Contains(t, testCase.returnedPath, getPathInRepo(&results[i])) } - results, err = locallyGeneratedDisabled.FilterLocallyGenerated(testCase.paths) + results, err = locallyGeneratedDisabled.FilterLocallyGenerated(testCase.paths, "generic") assert.NoError(t, err) assert.Equal(t, results, testCase.paths) }) @@ -84,7 +84,7 @@ func TestFilterLocallyGeneratedMaxRequests(t *testing.T) { wg.Add(1) go func() { defer wg.Done() - _, err := locallyGenerated.FilterLocallyGenerated([]utils.ResultItem{{Path: "a", Name: "b"}}) + _, err := locallyGenerated.FilterLocallyGenerated([]utils.ResultItem{{Path: "a", Name: "b"}}, "generic") assert.NoError(t, err) }() } diff --git a/artifactory/commands/transferfiles/manager.go b/artifactory/commands/transferfiles/manager.go index 7a6cb22c1..62478befc 100644 --- a/artifactory/commands/transferfiles/manager.go +++ b/artifactory/commands/transferfiles/manager.go @@ -1,16 +1,9 @@ package transferfiles import ( - "context" - "errors" - "fmt" "sync" - "time" - "github.com/jfrog/gofrog/datastructures" "github.com/jfrog/gofrog/parallel" - "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transfer" - "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/api" "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/state" clientUtils "github.com/jfrog/jfrog-client-go/utils" "github.com/jfrog/jfrog-client-go/utils/errorutils" @@ -18,7 +11,7 @@ import ( ) const ( - totalNumberPollingGoRoutines = 2 + totalNumberPollingGoRoutines = 1 tasksMaxCapacity = 5000000 ) @@ -33,7 +26,6 @@ func newTransferManager(base phaseBase, delayUploadComparisonFunctions []shouldD type transferActionWithProducerConsumerType func( pcWrapper *producerConsumerWrapper, - uploadChunkChan chan UploadedChunk, delayHelper delayUploadHelper, errorsChannelMng *ErrorsChannelMng) error @@ -41,22 +33,20 @@ type transferDelayAction func(phase phaseBase, addedDelayFiles []string) error // Transfer files using the 'producer-consumer' mechanism and apply a delay action. func (ftm *transferManager) doTransferWithProducerConsumer(transferAction transferActionWithProducerConsumerType, delayAction transferDelayAction) error { - // Set the producer-consumer value into the referenced value. This allow the Graceful Stop mechanism to access ftm.pcDetails when needed to stop the transfer. + // Set the producer-consumer value into the referenced value. This allows the Graceful Stop mechanism to access ftm.pcDetails when needed to stop the transfer. *ftm.pcDetails = newProducerConsumerWrapper() return ftm.doTransfer(ftm.pcDetails, transferAction, delayAction) } // This function handles a transfer process as part of a phase. // As part of the process, the transferAction gets executed. It may utilize a producer consumer or not. -// The transferAction collects artifacts to be uploaded into chunks, and sends them to the source Artifactory instance to handle asynchronously. -// An uuid token will be returned and sent in a channel to be polled on for status in pollUploads. +// The transferAction collects artifacts to be uploaded into chunks, and uploads them directly to the target Artifactory instance. // In some repositories the order of deployment is important. In these cases, any artifacts that should be delayed will be collected by // the delayedArtifactsMng and will later be handled by handleDelayedArtifactsFiles. // Any deployment failures will be written to a file by the transferErrorsMng to be handled on next run. -// The number of threads affect both the producer consumer if used, and limits the number of uploaded chunks. The number can be externally modified, -// and will be updated on runtime by periodicallyUpdateThreads. +// The number of threads affects both the producer consumer if used, and limits the number of chunks uploaded in parallel. The number can be externally modified, +// and will be updated on runtime by periodicallyUpdateThreadsAndStopStatus. func (ftm *transferManager) doTransfer(pcWrapper *producerConsumerWrapper, transferAction transferActionWithProducerConsumerType, delayAction transferDelayAction) error { - uploadChunkChan := make(chan UploadedChunk, transfer.MaxThreadsLimit) var runWaitGroup sync.WaitGroup var writersWaitGroup sync.WaitGroup @@ -87,7 +77,7 @@ func (ftm *transferManager) doTransfer(pcWrapper *producerConsumerWrapper, trans } pollingTasksManager := newPollingTasksManager(totalNumberPollingGoRoutines) - err = pollingTasksManager.start(&ftm.phaseBase, &runWaitGroup, pcWrapper, uploadChunkChan, &errorsChannelMng) + err = pollingTasksManager.start(&ftm.phaseBase, &runWaitGroup, pcWrapper) if err != nil { pollingTasksManager.stop() return err @@ -101,7 +91,7 @@ func (ftm *transferManager) doTransfer(pcWrapper *producerConsumerWrapper, trans } go func() { defer runWaitGroup.Done() - actionErr = transferAction(pcWrapper, uploadChunkChan, delayUploadHelper, &errorsChannelMng) + actionErr = transferAction(pcWrapper, delayUploadHelper, &errorsChannelMng) if pcWrapper == nil { pollingTasksManager.stop() } @@ -138,6 +128,11 @@ func (ftm *transferManager) doTransfer(pcWrapper *producerConsumerWrapper, trans return returnedError } +// PollingTasksManager coordinates the background go routines that run alongside a transfer phase. +// The name and 'doneChannel'/'totalGoRoutines' bookkeeping predate GET/PUT transfer, when this managed +// both a chunk-upload poller and the thread/stop-status poller; today only the latter +// (periodicallyUpdateThreadsAndStopStatus) runs, but the type is kept generic since more than one +// go routine may be added again in the future. type PollingTasksManager struct { // Done channel notifies the polling go routines that no more tasks are expected. doneChannel chan bool @@ -152,10 +147,9 @@ func newPollingTasksManager(totalGoRoutines int) PollingTasksManager { return PollingTasksManager{doneChannel: make(chan bool, totalGoRoutines), totalGoRoutines: totalGoRoutines} } -// Runs 2 go routines: -// 1. Periodically update the worker threads count & check whether the process should be stopped. -// 2. Poll for uploaded chunks. -func (ptm *PollingTasksManager) start(phaseBase *phaseBase, runWaitGroup *sync.WaitGroup, pcWrapper *producerConsumerWrapper, uploadChunkChan chan UploadedChunk, errorsChannelMng *ErrorsChannelMng) error { +// Runs the polling go routine(s) for a transfer phase. Currently starts a single go routine that +// periodically updates the worker threads count & checks whether the process should be stopped. +func (ptm *PollingTasksManager) start(phaseBase *phaseBase, runWaitGroup *sync.WaitGroup, pcWrapper *producerConsumerWrapper) error { // Update threads by polling on the settings file. runWaitGroup.Add(1) err := ptm.addGoRoutine() @@ -164,19 +158,14 @@ func (ptm *PollingTasksManager) start(phaseBase *phaseBase, runWaitGroup *sync.W } go func() { defer runWaitGroup.Done() - periodicallyUpdateThreadsAndStopStatus(pcWrapper, ptm.doneChannel, phaseBase.buildInfoRepo, phaseBase.stopSignal) + periodicallyUpdateThreadsAndStopStatus(pcWrapper, ptm.doneChannel, phaseBase.buildInfoRepo, phaseBase.stateManager, phaseBase.stopSignal) }() - // Check status of uploaded chunks. - runWaitGroup.Add(1) - err = ptm.addGoRoutine() - if err != nil { - return err + if phaseBase.stateManager != nil { + if err := phaseBase.stateManager.SetWorkingThreads(GetChunkUploaderThreads()); err != nil { + log.Error("Couldn't set the current number of working threads:", err.Error()) + } } - go func() { - defer runWaitGroup.Done() - pollUploads(pcWrapper, phaseBase, phaseBase.srcUpService, uploadChunkChan, ptm.doneChannel, errorsChannelMng) - }() return nil } @@ -189,7 +178,6 @@ func (ptm *PollingTasksManager) addGoRoutine() error { } func (ptm *PollingTasksManager) stop() { - // Notify the other go routines that work is done. for i := 0; i < ptm.totalRunningGoRoutines; i++ { ptm.doneChannel <- true } @@ -209,251 +197,24 @@ func newProducerConsumerWrapper() producerConsumerWrapper { } } -// Run the two producer consumer that run the transfer. -// When a producer consumer is idle for assumeProducerConsumerDoneWhenIdleForSeconds (no tasks are being handled) -// the work is assumed to be done. -// Order in this function matters! We want to make sure chunkUploaderProducerConsumer is only done after chunkBuilderProducerConsumer is done. func runProducerConsumers(pcWrapper *producerConsumerWrapper) (executionErr error) { go func() { pcWrapper.chunkUploaderProducerConsumer.Run() }() go func() { - // Wait till notified that the builder has no additional tasks, and close the builder producer consumer. <-pcWrapper.chunkBuilderProducerConsumer.GetFinishedNotification() log.Debug("Chunk builder producer consumer has completed all tasks. " + "All files relevant to this phase were found and added to chunks that are being uploaded...") pcWrapper.chunkBuilderProducerConsumer.Done() }() - // Run() is a blocking method, so once all chunk builders are idle, the tasks queue closes and Run() stops running. pcWrapper.chunkBuilderProducerConsumer.Run() if pcWrapper.chunkUploaderProducerConsumer.IsStarted() { - // There might be a moment when the chunk uploader has no upload tasks. - // This circumstance might lead to setting the finish notification before completing all file uploads. - // To address this, we reset the finish notification to ensure no remaining upload tasks after the next finish notification. pcWrapper.chunkUploaderProducerConsumer.ResetFinishNotificationIfActive() - // Wait till notified that the uploader finished its tasks, and it will not receive new tasks from the builder. <-pcWrapper.chunkUploaderProducerConsumer.GetFinishedNotification() log.Debug("Chunk uploaded producer consumer has completed all tasks. All files relevant to this phase have all been uploaded.") } - // Close the tasks queue with Done(). pcWrapper.chunkUploaderProducerConsumer.Done() executionErr = pcWrapper.errorsQueue.GetError() return } - -// This function polls on chunks of files that were uploaded during one of the phases. -// It does so by requesting the status of each chunk, by sending the uuid token that was returned when the chunk was uploaded. -// Number of chunks is limited by the number of threads. -// Whenever the status of a chunk was received and is DONE, its token is removed from the tokens batch, making room for a new chunk to be uploaded -// and a new token to be polled on. -func pollUploads(pcWrapper *producerConsumerWrapper, phaseBase *phaseBase, srcUpService *srcUserPluginService, uploadChunkChan chan UploadedChunk, doneChan chan bool, errorsChannelMng *ErrorsChannelMng) { - curTokensBatch := api.UploadChunksStatusBody{} - chunksLifeCycleManager := ChunksLifeCycleManager{ - deletedChunksSet: datastructures.MakeSet[api.ChunkId](), - nodeToChunksMap: make(map[api.NodeId]map[api.ChunkId]UploadedChunkData), - } - var timeEstMng *state.TimeEstimationManager - if phaseBase != nil { - timeEstMng = &phaseBase.stateManager.TimeEstimationManager - } - for i := 0; ; i++ { - if ShouldStop(phaseBase, nil, errorsChannelMng) { - log.Debug("Stop signal received while polling on uploads...") - return - } - time.Sleep(waitTimeBetweenChunkStatusSeconds * time.Second) - - // Run once per 5 minutes - if i%60 == 0 { - // 'Working threads' are determined by how many upload chunks are currently being processed by the source Artifactory instance. - if err := phaseBase.stateManager.SetWorkingThreads(pcWrapper.totalProcessedUploadChunks); err != nil { - log.Error("Couldn't set the current number of working threads:", err.Error()) - } - log.Debug("There are", len(phaseBase.stateManager.StaleChunks), "chunks in transit for more than 30 minutes") - log.Debug(fmt.Sprintf("Chunks in transit: %v", chunksLifeCycleManager.GetNodeIdToChunkIdsMap())) - } - - // Each uploading thread receives a token and a node id from the source via the uploadChunkChan, so this go routine can poll on its status. - activeChunks := fillChunkDataBatch(&chunksLifeCycleManager, uploadChunkChan) - if err := chunksLifeCycleManager.StoreStaleChunks(phaseBase.stateManager); err != nil { - log.Error("Couldn't store the stale chunks:", err.Error()) - } - // When totalChunks size is zero, it means that all the tokens are uploaded, - // we received 'DONE' for all of them, and we notified the source that they can be deleted from the memory. - // If during the polling some chunks data were lost due to network issues, either on the client or on the source, - // it will be written to the error channel - if activeChunks == 0 { - if shouldStopPolling(doneChan) { - log.Debug("Stopping to poll on uploads...") - return - } - log.Debug("Active chunks counter is 0, but the 'done' signal hasn't been received yet") - continue - } - - chunksStatus, err := sendSyncChunksRequest(curTokensBatch, &chunksLifeCycleManager, srcUpService) - if err != nil { - continue - } - // Clear body for the next request - curTokensBatch = api.UploadChunksStatusBody{} - removeDeletedChunksFromSet(chunksStatus.DeletedChunks, chunksLifeCycleManager.deletedChunksSet) - toStop := handleChunksStatuses(pcWrapper, phaseBase, &chunksStatus, &chunksLifeCycleManager, timeEstMng, errorsChannelMng) - if toStop { - return - } - } -} - -// Fill chunk data batch till full. Return if no new chunk data is available. -func fillChunkDataBatch(chunksLifeCycleManager *ChunksLifeCycleManager, uploadChunkChan chan UploadedChunk) (activeChunks int) { - for _, activeNodeChunks := range chunksLifeCycleManager.nodeToChunksMap { - activeChunks += len(activeNodeChunks) - } - for ; activeChunks < GetChunkUploaderThreads(); activeChunks++ { - select { - case data := <-uploadChunkChan: - currentNodeId := api.NodeId(data.NodeId) - currentChunkId := api.ChunkId(data.UuidToken) - if _, exist := chunksLifeCycleManager.nodeToChunksMap[currentNodeId]; !exist { - chunksLifeCycleManager.nodeToChunksMap[currentNodeId] = make(map[api.ChunkId]UploadedChunkData) - } - chunksLifeCycleManager.nodeToChunksMap[currentNodeId][currentChunkId] = data.UploadedChunkData - default: - // No new tokens are waiting. - return - } - } - return -} - -func shouldStopPolling(doneChan chan bool) bool { - select { - case done := <-doneChan: - return done - default: - } - return false -} - -// Send and handle. -func sendSyncChunksRequest(curTokensBatch api.UploadChunksStatusBody, chunksLifeCycleManager *ChunksLifeCycleManager, srcUpService *srcUserPluginService) (api.UploadChunksStatusResponse, error) { - curTokensBatch.AwaitingStatusChunks = chunksLifeCycleManager.GetInProgressTokensSlice() - curTokensBatch.ChunksToDelete = chunksLifeCycleManager.deletedChunksSet.ToSlice() - chunksStatus, err := srcUpService.syncChunks(curTokensBatch) - // Log the error only if the transfer wasn't interrupted by the user - if err != nil && !errors.Is(err, context.Canceled) { - log.Error("error returned when getting upload chunks statuses: " + err.Error()) - } - return chunksStatus, err -} - -func removeDeletedChunksFromSet(deletedChunks []string, deletedChunksSet *datastructures.Set[api.ChunkId]) { - // deletedChunks is an array received from the source, confirming which chunks were deleted from the source side. - // In deletedChunksSet, we keep only chunks for which we have yet to receive confirmation - for _, deletedChunk := range deletedChunks { - err := deletedChunksSet.Remove(api.ChunkId(deletedChunk)) - if err != nil { - log.Error(err.Error()) - continue - } - } -} - -// handleChunksStatuses handles the chunk statuses from the response received from the source Artifactory Instance. -// It syncs the chunk status between the CLI and the source Artifactory instance, -// When a chunk is DONE, the progress bar is updated, and the number of working threads is decreased. -func handleChunksStatuses(pcWrapper *producerConsumerWrapper, phase *phaseBase, chunksStatus *api.UploadChunksStatusResponse, - chunksLifeCycleManager *ChunksLifeCycleManager, timeEstMng *state.TimeEstimationManager, errorsChannelMng *ErrorsChannelMng) bool { - checkChunkStatusSync(pcWrapper, chunksStatus, chunksLifeCycleManager, errorsChannelMng) - for _, chunk := range chunksStatus.ChunksStatus { - if chunk.UuidToken == "" { - log.Error("Unexpected empty uuid token in status") - continue - } - switch chunk.Status { - case api.InProgress: - continue - case api.Done: - pcWrapper.decProcessedChunks() - log.Debug("Received status DONE for chunk '" + chunk.UuidToken + "'") - - chunkSentTime := chunksLifeCycleManager.nodeToChunksMap[api.NodeId(chunksStatus.NodeId)][api.ChunkId(chunk.UuidToken)].TimeSent - err := updateProgress(phase, timeEstMng, chunk, chunkSentTime) - if err != nil { - log.Error("Unexpected error in progress update: " + err.Error()) - continue - } - delete(chunksLifeCycleManager.nodeToChunksMap[api.NodeId(chunksStatus.NodeId)], api.ChunkId(chunk.UuidToken)) - // Using the deletedChunksSet, we inform the source that the 'DONE' message has been received, and it no longer has to keep those chunks UUIDs. - chunksLifeCycleManager.deletedChunksSet.Add(api.ChunkId(chunk.UuidToken)) - stopped := handleFilesOfCompletedChunk(chunk.Files, errorsChannelMng) - // In case an error occurred while writing errors status's to the errors file - stop transferring. - if stopped { - log.Debug("Stop signal received while handling chunks statuses...") - return true - } - err = setChunkCompletedInRepoSnapshot(phase.stateManager, chunk.Files) - if err != nil { - log.Error(err) - continue - } - } - } - return false -} - -func updateProgress(phase *phaseBase, timeEstMng *state.TimeEstimationManager, - chunk api.ChunkStatus, chunkSentTime time.Time) error { - if phase == nil { - return nil - } - - err := state.UpdateChunkInState(phase.stateManager, &chunk) - if err != nil { - return err - } - - if timeEstMng != nil { - return timeEstMng.AddChunkStatus(chunk, time.Since(chunkSentTime).Milliseconds()) - } - return nil -} - -// Verify and handle in progress chunks synchronization between the CLI and the Source Artifactory instance -func checkChunkStatusSync(pcWrapper *producerConsumerWrapper, chunkStatus *api.UploadChunksStatusResponse, chunksLifeCycleManager *ChunksLifeCycleManager, errorsChannelMng *ErrorsChannelMng) { - // Compare between the number of chunks received from the latest syncChunks request to the chunks data we handle locally in nodeToChunksMap. - // If the number of the in progress chunks of a node within nodeToChunksMap differs from the chunkStatus received, there is missing data on the source side. - expectedChunksInNode := len(chunksLifeCycleManager.nodeToChunksMap[api.NodeId(chunkStatus.NodeId)]) - actualChunksInNode := len(chunkStatus.ChunksStatus) - if actualChunksInNode != expectedChunksInNode { - log.Info(fmt.Printf("NodeID %s: Missing chunks detected. Expected: %d, Received: %d. Storing absent chunks in error channels for later retry.", - chunkStatus.NodeId, expectedChunksInNode, actualChunksInNode)) - // Get all the chunks uuids on the Artifactory side in a set of uuids - chunksUuidsSetFromResponse := datastructures.MakeSet[api.ChunkId]() - for _, chunk := range chunkStatus.ChunksStatus { - chunksUuidsSetFromResponse.Add(api.ChunkId(chunk.UuidToken)) - } - // Get all the chunks uuids on the CLI side - chunksUuidsSliceFromMap := chunksLifeCycleManager.GetInProgressTokensSliceByNodeId(api.NodeId(chunkStatus.NodeId)) - failedFile := api.FileUploadStatusResponse{ - Status: api.Fail, - StatusCode: SyncErrorStatusCode, - Reason: SyncErrorReason, - } - // Send all missing chunks from the source Artifactory instance to errorsChannelMng - // Missing chunks are those that are inside chunksUuidsSliceFromMap but not in chunksUuidsSetFromResponse - for _, chunkUuid := range chunksUuidsSliceFromMap { - if !chunksUuidsSetFromResponse.Exists(chunkUuid) { - for _, file := range chunksLifeCycleManager.nodeToChunksMap[api.NodeId(chunkStatus.NodeId)][chunkUuid].ChunkFiles { - failedFile.FileRepresentation = file - // errorsChannelMng will upload failed files again in phase 3 or in an additional transfer file run. - addErrorToChannel(errorsChannelMng, failedFile) - } - delete(chunksLifeCycleManager.nodeToChunksMap[api.NodeId(chunkStatus.NodeId)], chunkUuid) - pcWrapper.decProcessedChunks() - } - } - } -} diff --git a/artifactory/commands/transferfiles/phase.go b/artifactory/commands/transferfiles/phase.go index 01d3fcf73..699eb31f4 100644 --- a/artifactory/commands/transferfiles/phase.go +++ b/artifactory/commands/transferfiles/phase.go @@ -23,7 +23,6 @@ type transferPhase interface { setRepoKey(repoKey string) setCheckExistenceInFilestore(bool) shouldSkipPhase() (bool, error) - setSrcUserPluginService(*srcUserPluginService) setSourceDetails(*coreConfig.ServerDetails) getSourceDetails() *coreConfig.ServerDetails setTargetDetails(*coreConfig.ServerDetails) @@ -33,7 +32,6 @@ type transferPhase interface { setStateManager(stateManager *state.TransferStateManager) setLocallyGeneratedFilter(locallyGeneratedFilter *locallyGeneratedFilter) initProgressBar() error - setProxyKey(proxyKey string) setBuildInfo(setBuildInfo bool) setPackageType(packageType string) setDisabledDistinctiveAql() @@ -41,23 +39,27 @@ type transferPhase interface { setMinCheckSumDeploySize(minCheckSumDeploySize int64) setIncludeFilesPatterns(includeFilesPatterns []string) setTimestampFilter(filter *timestampFilter) + setFileTransfer(fileTransferExecutor) StopGracefully() } +type fileTransferExecutor interface { + TransferFile(ctx context.Context, candidate api.FileRepresentation) TransferResult +} + type phaseBase struct { context context.Context + fileTransfer fileTransferExecutor repoKey string buildInfoRepo bool packageType string phaseId int checkExistenceInFilestore bool startTime time.Time - srcUpService *srcUserPluginService srcRtDetails *coreConfig.ServerDetails targetRtDetails *coreConfig.ServerDetails progressBar *TransferProgressMng repoSummary serviceUtils.RepositorySummary - proxyKey string pcDetails *producerConsumerWrapper transferManager *transferManager stateManager *state.TransferStateManager @@ -101,6 +103,10 @@ func (pb *phaseBase) setContext(context context.Context) { pb.context = context } +func (pb *phaseBase) setFileTransfer(executor fileTransferExecutor) { + pb.fileTransfer = executor +} + func (pb *phaseBase) setRepoKey(repoKey string) { pb.repoKey = repoKey } @@ -109,10 +115,6 @@ func (pb *phaseBase) setCheckExistenceInFilestore(shouldCheck bool) { pb.checkExistenceInFilestore = shouldCheck } -func (pb *phaseBase) setSrcUserPluginService(service *srcUserPluginService) { - pb.srcUpService = service -} - func (pb *phaseBase) setSourceDetails(details *coreConfig.ServerDetails) { pb.srcRtDetails = details } @@ -129,10 +131,6 @@ func (pb *phaseBase) setProgressBar(progressbar *TransferProgressMng) { pb.progressBar = progressbar } -func (pb *phaseBase) setProxyKey(proxyKey string) { - pb.proxyKey = proxyKey -} - func (pb *phaseBase) setStateManager(stateManager *state.TransferStateManager) { pb.stateManager = stateManager } diff --git a/artifactory/commands/transferfiles/producerconsumer.go b/artifactory/commands/transferfiles/producerconsumer.go index 553b9d09a..b57a9e61a 100644 --- a/artifactory/commands/transferfiles/producerconsumer.go +++ b/artifactory/commands/transferfiles/producerconsumer.go @@ -1,42 +1,12 @@ package transferfiles import ( - "sync" - "github.com/jfrog/gofrog/parallel" clientUtils "github.com/jfrog/jfrog-client-go/utils" ) type producerConsumerWrapper struct { - // This Producer-Consumer is used to upload chunks, initialized in newProducerConsumerWrapper; each uploading thread waits to be given tasks from the queue. chunkUploaderProducerConsumer parallel.Runner - // This Producer-Consumer is used to execute AQLs and build chunks from the AQLs' results. The chunks data is sent to the go routines that will upload them. - // Initialized in newProducerConsumerWrapper; each builder thread waits to be given tasks from the queue. - chunkBuilderProducerConsumer parallel.Runner - // Errors related to chunkUploaderProducerConsumer and chunkBuilderProducerConsumer are logged in this queue. - errorsQueue *clientUtils.ErrorsQueue - // This variable holds the total number of upload chunk that were sent to the source Artifactory instance to process. - // Together with this mutex, they control the load on the user plugin and couple it to the local number of threads. - totalProcessedUploadChunks int - processedUploadChunksMutex sync.Mutex -} - -// Checks whether the total number of upload chunks sent is lower than the number of threads, and if so, increments it. -// Returns true if the total number was indeed incremented. -func (producerConsumerWrapper *producerConsumerWrapper) incProcessedChunksWhenPossible() bool { - producerConsumerWrapper.processedUploadChunksMutex.Lock() - defer producerConsumerWrapper.processedUploadChunksMutex.Unlock() - if producerConsumerWrapper.totalProcessedUploadChunks < GetChunkUploaderThreads() { - producerConsumerWrapper.totalProcessedUploadChunks++ - return true - } - return false -} - -// Reduces the current total number of upload chunks processed. Called when an upload chunks doesn't require polling for status - -// if it's done processing, or an error occurred when sending it. -func (producerConsumerWrapper *producerConsumerWrapper) decProcessedChunks() { - producerConsumerWrapper.processedUploadChunksMutex.Lock() - defer producerConsumerWrapper.processedUploadChunksMutex.Unlock() - producerConsumerWrapper.totalProcessedUploadChunks-- + chunkBuilderProducerConsumer parallel.Runner + errorsQueue *clientUtils.ErrorsQueue } diff --git a/artifactory/commands/transferfiles/proxy.go b/artifactory/commands/transferfiles/proxy.go new file mode 100644 index 000000000..5826babb5 --- /dev/null +++ b/artifactory/commands/transferfiles/proxy.go @@ -0,0 +1,271 @@ +package transferfiles + +import ( + "context" + "crypto/tls" + "encoding/json" + "fmt" + "net" + "net/http" + "net/url" + "path" + "strconv" + "strings" + "time" + + "github.com/jfrog/jfrog-cli-core/v2/utils/config" + "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" + "github.com/jfrog/jfrog-client-go/auth/cert" + "github.com/jfrog/jfrog-client-go/http/httpclient" + clientutils "github.com/jfrog/jfrog-client-go/utils" + "github.com/jfrog/jfrog-client-go/utils/errorutils" +) + +const proxyKeyUsageHint = "pass an HTTP or HTTPS proxy URL (for example http://proxy:3128), or configure HTTPS_PROXY and NO_PROXY on the CLI host. " + + "Note: when --proxy-key is set, it overrides ProxyFromEnvironment for the target, so NO_PROXY is not consulted" + +// streamResponseHeaderTimeout bounds only how long a streaming GET/PUT waits for the peer to +// start responding (TCP connect + send the response's status line/headers). It deliberately does +// NOT bound how long transferring a large file's body may take afterward: the streaming service +// managers are built with an overall http.Client.Timeout of 0 (see createStreamingTransferServiceManager), +// so once headers arrive, streaming can run for as long as the file takes. Without this, a peer +// that accepts a TCP connection but never writes any response (a hung reverse proxy, a stalled +// backend, ...) leaves the transfer blocked indefinitely with no way to detect or recover from it. +// +// A var (not a const) so tests can shrink it to avoid a multi-minute-long test run. +var streamResponseHeaderTimeout = 5 * time.Minute + +type sourceProxyDescriptor struct { + Key string `json:"key"` + Host string `json:"host"` + Port int `json:"port"` + Username string `json:"username"` + Password string `json:"password"` // #nosec G117 -- proxy credential returned by source Artifactory + Https bool `json:"https"` +} + +type sourceProxiesResponse struct { + Proxies []sourceProxyDescriptor `json:"proxies"` +} + +// resolveProxyKeyURL dispatches --proxy-key by shape: a value with a URL scheme is parsed +// directly (parseProxyKeyURL); a bare name is looked up against the named proxy configured on +// the source Artifactory instance (lookupSourceProxyURL), matching the plugin-era contract. +func resolveProxyKeyURL(ctx context.Context, raw string, sourceServerDetails *config.ServerDetails) (*url.URL, error) { + parsed, err := url.Parse(raw) + if err != nil { + return nil, errorutils.CheckErrorf("--proxy-key value %q is not a valid URL. %s", + redactProxyKey(raw), proxyKeyUsageHint) + } + if parsed.Scheme != "" { + return parseProxyKeyURL(raw) + } + return lookupSourceProxyURL(ctx, raw, sourceServerDetails) +} + +func lookupSourceProxyURL(ctx context.Context, proxyKey string, sourceServerDetails *config.ServerDetails) (*url.URL, error) { + if sourceServerDetails == nil { + return nil, namedProxyLookupError(proxyKey, "source Artifactory is not configured") + } + serviceManager, err := createTransferServiceManager(ctx, sourceServerDetails, nil) + if err != nil { + return nil, namedProxyLookupError(proxyKey, err.Error()) + } + artDetails := serviceManager.GetConfig().GetServiceDetails() + proxiesURL, err := clientutils.BuildUrl(artDetails.GetUrl(), + path.Join("api/system/configuration/platform/proxies"), map[string]string{}) + if err != nil { + return nil, namedProxyLookupError(proxyKey, err.Error()) + } + httpDetails := artDetails.CreateHttpClientDetails() + resp, body, _, err := serviceManager.Client().SendGet(proxiesURL, true, &httpDetails) + if err != nil { + return nil, namedProxyLookupError(proxyKey, err.Error()) + } + if resp.StatusCode != http.StatusOK { + return nil, namedProxyLookupError(proxyKey, "source Artifactory returned "+resp.Status) + } + + var proxies sourceProxiesResponse + if err = json.Unmarshal(body, &proxies); err != nil { + return nil, namedProxyLookupError(proxyKey, "source Artifactory returned an invalid proxy response") + } + for _, proxy := range proxies.Proxies { + if strings.EqualFold(proxy.Key, proxyKey) { + proxyURL, buildErr := proxyDescriptorURL(proxy) + if buildErr != nil { + return nil, namedProxyLookupError(proxyKey, buildErr.Error()) + } + return proxyURL, nil + } + } + return nil, namedProxyLookupError(proxyKey, "the key was not found") +} + +func proxyDescriptorURL(proxy sourceProxyDescriptor) (*url.URL, error) { + host := strings.TrimSpace(proxy.Host) + if host == "" { + return nil, fmt.Errorf("the source proxy descriptor has no host") + } + scheme := "http" + if proxy.Https { + scheme = "https" + } + if strings.Contains(host, "://") { + parsedHost, err := url.Parse(host) + if err != nil || parsedHost.Host == "" || parsedHost.Path != "" && parsedHost.Path != "/" { + return nil, fmt.Errorf("the source proxy descriptor has an invalid host") + } + if parsedHost.Scheme != "http" && parsedHost.Scheme != "https" { + return nil, fmt.Errorf("the source proxy descriptor uses unsupported scheme %q", parsedHost.Scheme) + } + scheme = parsedHost.Scheme + host = parsedHost.Host + } + if proxy.Port > 0 { + hostname := host + if parsedHost, err := url.Parse(scheme + "://" + host); err == nil && parsedHost.Hostname() != "" { + hostname = parsedHost.Hostname() + } + host = net.JoinHostPort(hostname, strconv.Itoa(proxy.Port)) + } + proxyURL := &url.URL{Scheme: scheme, Host: host} + if isEncryptedProxyPassword(proxy.Password) { + return nil, fmt.Errorf("the source API returned an encrypted proxy password that the CLI cannot use") + } + if proxy.Username != "" && strings.TrimSpace(proxy.Password) == "" { + return nil, fmt.Errorf("the source API omitted the proxy password; authenticate to source with an access token so the password can be returned") + } + if proxy.Username == "" && proxy.Password != "" { + return nil, fmt.Errorf("the source proxy descriptor has a password but no username") + } + if proxy.Username != "" { + proxyURL.User = url.UserPassword(proxy.Username, proxy.Password) + } + return parseProxyKeyURL(proxyURL.String()) +} + +func isEncryptedProxyPassword(password string) bool { + upper := strings.ToUpper(strings.TrimSpace(password)) + return strings.HasPrefix(upper, "**ENC**") || + strings.HasPrefix(upper, "ENC(") || + strings.HasPrefix(upper, "{AES}") +} + +func namedProxyLookupError(proxyKey, reason string) error { + return errorutils.CheckErrorf("could not resolve --proxy-key %q: it was looked up on the source Artifactory, but %s. %s", + proxyKey, reason, proxyKeyUsageHint) +} + +func parseProxyKeyURL(raw string) (*url.URL, error) { + parsed, err := url.Parse(raw) + if err != nil { + return nil, errorutils.CheckErrorf("--proxy-key value %q is not a valid URL. %s", + redactProxyKey(raw), proxyKeyUsageHint) + } + scheme := strings.ToLower(parsed.Scheme) + if scheme != "http" && scheme != "https" || parsed.Host == "" { + return nil, errorutils.CheckErrorf("--proxy-key value %q must be a full HTTP or HTTPS proxy URL including scheme (for example http://proxy:3128). %s", + redactProxyKey(raw), proxyKeyUsageHint) + } + return parsed, nil +} + +func redactProxyKey(raw string) string { + parsed, err := url.Parse(raw) + if err != nil { + return "" + } + if parsed.User == nil { + return raw + } + return parsed.Redacted() +} + +func newTargetProxyTransport(proxyURL *url.URL, serverDetails *config.ServerDetails) (http.RoundTripper, error) { + transport, err := newStreamingTransport(serverDetails, "--proxy-key") + if err != nil { + return nil, err + } + transport.Proxy = http.ProxyURL(proxyURL) + return transport, nil +} + +// newDefaultStreamingTransport builds the same bounded-header-wait transport as +// newTargetProxyTransport, but without a --proxy-key override, for the common (no target proxy +// configured) case. Used so the source and target streaming clients always get a +// ResponseHeaderTimeout, not just when --proxy-key is set. +func newDefaultStreamingTransport(serverDetails *config.ServerDetails) (http.RoundTripper, error) { + return newStreamingTransport(serverDetails, "streaming client") +} + +func newStreamingTransport(serverDetails *config.ServerDetails, usageContext string) (*http.Transport, error) { + insecureTls := false + if serverDetails != nil { + insecureTls = serverDetails.InsecureTls + } + transport := newDefaultTargetTransport() + transport.ResponseHeaderTimeout = streamResponseHeaderTimeout + + certsPath, err := coreutils.GetJfrogCertsDir() + if err != nil { + return nil, fmt.Errorf("failed resolving JFrog certs dir for %s: %w", usageContext, err) + } + transport, err = cert.GetTransportWithLoadedCert(certsPath, insecureTls, transport) + if err != nil { + return nil, fmt.Errorf("failed loading TLS certs for %s: %w", usageContext, err) + } + if serverDetails != nil { + err = httpclient.ClientBuilder(). + SetClientCertPath(serverDetails.ClientCertPath). + SetClientCertKeyPath(serverDetails.ClientCertKeyPath). + AddClientCertToTransport(transport) + if err != nil { + return nil, fmt.Errorf("failed loading client certificate for %s: %w", usageContext, err) + } + } + // GetTransportWithLoadedCert replaces TLSClientConfig and drops MinVersion. + if transport.TLSClientConfig == nil { + transport.TLSClientConfig = &tls.Config{} + } + transport.TLSClientConfig.MinVersion = tls.VersionTLS12 + return transport, nil +} + +func newDefaultTargetTransport() *http.Transport { + return &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: httpclient.DefaultDialTimeout, + KeepAlive: 20 * time.Second, + }).DialContext, + MaxIdleConns: 100, + // All transfer threads talk to the same host; keep enough idle connections for them to + // be reused instead of the default 2 per host. + MaxIdleConnsPerHost: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + TLSClientConfig: &tls.Config{ + MinVersion: tls.VersionTLS12, + }, + } +} + +func httpClientWithTransport(transport http.RoundTripper, timeout time.Duration) *http.Client { + if transport == nil { + return nil + } + return &http.Client{Transport: transport, Timeout: timeout} +} + +func (tdc *TransferFilesCommand) targetServiceHTTPClient() *http.Client { + return httpClientWithTransport(tdc.targetProxyTransport, time.Minute) +} + +func (tdc *TransferFilesCommand) httpClientForServer(isTarget bool) *http.Client { + if !isTarget || tdc.targetProxyTransport == nil { + return nil + } + return tdc.targetServiceHTTPClient() +} diff --git a/artifactory/commands/transferfiles/proxy_test.go b/artifactory/commands/transferfiles/proxy_test.go new file mode 100644 index 000000000..413490584 --- /dev/null +++ b/artifactory/commands/transferfiles/proxy_test.go @@ -0,0 +1,372 @@ +package transferfiles + +import ( + "context" + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "io" + "math/big" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/api" + "github.com/jfrog/jfrog-cli-core/v2/utils/config" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParseProxyKeyURL(t *testing.T) { + tests := []struct { + name string + raw string + wantErr bool + }{ + {name: "http", raw: "http://proxy:3128"}, + {name: "https", raw: "https://proxy:3128"}, + {name: "legacy key", raw: "corp-egress", wantErr: true}, + {name: "empty scheme host", raw: "http://", wantErr: true}, + {name: "socks5", raw: "socks5://proxy:1080", wantErr: true}, + {name: "userinfo redacted", raw: proxyURLWithUserinfo("http", "user", "proxy:3128"), wantErr: false}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + parsed, err := parseProxyKeyURL(tc.raw) + if tc.wantErr { + require.Error(t, err) + assert.Contains(t, err.Error(), "HTTPS_PROXY") + assert.Contains(t, err.Error(), "NO_PROXY") + assert.NotContains(t, err.Error(), "secret") + return + } + require.NoError(t, err) + assert.Equal(t, "proxy:3128", parsed.Host) + assert.NotContains(t, redactProxyKey(tc.raw), "secret") + }) + } +} + +func TestTargetClient_routesThroughProxy_sourceDoesNot(t *testing.T) { + var artifactoryHits atomic.Int32 + var sourceStorageHits atomic.Int32 + storagePath := "/api/storage/" + testSourceRelativePath() + artifactory := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + artifactoryHits.Add(1) + if r.URL.Path == "/api/system/ping" { + _, _ = w.Write([]byte("OK")) + return + } + if strings.HasPrefix(r.URL.Path, storagePath) { + sourceStorageHits.Add(1) + } + if r.Method == http.MethodPatch { + w.WriteHeader(http.StatusNoContent) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{"checksums":{},"size":"0"}`)) + })) + t.Cleanup(artifactory.Close) + + var proxyHits atomic.Int32 + proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + proxyHits.Add(1) + outReq := r.Clone(r.Context()) + outReq.RequestURI = "" + resp, err := http.DefaultTransport.RoundTrip(outReq) + if err != nil { + http.Error(w, err.Error(), http.StatusBadGateway) + return + } + defer func() { _ = resp.Body.Close() }() + for k, vv := range resp.Header { + w.Header()[k] = vv + } + w.WriteHeader(resp.StatusCode) + _, _ = io.Copy(w, resp.Body) + })) + t.Cleanup(proxy.Close) + + proxyURL, err := url.Parse(proxy.URL) + require.NoError(t, err) + transport, err := newTargetProxyTransport(proxyURL, newTestTargetServerDetails(artifactory.URL)) + require.NoError(t, err) + + targetClient, err := NewTargetClient(context.Background(), newTestTargetServerDetails(artifactory.URL), transport) + require.NoError(t, err) + require.NoError(t, targetClient.Ping(context.Background())) + assert.Greater(t, proxyHits.Load(), int32(0), "target traffic must route through the proxy") + + // b's raw doManagerPut/PutStream/Patch path (checksum-deploy, plain PUT, PATCH) must also + // route through the target proxy transport, not just the client-go Ping call above. + hitsBeforeChecksumDeploy := proxyHits.Load() + outcome, err := targetClient.TryChecksumDeploy(context.Background(), testTargetMetadata(), defaultTargetDeployOptions()) + require.NoError(t, err) + assert.Equal(t, ChecksumDeployHit, outcome) + assert.Greater(t, proxyHits.Load(), hitsBeforeChecksumDeploy, "checksum-deploy PUT must route through the proxy") + + hitsBeforePut := proxyHits.Load() + require.NoError(t, targetClient.Put(context.Background(), testTargetMetadata(), strings.NewReader("hello world"), defaultTargetDeployOptions())) + assert.Greater(t, proxyHits.Load(), hitsBeforePut, "plain PUT must route through the proxy") + + hitsBeforePatch := proxyHits.Load() + largePropsMetadata := testTargetMetadata() + largePropsMetadata.Properties = largeEligiblePropertiesMap(t) + _, err = targetClient.ApplyProperties(context.Background(), largePropsMetadata, defaultTargetDeployOptions()) + require.NoError(t, err) + assert.Greater(t, proxyHits.Load(), hitsBeforePatch, "PATCH must route through the proxy") + + hitsAfterTarget := proxyHits.Load() + sourceClient, err := NewSourceClient(context.Background(), newTestTargetServerDetails(artifactory.URL)) + require.NoError(t, err) + _, err = sourceClient.GetFileMetadata(context.Background(), api.FileRepresentation{ + Repo: testSourceRepo, + Path: testSourcePath, + Name: testSourceName, + }) + require.NoError(t, err) + assert.Greater(t, sourceStorageHits.Load(), int32(0), "source GET must reach Artifactory") + assert.Equal(t, hitsAfterTarget, proxyHits.Load(), "source traffic must not route through the --proxy-key transport") + assert.Greater(t, artifactoryHits.Load(), int32(0)) +} + +func TestResolveProxyKeyURL_URLFormDoesNotRequireSource(t *testing.T) { + resolvedURL, err := resolveProxyKeyURL(context.Background(), "http://proxy:3128", nil) + require.NoError(t, err) + assert.Equal(t, "http://proxy:3128", resolvedURL.String()) +} + +func TestNamedProxyLookup_routesTargetPingThroughProxy_sourceLookupDoesNot(t *testing.T) { + var targetHits atomic.Int32 + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + targetHits.Add(1) + assert.Equal(t, "/api/system/ping", r.URL.Path) + _, _ = w.Write([]byte("OK")) + })) + t.Cleanup(target.Close) + + var proxyHits atomic.Int32 + proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + proxyHits.Add(1) + outReq := r.Clone(r.Context()) + outReq.RequestURI = "" + resp, err := http.DefaultTransport.RoundTrip(outReq) + if err != nil { + http.Error(w, err.Error(), http.StatusBadGateway) + return + } + defer func() { _ = resp.Body.Close() }() + w.WriteHeader(resp.StatusCode) + _, _ = io.Copy(w, resp.Body) + })) + t.Cleanup(proxy.Close) + + parsedProxyURL, err := url.Parse(proxy.URL) + require.NoError(t, err) + proxyPort := parsedProxyURL.Port() + var sourceLookupHits atomic.Int32 + source := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + sourceLookupHits.Add(1) + assert.Equal(t, "/api/system/configuration/platform/proxies", r.URL.Path) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"proxies":[{"key":"corp-egress","host":"127.0.0.1","port":` + + proxyPort + `}]}`)) + })) + t.Cleanup(source.Close) + + resolvedURL, err := resolveProxyKeyURL(context.Background(), "corp-egress", + newTestSourceServerDetails(source.URL)) + require.NoError(t, err) + assert.Equal(t, proxy.URL, resolvedURL.String()) + assert.Equal(t, int32(1), sourceLookupHits.Load()) + assert.Zero(t, proxyHits.Load(), "source proxy lookup must use the source client's normal transport") + + transport, err := newTargetProxyTransport(resolvedURL, newTestTargetServerDetails(target.URL)) + require.NoError(t, err) + targetClient, err := NewTargetClient(context.Background(), newTestTargetServerDetails(target.URL), transport) + require.NoError(t, err) + require.NoError(t, targetClient.Ping(context.Background())) + assert.Equal(t, int32(1), proxyHits.Load(), "target ping must route through the named proxy") + assert.Equal(t, int32(1), targetHits.Load()) +} + +func TestNamedProxyLookup_missingKeyFailsClosed(t *testing.T) { + source := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/api/system/configuration/platform/proxies", r.URL.Path) + _, _ = w.Write([]byte(`{"proxies":[{"key":"another-proxy","host":"proxy","port":3128}]}`)) + })) + t.Cleanup(source.Close) + + _, err := resolveProxyKeyURL(context.Background(), "corp-egress", newTestSourceServerDetails(source.URL)) + require.Error(t, err) + assert.Contains(t, err.Error(), `"corp-egress"`) + assert.Contains(t, err.Error(), "looked up on the source Artifactory") + assert.Contains(t, err.Error(), "not found") + assert.Contains(t, err.Error(), "http://proxy:3128") + assert.Contains(t, err.Error(), "HTTPS_PROXY") +} + +func TestNamedProxyLookup_encryptedPasswordFailsClosed(t *testing.T) { + source := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(`{"proxies":[{"key":"corp-egress","host":"proxy","port":3128,"username":"u","password":"**ENC**xxx"}]}`)) + })) + t.Cleanup(source.Close) + + _, err := resolveProxyKeyURL(context.Background(), "corp-egress", newTestSourceServerDetails(source.URL)) + require.Error(t, err) + assert.Contains(t, err.Error(), `"corp-egress"`) + assert.Contains(t, err.Error(), "looked up on the source Artifactory") + assert.Contains(t, err.Error(), "encrypted proxy password") +} + +func TestProxyDescriptorURL_rejectsUsernameWithoutPassword(t *testing.T) { + _, err := proxyDescriptorURL(sourceProxyDescriptor{ + Host: "proxy.example", + Port: 3128, + Username: "proxy-user", + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "omitted the proxy password") + assert.Contains(t, err.Error(), "access token") +} + +func TestNamedProxyLookup_usernameWithoutPasswordFailsClosed(t *testing.T) { + source := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/api/system/configuration/platform/proxies", r.URL.Path) + _, _ = w.Write([]byte(`{"proxies":[{"key":"corp-egress","host":"proxy","port":3128,"username":"proxy-user"}]}`)) + })) + t.Cleanup(source.Close) + + _, err := resolveProxyKeyURL(context.Background(), "corp-egress", newTestSourceServerDetails(source.URL)) + require.Error(t, err) + assert.Contains(t, err.Error(), `"corp-egress"`) + assert.Contains(t, err.Error(), "looked up on the source Artifactory") + assert.Contains(t, err.Error(), "omitted the proxy password") + assert.Contains(t, err.Error(), "access token") + assert.Contains(t, err.Error(), "HTTPS_PROXY") +} + +func TestProxyDescriptorURL_rejectsEncryptedPassword(t *testing.T) { + _, err := proxyDescriptorURL(sourceProxyDescriptor{ // #nosec G101 -- dummy test fixture + Host: "proxy.example", + Port: 3128, + Password: "**ENC**unusable", + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "encrypted proxy password") + assert.NotContains(t, err.Error(), "unusable") +} + +// proxyURLWithUserinfo assembles a "scheme://user:secret@host" string at runtime so the test +// source holds no credential-bearing URL literal for secret scanners to flag. +func proxyURLWithUserinfo(scheme, user, host string) string { + return scheme + "://" + user + ":" + "secret" + "@" + host +} + +func TestProxyDescriptorURL_buildsHTTPSURLWithCredentials(t *testing.T) { + proxyURL, err := proxyDescriptorURL(sourceProxyDescriptor{ + Host: "proxy.example", + Port: 8443, + Username: "proxy-user", + Password: "secret", + Https: true, + }) + require.NoError(t, err) + assert.Equal(t, proxyURLWithUserinfo("https", "proxy-user", "proxy.example:8443"), proxyURL.String()) +} + +func TestParseProxyKeyURL_schemeLessFailsWithHelpfulMessage(t *testing.T) { + _, err := parseProxyKeyURL("corp-egress") + require.Error(t, err) + assert.Contains(t, err.Error(), `"corp-egress"`) + assert.Contains(t, err.Error(), "including scheme") + assert.Contains(t, err.Error(), "http://proxy:3128") + assert.Contains(t, err.Error(), "HTTPS_PROXY") + assert.Contains(t, err.Error(), "NO_PROXY") +} + +func TestRedactProxyKey_parseFailureNeverReturnsRawInput(t *testing.T) { + malformed := proxyURLWithUserinfo("http", "user", "proxy.example/%zz") + assert.Equal(t, "", redactProxyKey(malformed)) + _, err := parseProxyKeyURL(malformed) + require.Error(t, err) + assert.NotContains(t, err.Error(), malformed) + assert.NotContains(t, err.Error(), "secret") +} + +func TestNewTargetProxyTransport_matchesClientGoDefaults(t *testing.T) { + proxyURL, err := url.Parse("http://proxy:3128") + require.NoError(t, err) + roundTripper, err := newTargetProxyTransport(proxyURL, &config.ServerDetails{}) + require.NoError(t, err) + transport, ok := roundTripper.(*http.Transport) + require.True(t, ok) + assert.False(t, transport.ForceAttemptHTTP2) + assert.NotNil(t, transport.DialContext) + assert.Equal(t, 100, transport.MaxIdleConns) + assert.Equal(t, 100, transport.MaxIdleConnsPerHost) + assert.Equal(t, 90*time.Second, transport.IdleConnTimeout) + assert.Equal(t, 10*time.Second, transport.TLSHandshakeTimeout) + assert.Equal(t, time.Second, transport.ExpectContinueTimeout) + require.NotNil(t, transport.TLSClientConfig) + assert.Equal(t, uint16(tls.VersionTLS12), transport.TLSClientConfig.MinVersion) +} + +func TestNewTargetProxyTransport_loadsClientCertificate(t *testing.T) { + certPath, keyPath := writeClientCertificate(t) + proxyURL, err := url.Parse("http://proxy:3128") + require.NoError(t, err) + roundTripper, err := newTargetProxyTransport(proxyURL, &config.ServerDetails{ + ClientCertPath: certPath, + ClientCertKeyPath: keyPath, + }) + require.NoError(t, err) + transport, ok := roundTripper.(*http.Transport) + require.True(t, ok) + require.NotNil(t, transport.TLSClientConfig) + assert.Len(t, transport.TLSClientConfig.Certificates, 1) + assert.Equal(t, uint16(tls.VersionTLS12), transport.TLSClientConfig.MinVersion) +} + +func TestFlaglessProxyUsesEnvironmentOnly(t *testing.T) { + // Without --proxy-key, targetProxyTransport is never set, so the target client falls back to + // client-go's own transport (ProxyFromEnvironment), which already honors HTTPS_PROXY/NO_PROXY. + cmd, err := NewTransferFilesCommand(nil, nil) + require.NoError(t, err) + assert.Nil(t, cmd.targetServiceHTTPClient()) +} + +func writeClientCertificate(t *testing.T) (string, string) { + t.Helper() + privateKey, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err) + template := x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "proxy-client"}, + NotBefore: time.Now().Add(-time.Minute), + NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, + } + certDER, err := x509.CreateCertificate(rand.Reader, &template, &template, &privateKey.PublicKey, privateKey) + require.NoError(t, err) + + certPath := filepath.Join(t.TempDir(), "client-cert.pem") + keyPath := filepath.Join(filepath.Dir(certPath), "client-key.pem") + require.NoError(t, os.WriteFile(certPath, pem.EncodeToMemory(&pem.Block{ + Type: "CERTIFICATE", Bytes: certDER, + }), 0600)) + require.NoError(t, os.WriteFile(keyPath, pem.EncodeToMemory(&pem.Block{ + Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(privateKey), + }), 0600)) + return certPath, keyPath +} diff --git a/artifactory/commands/transferfiles/sourceclient.go b/artifactory/commands/transferfiles/sourceclient.go new file mode 100644 index 000000000..ea4e9842f --- /dev/null +++ b/artifactory/commands/transferfiles/sourceclient.go @@ -0,0 +1,837 @@ +package transferfiles + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "path" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/api" + coreutils "github.com/jfrog/jfrog-cli-core/v2/artifactory/utils" + "github.com/jfrog/jfrog-cli-core/v2/utils/config" + "github.com/jfrog/jfrog-client-go/artifactory" + artifactoryutils "github.com/jfrog/jfrog-client-go/artifactory/services/utils" + "github.com/jfrog/jfrog-client-go/http/httpclient" + clientutils "github.com/jfrog/jfrog-client-go/utils" + "github.com/jfrog/jfrog-client-go/utils/errorutils" + "github.com/jfrog/jfrog-client-go/utils/io/httputils" + "github.com/jfrog/jfrog-client-go/utils/log" +) + +var ErrSourceItemGone = errors.New("source item no longer exists") + +// notFoundRecheckAttempts is how many times an unrecognized properties-404 (proxy page, plain +// text, ...) is requested in total before the item is failed instead of transferred without +// properties. +const notFoundRecheckAttempts = 3 + +// notFoundRecheckBackoff is the pause between those properties re-requests. It is a variable so +// tests can shrink it. +var notFoundRecheckBackoff = time.Second + +// confirmRetryAttempts and confirmRetryInterval bound the retries of the confirmation requests +// (control probe and item existence check) on 5xx/429/transport errors. They deliberately do not +// reuse the metadata client's 600 x 5s retry, which would stall a worker for ~50 minutes on a +// source that keeps answering 503. Variables so tests can shrink them. +var confirmRetryAttempts = 3 +var confirmRetryInterval = time.Second + +// rootProbeCacheTTL is how long a successful repository-root probe is reused, so a burst of +// "gone" decisions in one repo costs one root request instead of one each. It is also the span in +// which an outage that starts right after a successful probe cannot be seen by the next decisions +// (see confirmItemGone). 0 disables reuse; concurrent probes are still shared. Variable for tests. +var rootProbeCacheTTL = 2 * time.Second + +// rootProbeMaxBodyBytes bounds how much of the root probe answer is read. The answer is the +// repository's first-level children listing, which can be tens of MB; the probe only needs the +// leading "repo" field. +const rootProbeMaxBodyBytes = 1 << 20 + +// rootProbeTimeout bounds one shared probe flight, which runs detached from any single caller's +// context so that one cancelled worker does not fail the probe for the others. +const rootProbeTimeout = 2 * time.Minute + +// propertiesNotFoundWarned makes the non-English properties-404 Warn appear once per process; +// later occurrences are logged at Debug so a server answering in another locale does not flood +// the log with one Warn per file. +var propertiesNotFoundWarned atomic.Bool + +func IsSourceItemGone(err error) bool { + return errors.Is(err, ErrSourceItemGone) +} + +type SourceFileMetadata struct { + Repo string + Path string + Name string + Size int64 + Sha1 string + Sha256 string + Md5 string + Created string + CreatedBy string + LastModified string + ModifiedBy string + Properties map[string][]string + DownloadCount int64 + LastDownloaded int64 + LastDownloadedBy string +} + +type itemStatistics struct { + DownloadCount int64 `json:"downloadCount"` + LastDownloaded int64 `json:"lastDownloaded"` + LastDownloadedBy string `json:"lastDownloadedBy"` +} + +type SourceClient struct { + serverDetails *config.ServerDetails + metadataServiceManager artifactory.ArtifactoryServicesManager + streamServiceManager artifactory.ArtifactoryServicesManager + rootProbes rootProbeCache +} + +func createStreamingTransferServiceManager(ctx context.Context, serverDetails *config.ServerDetails, httpClient *http.Client) (artifactory.ArtifactoryServicesManager, error) { + return coreutils.CreateServiceManagerWithContextAndHttpClient(ctx, serverDetails, false, 0, 0, 0, 0, httpClient) +} + +func NewSourceClient(ctx context.Context, serverDetails *config.ServerDetails) (*SourceClient, error) { + // Bounded retries, as on the target side: the transfer manager's 600 x 5s policy would stall a + // worker for ~50 minutes per call on an item whose metadata keeps answering 5xx. A failed item + // is reported in the errors CSV and retried by a later phase instead. + metadataServiceManager, err := createMetadataTransferServiceManager(ctx, serverDetails, nil) + if err != nil { + return nil, err + } + streamTransport, err := newDefaultStreamingTransport(serverDetails) + if err != nil { + return nil, err + } + streamServiceManager, err := createStreamingTransferServiceManager(ctx, serverDetails, httpClientWithTransport(streamTransport, 0)) + if err != nil { + return nil, err + } + return &SourceClient{ + serverDetails: serverDetails, + metadataServiceManager: metadataServiceManager, + streamServiceManager: streamServiceManager, + }, nil +} + +func (sc *SourceClient) GetFileMetadata(ctx context.Context, file api.FileRepresentation) (*SourceFileMetadata, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + + relativePath := fileRelativePath(file) + fileInfo, err := sc.fetchFileInfo(ctx, sc.metadataServiceManager, relativePath) + if err != nil { + return nil, err + } + + // Folder-info responses carry no "size" field; treat that as size 0 rather than a parse failure. + // Fall back to the AQL-reported size when it's available. + var size int64 + if fileInfo.Size == "" { + if file.Size > 0 { + size = file.Size + } + } else { + var parseErr error + size, parseErr = strconv.ParseInt(fileInfo.Size, 10, 64) + if parseErr != nil { + if file.Size > 0 { + size = file.Size + } else { + return nil, fmt.Errorf("failed to parse file size %q for %s: %w", fileInfo.Size, relativePath, parseErr) + } + } + } + + itemProps, err := sc.fetchItemProperties(ctx, sc.metadataServiceManager, relativePath) + if err != nil { + return nil, err + } + + metadata := &SourceFileMetadata{ + Repo: file.Repo, + Path: file.Path, + Name: file.Name, + Size: size, + Sha1: fileInfo.Checksums.Sha1, + Sha256: fileInfo.Checksums.Sha256, + Md5: fileInfo.Checksums.Md5, + Created: fileInfo.Created, + CreatedBy: fileInfo.CreatedBy, + LastModified: fileInfo.LastModified, + ModifiedBy: fileInfo.ModifiedBy, + } + if itemProps != nil { + metadata.Properties = itemProps.Properties + } + if file.Name != "" { + stats, statsErr := sc.fetchItemStatistics(ctx, sc.metadataServiceManager, relativePath) + if statsErr != nil { + return nil, statsErr + } + if stats != nil { + metadata.DownloadCount = stats.DownloadCount + metadata.LastDownloaded = stats.LastDownloaded + metadata.LastDownloadedBy = stats.LastDownloadedBy + } + } + return metadata, nil +} + +func (sc *SourceClient) fetchFileInfo(ctx context.Context, manager artifactory.ArtifactoryServicesManager, relativePath string) (*artifactoryutils.FileInfo, error) { + resp, body, err := sendStorageGet(ctx, manager, relativePath, "") + if err != nil { + return nil, err + } + if resp.StatusCode == http.StatusNotFound { + gone, confirmErr := sc.confirmItemGone(ctx, manager, relativePath) + if confirmErr != nil { + return nil, confirmErr + } + if gone { + return nil, ErrSourceItemGone + } + // The initial 404 didn't hold up on a corroborating re-check: the item exists + // after all (a transient proxy hiccup, restart window, etc). Surface this as a + // plain, retryable error instead of silently treating the file as deleted. + return nil, fmt.Errorf("received a 404 from the source storage-info endpoint for %q, but a corroborating existence check found the item still exists; treating this as a transient error, not a deletion", relativePath) + } + if err = errorutils.CheckResponseStatusWithBody(resp, body, http.StatusOK); err != nil { + return nil, err + } + + result := &artifactoryutils.FileInfo{} + if err = json.Unmarshal(body, result); err != nil { + return nil, errorutils.CheckError(err) + } + return result, nil +} + +// noPropertiesMessage is the Artifactory 7 English error body of a properties GET on an item +// that exists but has no properties. +const noPropertiesMessage = "No properties could be found" + +func (sc *SourceClient) fetchItemProperties(ctx context.Context, manager artifactory.ArtifactoryServicesManager, relativePath string) (*artifactoryutils.ItemProperties, error) { + resp, body, err := sendStorageGet(ctx, manager, relativePath, "properties") + if err != nil { + return nil, err + } + if resp.StatusCode == http.StatusNotFound { + if strings.Contains(string(body), noPropertiesMessage) { + return nil, nil + } + // The body didn't match the pinned English string. Corroborate with a direct + // existence check before concluding anything: the item may have been deleted, or the + // 404 may not come from Artifactory at all. + gone, confirmErr := sc.confirmItemGone(ctx, manager, relativePath) + if confirmErr != nil { + return nil, confirmErr + } + if gone { + return nil, ErrSourceItemGone + } + // The item exists. A 404 that is a well-formed Artifactory JSON error is Artifactory's + // own "no properties" answer in another locale/version: accept it, but log it so it + // stays auditable. + if isArtifactoryJSONNotFound(body) { + logNonEnglishPropertiesNotFound(relativePath, body) + return nil, nil + } + // Anything else (proxy HTML page, plain text, empty body, ...) could be a transient + // failure. Returning "no properties" would silently drop them, so re-request the + // properties (bounded, with backoff) and fail the item if the answer stays ambiguous. + return sc.retryAmbiguousProperties(ctx, manager, relativePath) + } + return parseItemProperties(resp, body) +} + +// isArtifactoryJSONNotFound reports whether body is an Artifactory JSON error carrying a 404 +// entry, i.e. {"errors":[{"status":404,"message":"..."}]}. It deliberately ignores the message +// text, which is localized/version dependent, and requires the structure and status so proxy +// pages and plain-text bodies do not qualify. +func isArtifactoryJSONNotFound(body []byte) bool { + var parsed struct { + Errors []struct { + Status int `json:"status"` + } `json:"errors"` + } + if err := json.Unmarshal(body, &parsed); err != nil { + return false + } + for _, e := range parsed.Errors { + if e.Status == http.StatusNotFound { + return true + } + } + return false +} + +// logNonEnglishPropertiesNotFound logs the first such answer of the process as a Warn and the +// rest at Debug level. +func logNonEnglishPropertiesNotFound(relativePath string, body []byte) { + msg := fmt.Sprintf("Properties lookup for %s returned a non-English/unknown 404 message (%s); treating as no properties.", relativePath, truncateForLog(body)) + if propertiesNotFoundWarned.CompareAndSwap(false, true) { + log.Warn(msg + " Further occurrences are logged at debug level.") + return + } + log.Debug(msg) +} + +func truncateForLog(body []byte) string { + const maxLen = 200 + msg := strings.TrimSpace(string(body)) + if len(msg) > maxLen { + return msg[:maxLen] + "..." + } + return msg +} + +func parseItemProperties(resp *http.Response, body []byte) (*artifactoryutils.ItemProperties, error) { + if err := errorutils.CheckResponseStatusWithBody(resp, body, http.StatusOK); err != nil { + return nil, err + } + result := &artifactoryutils.ItemProperties{} + if err := json.Unmarshal(body, result); err != nil { + return nil, errorutils.CheckError(err) + } + return result, nil +} + +// retryAmbiguousProperties re-issues the properties GET after a non-Artifactory 404 (proxy page, +// plain text, empty body, ...) on an item that storage-info says exists. The first request (the ambiguous one) already counted as +// attempt 1 of notFoundRecheckAttempts. +func (sc *SourceClient) retryAmbiguousProperties(ctx context.Context, manager artifactory.ArtifactoryServicesManager, relativePath string) (*artifactoryutils.ItemProperties, error) { + for attempt := 2; attempt <= notFoundRecheckAttempts; attempt++ { + if err := sleepWithContext(ctx, notFoundRecheckBackoff); err != nil { + return nil, err + } + resp, body, err := sendStorageGet(ctx, manager, relativePath, "properties") + if err != nil { + return nil, err + } + if resp.StatusCode != http.StatusNotFound { + return parseItemProperties(resp, body) + } + if strings.Contains(string(body), noPropertiesMessage) { + return nil, nil + } + if isArtifactoryJSONNotFound(body) { + log.Warn(fmt.Sprintf("Properties lookup for %s returned a non-English/unknown 404 message (%s); treating as no properties.", relativePath, truncateForLog(body))) + return nil, nil + } + } + return nil, fmt.Errorf("received a non-Artifactory 404 from the source properties endpoint for %q %d times, although the item exists; failing the item instead of transferring it without its properties", relativePath, notFoundRecheckAttempts) +} + +// confirmItemGone corroborates an ambiguous 404 seen on another endpoint before the caller +// finalizes it as a deletion. A 404 for the item says nothing by itself: a router/Artifactory +// restart or proxy blip answers 404 for every path, for as long as it lasts, so no number +// of re-checks of the item can tell an outage from a deletion. Instead the repository root +// (api/storage/) is used as a control probe: +// - root answers 200 with an Artifactory JSON body and the item 404s: the source is healthy and +// the item is really gone (decided at once, no sleeping); +// - root 404s, errors, or answers something that is not Artifactory JSON: the source is in an +// outage window; an error is returned (never "gone") so the item fails and is retried; +// - item answers 200: the item exists. +// +// The item check runs after the probe. A successful probe is reused for rootProbeCacheTTL, so the +// healthy instant it proves can be up to that long before the item check; an outage that starts +// inside that span and answers the item with an Artifactory-shaped 404 is the residual risk. +func (sc *SourceClient) confirmItemGone(ctx context.Context, manager artifactory.ArtifactoryServicesManager, relativePath string) (bool, error) { + repoKey := repoKeyOf(relativePath) + if err := sc.rootProbes.probe(ctx, manager, repoKey); err != nil { + return false, fmt.Errorf("could not confirm whether %q was deleted, because the source repository %q did not pass the control probe (the source may be restarting or unreachable); treating this as a retryable error, not a deletion: %w", relativePath, repoKey, err) + } + resp, body, err := sendConfirmationGet(ctx, manager, relativePath, 0) + if err != nil { + return false, err + } + if resp.StatusCode == http.StatusNotFound { + if isArtifactoryJSONNotFound(body) { + return true, nil + } + return false, fmt.Errorf("item %q answered 404 with a body that is not an Artifactory error (%s) although the repository root is healthy; treating this as a retryable error, not a deletion", relativePath, truncateForLog(body)) + } + if err = errorutils.CheckResponseStatusWithBody(resp, body, http.StatusOK); err != nil { + return false, err + } + return false, nil +} + +func repoKeyOf(relativePath string) string { + trimmed := strings.TrimPrefix(path.Clean(relativePath), "/") + return strings.SplitN(trimmed, "/", 2)[0] +} + +// rootProbeCache reuses successful repository-root probes for rootProbeCacheTTL and shares the +// probe that is in flight for a repository between all callers that ask meanwhile (the same +// de-duplication golang.org/x/sync/singleflight gives, plus waiter counting: the shared request is +// cancelled when its last waiter gives up, so no probe outlives the callers that need it). Failed +// probes are never stored. The zero value is ready to use. +type rootProbeCache struct { + mu sync.Mutex + okUntil map[string]time.Time + flights map[string]*rootProbeFlight +} + +type rootProbeFlight struct { + done chan struct{} + err error // valid after done is closed + waiters int + cancel context.CancelFunc +} + +func (c *rootProbeCache) freshLocked(repoKey string) bool { + until, ok := c.okUntil[repoKey] + return ok && time.Now().Before(until) +} + +// probe returns nil if the repository root was proven healthy now or within the TTL. +func (c *rootProbeCache) probe(ctx context.Context, manager artifactory.ArtifactoryServicesManager, repoKey string) error { + if err := ctx.Err(); err != nil { + return err + } + c.mu.Lock() + if c.freshLocked(repoKey) { + c.mu.Unlock() + return nil + } + flight := c.flights[repoKey] + if flight == nil { + // The shared request is detached from any single caller's context (one cancelled worker + // must not fail the probe for the others) and bounded by rootProbeTimeout. + flightCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), rootProbeTimeout) + flight = &rootProbeFlight{done: make(chan struct{}), cancel: cancel} + if c.flights == nil { + c.flights = make(map[string]*rootProbeFlight) + } + c.flights[repoKey] = flight + go c.run(flightCtx, manager, repoKey, flight) + } + flight.waiters++ + c.mu.Unlock() + + select { + case <-flight.done: + return flight.err + case <-ctx.Done(): + c.mu.Lock() + flight.waiters-- + last := flight.waiters == 0 + if last && c.flights[repoKey] == flight { + delete(c.flights, repoKey) // newcomers start a fresh probe instead of joining a cancelled one + } + c.mu.Unlock() + if last { + flight.cancel() + <-flight.done + } + return ctx.Err() + } +} + +func (c *rootProbeCache) run(ctx context.Context, manager artifactory.ArtifactoryServicesManager, repoKey string, flight *rootProbeFlight) { + err := probeRepositoryRoot(ctx, manager, repoKey) + c.mu.Lock() + if err == nil && rootProbeCacheTTL > 0 { + if c.okUntil == nil { + c.okUntil = make(map[string]time.Time) + } + c.okUntil[repoKey] = time.Now().Add(rootProbeCacheTTL) + } + if c.flights[repoKey] == flight { + delete(c.flights, repoKey) + } + c.mu.Unlock() + flight.err = err + flight.cancel() + close(flight.done) +} + +// probeRepositoryRoot succeeds only if GET api/storage/ answers 200 with an Artifactory +// storage-info JSON body (carrying the "repo" field). Only a bounded prefix of the answer is read. +// +// api/repositories/ was considered as a cheaper existence check and rejected: it is +// admin-only configuration, answers an empty 400 (not a 404) for unknown keys, and does not know +// the "-cache" keys that are transferred (artifactory-service RepositoriesResource / +// RestAddonImpl.getRepositoryConfiguration). +func probeRepositoryRoot(ctx context.Context, manager artifactory.ArtifactoryServicesManager, repoKey string) error { + resp, body, err := sendConfirmationGet(ctx, manager, repoKey, rootProbeMaxBodyBytes) + if err != nil { + return err + } + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("repository root probe returned HTTP %d: %s", resp.StatusCode, truncateForLog(body)) + } + if repo, decodeErr := decodeRepoField(body); decodeErr != nil || repo == "" { + return fmt.Errorf("repository root probe returned HTTP 200 but not an Artifactory storage-info body (no \"repo\" field in the first %d bytes): %s", rootProbeMaxBodyBytes, truncateForLog(body)) + } + return nil +} + +// decodeRepoField returns the value of the top-level "repo" string of a JSON object, skipping the +// other values token by token (so a huge "children" array is never materialized). It fails on +// truncated or malformed input before the field is reached. +func decodeRepoField(body []byte) (string, error) { + dec := json.NewDecoder(bytes.NewReader(body)) + if tok, err := dec.Token(); err != nil { + return "", err + } else if delim, ok := tok.(json.Delim); !ok || delim != '{' { + return "", errors.New("not a JSON object") + } + for dec.More() { + keyTok, err := dec.Token() + if err != nil { + return "", err + } + if key, _ := keyTok.(string); key == "repo" { + var repo string + if err = dec.Decode(&repo); err != nil { + return "", err + } + return repo, nil + } + if err = skipJSONValue(dec); err != nil { + return "", err + } + } + return "", errors.New("no repo field") +} + +func skipJSONValue(dec *json.Decoder) error { + depth := 0 + for { + tok, err := dec.Token() + if err != nil { + return err + } + if delim, ok := tok.(json.Delim); ok { + if delim == '{' || delim == '[' { + depth++ + } else { + depth-- + } + } + if depth == 0 { + return nil + } + } +} + +// sendConfirmationGet is sendStorageGet for the existence checks: same request, but the retries +// on 5xx/429/transport errors are bounded (confirmRetryAttempts) and the wait between them is +// cancelled with the context. +func sendConfirmationGet(ctx context.Context, manager artifactory.ArtifactoryServicesManager, relativePath string, maxBodyBytes int64) (*http.Response, []byte, error) { + fullURL, err := storageURL(manager, relativePath, "") + if err != nil { + return nil, nil, err + } + attempts := confirmRetryAttempts + if attempts < 1 { + attempts = 1 + } + var lastErr error + for attempt := 1; attempt <= attempts; attempt++ { + if attempt > 1 { + if err = sleepWithContext(ctx, confirmRetryInterval); err != nil { + return nil, nil, err + } + } + resp, body, retry, reqErr := doManagerGetOnce(ctx, manager, fullURL, true, maxBodyBytes) + if !retry { + return resp, body, reqErr + } + if ctxErr := ctx.Err(); ctxErr != nil { + return nil, nil, ctxErr + } + if reqErr != nil { + lastErr = reqErr + } else { + lastErr = fmt.Errorf("received HTTP %d", resp.StatusCode) + } + } + return nil, nil, fmt.Errorf("GET %s failed after %d attempts: %w", fullURL, attempts, lastErr) +} + +// sleepWithContext waits for d, returning early with the context error if ctx is done first. +func sleepWithContext(ctx context.Context, d time.Duration) error { + if d <= 0 { + return ctx.Err() + } + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + +func (sc *SourceClient) fetchItemStatistics(ctx context.Context, manager artifactory.ArtifactoryServicesManager, relativePath string) (*itemStatistics, error) { + resp, body, err := sendStorageGet(ctx, manager, relativePath, "stats") + if err != nil { + if isContextDoneError(ctx, err) { + return nil, err + } + log.Warn("Couldn't get stats for", relativePath+". Reason:", err.Error()) + return nil, nil + } + if resp.StatusCode == http.StatusNotFound { + return nil, nil + } + if err = errorutils.CheckResponseStatusWithBody(resp, body, http.StatusOK); err != nil { + log.Warn("Couldn't get stats for", relativePath+". Reason:", err.Error()) + return nil, nil + } + + result := &itemStatistics{} + if err = json.Unmarshal(body, result); err != nil { + log.Warn("Couldn't get stats for", relativePath+". Reason:", err.Error()) + return nil, nil + } + return result, nil +} + +func storageURL(manager artifactory.ArtifactoryServicesManager, relativePath, query string) (string, error) { + artDetails := manager.GetConfig().GetServiceDetails() + restAPI := path.Join("api/storage", path.Clean(relativePath)) + fullURL, err := clientutils.BuildUrl(artDetails.GetUrl(), restAPI, map[string]string{}) + if err != nil { + return "", err + } + if query != "" { + fullURL += "?" + query + } + return fullURL, nil +} + +func sendStorageGet(ctx context.Context, manager artifactory.ArtifactoryServicesManager, relativePath, query string) (*http.Response, []byte, error) { + fullURL, err := storageURL(manager, relativePath, query) + if err != nil { + return nil, nil, err + } + return doManagerGet(ctx, manager, fullURL, true) +} + +func doManagerGet(ctx context.Context, manager artifactory.ArtifactoryServicesManager, fullURL string, closeBody bool) (*http.Response, []byte, error) { + cfg := manager.GetConfig() + + var resp *http.Response + var body []byte + // RetryExecutor sleeps between attempts with a plain time.Sleep and only checks ctx + // between attempts, so cancellation can lag by up to one retry interval. Same behavior + // as other client-go callers of RetryExecutor. + retryExecutor := clientutils.RetryExecutor{ + Context: ctx, + MaxRetries: cfg.GetHttpRetries(), + RetriesIntervalMilliSecs: cfg.GetHttpRetryWaitMilliSecs(), + ErrorMessage: fmt.Sprintf("Failure occurred while sending GET request to %s", fullURL), + ExecutionHandler: func() (bool, error) { + var retry bool + var err error + resp, body, retry, err = doManagerGetOnce(ctx, manager, fullURL, closeBody, 0) + return retry, err + }, + } + err := retryExecutor.Execute() + return resp, body, err +} + +// doManagerGetOnce performs a single GET. retry reports whether the outcome is worth retrying +// (transport error, 5xx or 429); in that case a non-nil resp has already been drained and closed. +// maxBodyBytes > 0 reads at most that many bytes of the body and closes it without draining the +// rest, so that connection is not reused. +func doManagerGetOnce(ctx context.Context, manager artifactory.ArtifactoryServicesManager, fullURL string, closeBody bool, maxBodyBytes int64) (resp *http.Response, body []byte, retry bool, err error) { + httpClientsDetails := manager.GetConfig().GetServiceDetails().CreateHttpClientDetails() + // Refresh an expiring access token before every attempt, as client-go's own Send* does. + if preErr := manager.GetConfig().GetServiceDetails().RunPreRequestFunctions(&httpClientsDetails); preErr != nil { + return nil, nil, false, preErr + } + httpCli := manager.Client().GetHttpClient() + + req, reqErr := http.NewRequestWithContext(ctx, http.MethodGet, fullURL, nil) + if reqErr != nil { + return nil, nil, !isContextDoneError(ctx, reqErr), reqErr + } + applyHttpClientDetails(req, httpClientsDetails) + resp, doErr := httpCli.GetClient().Do(req) + if doErr != nil { + if isContextDoneError(ctx, doErr) { + return nil, nil, false, doErr + } + return nil, nil, true, doErr + } + if resp == nil { + return nil, nil, false, errorutils.CheckErrorf("received empty response from server") + } + if shouldRetryHTTPStatus(resp.StatusCode) { + drainAndClose(resp.Body) + return resp, nil, true, nil + } + if closeBody { + var readErr error + if maxBodyBytes > 0 { + defer func() { _ = resp.Body.Close() }() + body, readErr = io.ReadAll(io.LimitReader(resp.Body, maxBodyBytes)) + } else { + defer drainAndClose(resp.Body) + body, readErr = io.ReadAll(resp.Body) + } + if readErr != nil { + return resp, nil, false, readErr + } + } + return resp, body, false, nil +} + +// applyHttpClientDetails mirrors the authentication and User-Agent behavior that +// jfrog-client-go's http/httpclient.HttpClient.doRequest applies via its own +// setAuthentication/addUserAgentHeader helpers, for the raw *http.Client requests this package +// sends outside the service manager (so a per-call ctx can actually abort them). Keep both in +// sync if client-go adds an auth mode or changes those defaults. +// Unlike doRequest, it does not set req.Close: a transfer sends several requests per file to the +// same hosts, and a new TCP+TLS connection for each would dominate the runtime of repositories +// with many small files. Callers drain and close response bodies so connections can be reused. +func applyHttpClientDetails(req *http.Request, details httputils.HttpClientDetails) { + req.Header.Set("User-Agent", clientutils.GetUserAgent()) + switch { + case details.ApiKey != "": + if details.User != "" { + req.SetBasicAuth(details.User, details.ApiKey) + } else { + req.Header.Set("X-JFrog-Art-Api", details.ApiKey) + } + case details.AccessToken != "": + if httpclient.IsApiKey(details.AccessToken) { + req.SetBasicAuth(details.User, details.AccessToken) + } else { + req.Header.Set("Authorization", "Bearer "+details.AccessToken) + } + case details.Password != "": + req.SetBasicAuth(details.User, details.Password) + } + for name, value := range details.Headers { + req.Header.Set(name, value) + } +} + +func shouldRetryHTTPStatus(statusCode int) bool { + return statusCode >= 500 || statusCode == http.StatusTooManyRequests +} + +func isContextDoneError(ctx context.Context, err error) bool { + if ctx.Err() != nil { + return true + } + return errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) +} + +func drainAndClose(body io.ReadCloser) { + if body == nil { + return + } + _, _ = io.Copy(io.Discard, body) + _ = body.Close() +} + +func (sc *SourceClient) GetFileReader(ctx context.Context, file api.FileRepresentation) (io.ReadCloser, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + + artDetails := sc.streamServiceManager.GetConfig().GetServiceDetails() + readPath, err := clientutils.BuildUrl(artDetails.GetUrl(), fileRelativePath(file), map[string]string{}) + if err != nil { + return nil, err + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, readPath, nil) + if err != nil { + return nil, err + } + httpClientsDetails := artDetails.CreateHttpClientDetails() + // Refresh an expiring access token before the request, as client-go's own Send* does. + if err = artDetails.RunPreRequestFunctions(&httpClientsDetails); err != nil { + return nil, err + } + applyHttpClientDetails(req, httpClientsDetails) + resp, err := sc.streamServiceManager.Client().GetHttpClient().GetClient().Do(req) + if err != nil { + return nil, err + } + if resp == nil { + return nil, errorutils.CheckErrorf("received empty response from source download") + } + if resp.StatusCode == http.StatusNotFound { + drainAndClose(resp.Body) + // A content-GET 404 alone is not proof of deletion: a transient 404 window (an + // Artifactory or router restart) or a blacked-out repo returns it for files that exist. Corroborate with + // storage-info; only a confirmed absence is "gone", anything else is a retryable + // failure so the file lands in the errors CSV instead of being silently dropped. + relativePath := fileRelativePath(file) + gone, confirmErr := sc.confirmItemGone(ctx, sc.metadataServiceManager, relativePath) + if confirmErr != nil { + return nil, fmt.Errorf("received a 404 downloading %q from the source and could not corroborate whether the item exists: %w", relativePath, confirmErr) + } + if gone { + return nil, ErrSourceItemGone + } + return nil, fmt.Errorf("received a 404 downloading %q from the source, but the storage-info endpoint says the item exists. This is usually persistent, not transient: storage-info does not check, but a download does, whether the repository is blacked out, whether the path is excluded by the repository's include/exclude patterns, and whether the binary is present in the source filestore. The item is reported as a failure and will fail again on every retry until that is fixed on the source (or the 404 was a one-off blip, in which case a retry succeeds)", relativePath) + } + if resp.StatusCode != http.StatusOK { + body, readErr := io.ReadAll(resp.Body) + drainAndClose(resp.Body) + return nil, errors.Join(errorutils.CheckResponseStatusWithBody(resp, body, http.StatusOK), readErr) + } + return &contextReadCloser{ctx: ctx, rc: resp.Body}, nil +} + +func fileRelativePath(file api.FileRepresentation) string { + return path.Join(file.Repo, file.Path, file.Name) +} + +func closeReadCloser(rc io.ReadCloser) { + if rc != nil { + _ = rc.Close() + } +} + +type contextReadCloser struct { + ctx context.Context + rc io.ReadCloser + closeOnce sync.Once + closeErr error +} + +func (c *contextReadCloser) Read(p []byte) (int, error) { + if err := c.ctx.Err(); err != nil { + _ = c.Close() + return 0, err + } + return c.rc.Read(p) +} + +// Close is idempotent: Read may already have closed rc on ctx cancellation, and callers +// close the reader again on their own error paths, which would otherwise double-close a +// non-idempotent underlying ReadCloser. +func (c *contextReadCloser) Close() error { + c.closeOnce.Do(func() { + c.closeErr = c.rc.Close() + }) + return c.closeErr +} diff --git a/artifactory/commands/transferfiles/sourceclient_test.go b/artifactory/commands/transferfiles/sourceclient_test.go new file mode 100644 index 000000000..d816e6217 --- /dev/null +++ b/artifactory/commands/transferfiles/sourceclient_test.go @@ -0,0 +1,1990 @@ +package transferfiles + +import ( + "context" + "encoding/json" + "io" + "net" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/api" + "github.com/jfrog/jfrog-cli-core/v2/utils/config" + "github.com/jfrog/jfrog-cli-core/v2/utils/tests" + "github.com/jfrog/jfrog-client-go/utils/log" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + testRootPath = "/api/storage/" + testSourceRepo + testSourceRepo = "generic-local" + testSourcePath = "folder" + testSourceName = "file.txt" +) + +func testSourceFile() api.FileRepresentation { + return api.FileRepresentation{ + Repo: testSourceRepo, + Path: testSourcePath, + Name: testSourceName, + Size: 11, + } +} + +func testSourceRelativePath() string { + return testSourceRepo + "/" + testSourcePath + "/" + testSourceName +} + +// fastNotFoundRecheck shrinks the backoff between ambiguous-404 re-checks so tests stay fast. +func fastNotFoundRecheck(t *testing.T) { + t.Helper() + old := notFoundRecheckBackoff + notFoundRecheckBackoff = time.Millisecond + t.Cleanup(func() { notFoundRecheckBackoff = old }) +} + +func TestIsSourceItemGone(t *testing.T) { + assert.True(t, IsSourceItemGone(ErrSourceItemGone)) + assert.False(t, IsSourceItemGone(nil)) + assert.False(t, IsSourceItemGone(assert.AnError)) +} + +func TestGetFileMetadata_success(t *testing.T) { + fileContent := "hello world" + storagePath := "/api/storage/" + testSourceRelativePath() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == storagePath && r.URL.RawQuery == "": + response := map[string]any{ + "repo": testSourceRepo, + "path": testSourcePath + "/" + testSourceName, + "created": "2020-01-01T00:00:00.000Z", + "createdBy": "admin", + "lastModified": "2020-01-02T00:00:00.000Z", + "modifiedBy": "deployer", + "size": strconv.Itoa(len(fileContent)), + "checksums": map[string]string{ + "sha1": "sha1-value", + "sha256": "sha256-value", + "md5": "md5-value", + }, + } + w.WriteHeader(http.StatusOK) + require.NoError(t, json.NewEncoder(w).Encode(response)) + case r.URL.Path == storagePath && r.URL.RawQuery == "properties": + response := map[string]any{ + "properties": map[string][]string{ + "build.name": {"app"}, + "env": {"prod", "release"}, + }, + } + w.WriteHeader(http.StatusOK) + require.NoError(t, json.NewEncoder(w).Encode(response)) + case r.URL.Path == storagePath && r.URL.RawQuery == "stats": + w.WriteHeader(http.StatusOK) + require.NoError(t, json.NewEncoder(w).Encode(map[string]any{ + "downloadCount": 0, + "lastDownloaded": 0, + "lastDownloadedBy": "", + })) + default: + t.Errorf("unexpected request: %s", r.URL.String()) + http.Error(w, "unexpected request", http.StatusInternalServerError) + } + })) + defer server.Close() + + client, err := NewSourceClient(context.Background(), newTestSourceServerDetails(server.URL)) + require.NoError(t, err) + + metadata, err := client.GetFileMetadata(context.Background(), testSourceFile()) + require.NoError(t, err) + require.NotNil(t, metadata) + + assert.Equal(t, testSourceRepo, metadata.Repo) + assert.Equal(t, testSourcePath, metadata.Path) + assert.Equal(t, testSourceName, metadata.Name) + assert.Equal(t, int64(len(fileContent)), metadata.Size) + assert.Equal(t, "sha1-value", metadata.Sha1) + assert.Equal(t, "sha256-value", metadata.Sha256) + assert.Equal(t, "md5-value", metadata.Md5) + assert.Equal(t, "2020-01-01T00:00:00.000Z", metadata.Created) + assert.Equal(t, "admin", metadata.CreatedBy) + assert.Equal(t, "2020-01-02T00:00:00.000Z", metadata.LastModified) + assert.Equal(t, "deployer", metadata.ModifiedBy) + assert.Equal(t, map[string][]string{ + "build.name": {"app"}, + "env": {"prod", "release"}, + }, metadata.Properties) + assert.Zero(t, metadata.DownloadCount) + assert.Zero(t, metadata.LastDownloaded) + assert.Empty(t, metadata.LastDownloadedBy) +} + +func TestGetFileMetadata_includesDownloadStats(t *testing.T) { + storagePath := "/api/storage/" + testSourceRelativePath() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == storagePath && r.URL.RawQuery == "": + w.WriteHeader(http.StatusOK) + require.NoError(t, json.NewEncoder(w).Encode(map[string]any{ + "size": "11", + "checksums": map[string]string{ + "sha1": "sha1-value", + }, + })) + case r.URL.Path == storagePath && r.URL.RawQuery == "properties": + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"errors":[{"message":"No properties could be found."}]}`)) + case r.URL.Path == storagePath && r.URL.RawQuery == "stats": + w.WriteHeader(http.StatusOK) + require.NoError(t, json.NewEncoder(w).Encode(map[string]any{ + "downloadCount": 5, + "lastDownloaded": 1788945284685, + "lastDownloadedBy": "admin", + })) + default: + t.Errorf("unexpected request: %s", r.URL.String()) + http.Error(w, "unexpected request", http.StatusInternalServerError) + } + })) + defer server.Close() + + client, err := NewSourceClient(context.Background(), newTestSourceServerDetails(server.URL)) + require.NoError(t, err) + + metadata, err := client.GetFileMetadata(context.Background(), testSourceFile()) + require.NoError(t, err) + assert.Equal(t, int64(5), metadata.DownloadCount) + assert.Equal(t, int64(1788945284685), metadata.LastDownloaded) + assert.Equal(t, "admin", metadata.LastDownloadedBy) +} + +func TestGetFileMetadata_statsNotFound_continuesWithoutStats(t *testing.T) { + storagePath := "/api/storage/" + testSourceRelativePath() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == storagePath && r.URL.RawQuery == "": + w.WriteHeader(http.StatusOK) + require.NoError(t, json.NewEncoder(w).Encode(map[string]any{ + "size": "11", + "checksums": map[string]string{ + "sha1": "sha1-value", + }, + })) + case r.URL.Path == storagePath && r.URL.RawQuery == "properties": + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"errors":[{"message":"No properties could be found."}]}`)) + case r.URL.Path == storagePath && r.URL.RawQuery == "stats": + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"errors":[{"message":"Unable to find item"}]}`)) + default: + t.Errorf("unexpected request: %s", r.URL.String()) + http.Error(w, "unexpected request", http.StatusInternalServerError) + } + })) + defer server.Close() + + client, err := NewSourceClient(context.Background(), newTestSourceServerDetails(server.URL)) + require.NoError(t, err) + + metadata, err := client.GetFileMetadata(context.Background(), testSourceFile()) + require.NoError(t, err) + assert.Zero(t, metadata.DownloadCount) + assert.Empty(t, metadata.LastDownloadedBy) +} + +func TestGetFileMetadata_statsForbidden_continuesWithoutStats(t *testing.T) { + storagePath := "/api/storage/" + testSourceRelativePath() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == storagePath && r.URL.RawQuery == "": + w.WriteHeader(http.StatusOK) + require.NoError(t, json.NewEncoder(w).Encode(map[string]any{ + "size": "11", + "checksums": map[string]string{ + "sha1": "sha1-value", + }, + })) + case r.URL.Path == storagePath && r.URL.RawQuery == "properties": + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"errors":[{"message":"No properties could be found."}]}`)) + case r.URL.Path == storagePath && r.URL.RawQuery == "stats": + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write([]byte(`{"errors":[{"message":"Forbidden"}]}`)) + default: + t.Errorf("unexpected request: %s", r.URL.String()) + http.Error(w, "unexpected request", http.StatusInternalServerError) + } + })) + defer server.Close() + + client, err := NewSourceClient(context.Background(), newTestSourceServerDetails(server.URL)) + require.NoError(t, err) + + metadata, err := client.GetFileMetadata(context.Background(), testSourceFile()) + require.NoError(t, err) + assert.Equal(t, "sha1-value", metadata.Sha1) + assert.Zero(t, metadata.DownloadCount) + assert.Empty(t, metadata.LastDownloadedBy) +} + +func TestGetFileMetadata_statsContextCancel_returnsError(t *testing.T) { + storagePath := "/api/storage/" + testSourceRelativePath() + statsReceived := make(chan struct{}) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == storagePath && r.URL.RawQuery == "": + w.WriteHeader(http.StatusOK) + require.NoError(t, json.NewEncoder(w).Encode(map[string]any{ + "size": "11", + "checksums": map[string]string{ + "sha1": "sha1-value", + }, + })) + case r.URL.Path == storagePath && r.URL.RawQuery == "properties": + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"errors":[{"message":"No properties could be found."}]}`)) + case r.URL.Path == storagePath && r.URL.RawQuery == "stats": + close(statsReceived) + <-r.Context().Done() + default: + t.Errorf("unexpected request: %s", r.URL.String()) + } + })) + defer server.Close() + + client, err := NewSourceClient(context.Background(), newTestSourceServerDetails(server.URL)) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + done := make(chan struct { + metadata *SourceFileMetadata + err error + }, 1) + go func() { + metadata, err := client.GetFileMetadata(ctx, testSourceFile()) + done <- struct { + metadata *SourceFileMetadata + err error + }{metadata, err} + }() + + select { + case <-statsReceived: + case <-time.After(5 * time.Second): + t.Fatal("server did not receive stats request in time") + } + cancel() + + select { + case result := <-done: + assert.Nil(t, result.metadata) + assert.ErrorIs(t, result.err, context.Canceled) + case <-time.After(2 * time.Second): + t.Fatal("GetFileMetadata did not return promptly after stats cancellation") + } +} + +// TestGetFileMetadata_propertiesNotFoundOtherBody_itemStillExists_failsNotSilentlyDropped covers +// B-14: an ambiguous properties-404 (a body that isn't the pinned "No properties could be found" +// string) on an item that still exists must NOT be classified as a deletion (B-13/B-20), and must +// NOT silently yield empty properties either (that transfers the file and drops its properties +// with no warning). The properties GET is retried (bounded) and, if it stays ambiguous, the item +// fails so it lands in the errors CSV and is retried on the next run. +func TestGetFileMetadata_propertiesNotFoundOtherBody_itemStillExists_failsNotSilentlyDropped(t *testing.T) { + fastNotFoundRecheck(t) + storagePath := "/api/storage/" + testSourceRelativePath() + var storageInfoRequests, propertiesRequests int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == storagePath && r.URL.RawQuery == "": + storageInfoRequests++ + w.WriteHeader(http.StatusOK) + require.NoError(t, json.NewEncoder(w).Encode(map[string]any{ + "size": "11", + "checksums": map[string]string{ + "sha1": "sha1-value", + }, + })) + case r.URL.Path == storagePath && r.URL.RawQuery == "properties": + propertiesRequests++ + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"errors":[{"message":"Not found"}]}`)) + case r.URL.Path == storagePath && r.URL.RawQuery == "stats": + w.WriteHeader(http.StatusNotFound) + case r.URL.Path == testRootPath: + _, _ = w.Write([]byte(testRootBody)) + default: + t.Errorf("unexpected request: %s", r.URL.String()) + http.Error(w, "unexpected request", http.StatusInternalServerError) + } + })) + defer server.Close() + + client, err := NewSourceClient(context.Background(), newTestSourceServerDetails(server.URL)) + require.NoError(t, err) + + metadata, err := client.GetFileMetadata(context.Background(), testSourceFile()) + assert.Nil(t, metadata) + require.Error(t, err) + assert.False(t, IsSourceItemGone(err), "an existing item must not be classified as deleted") + assert.Equal(t, notFoundRecheckAttempts, propertiesRequests, "properties GET must be retried a bounded number of times") + // Once for the initial fetch, once to corroborate that the item still exists. + assert.Equal(t, 2, storageInfoRequests) +} + +// TestGetFileMetadata_propertiesNotFoundGermanJSONBody_treatedAsNoProperties is the B-14 lab +// repro after the structural classification: a properties 404 that is a genuine Artifactory JSON +// error ({"errors":[{"status":404,...}]}) means "no properties" whatever language the message is +// in, so the file is transferred (no failure, no retries) and the unknown message is logged. +func TestGetFileMetadata_propertiesNotFoundGermanJSONBody_treatedAsNoProperties(t *testing.T) { + fastNotFoundRecheck(t) + propertiesNotFoundWarned.Store(false) + _, stderrBuffer, previousLog := tests.RedirectLogOutputToBuffer() + defer log.SetLogger(previousLog) + storagePath := "/api/storage/" + testSourceRelativePath() + var propertiesRequests int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == storagePath && r.URL.RawQuery == "": + w.WriteHeader(http.StatusOK) + require.NoError(t, json.NewEncoder(w).Encode(map[string]any{"size": "11"})) + case r.URL.Path == storagePath && r.URL.RawQuery == "properties": + propertiesRequests++ + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"errors":[{"status":404,"message":"Keine Eigenschaften gefunden"}]}`)) + case r.URL.Path == storagePath && r.URL.RawQuery == "stats": + w.WriteHeader(http.StatusNotFound) + case r.URL.Path == testRootPath: + _, _ = w.Write([]byte(testRootBody)) + default: + http.Error(w, "unexpected request", http.StatusInternalServerError) + } + })) + defer server.Close() + + client, err := NewSourceClient(context.Background(), newTestSourceServerDetails(server.URL)) + require.NoError(t, err) + + metadata, err := client.GetFileMetadata(context.Background(), testSourceFile()) + require.NoError(t, err) + require.NotNil(t, metadata) + assert.Nil(t, metadata.Properties) + assert.Equal(t, 1, propertiesRequests, "a structural Artifactory 404 must not be retried") + assert.Contains(t, stderrBuffer.String(), "non-English/unknown 404 message") + assert.Contains(t, stderrBuffer.String(), testSourceRelativePath()) +} + +// TestGetFileMetadata_propertiesNotFoundGermanJSONBody_warnsOnce: a server answering every +// properties 404 in another locale must not produce one Warn per file; the first occurrence warns, +// the rest are Debug. +func TestGetFileMetadata_propertiesNotFoundGermanJSONBody_warnsOnce(t *testing.T) { + propertiesNotFoundWarned.Store(false) + _, stderrBuffer, previousLog := tests.RedirectLogOutputToBuffer() + defer log.SetLogger(previousLog) + storagePath := "/api/storage/" + testSourceRelativePath() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == storagePath && r.URL.RawQuery == "": + require.NoError(t, json.NewEncoder(w).Encode(map[string]any{"size": "11"})) + case r.URL.Path == storagePath && r.URL.RawQuery == "properties": + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"errors":[{"status":404,"message":"Keine Eigenschaften gefunden"}]}`)) + case r.URL.Path == storagePath && r.URL.RawQuery == "stats": + w.WriteHeader(http.StatusNotFound) + case r.URL.Path == testRootPath: + _, _ = w.Write([]byte(testRootBody)) + default: + http.Error(w, "unexpected request", http.StatusInternalServerError) + } + })) + defer server.Close() + client, err := NewSourceClient(context.Background(), newTestSourceServerDetails(server.URL)) + require.NoError(t, err) + + for i := 0; i < 5; i++ { + _, err = client.GetFileMetadata(context.Background(), testSourceFile()) + require.NoError(t, err) + } + warnCount := 0 + for _, line := range strings.Split(stderrBuffer.String(), "\n") { + // The debug-level repeats carry the same text, so count only the Warn lines (CI runs with JFROG_CLI_LOG_LEVEL=DEBUG). + if strings.Contains(line, "[Warn]") && strings.Contains(line, "non-English/unknown 404 message") { + warnCount++ + } + } + assert.Equal(t, 1, warnCount, "only the first occurrence may be logged at Warn level") +} + +// TestGetFileMetadata_propertiesNotFoundGermanJSONBody_itemGone: the item vanished between the +// storage-info and properties requests; a structural 404 must not mask the deletion. +func TestGetFileMetadata_propertiesNotFoundGermanJSONBody_itemGone(t *testing.T) { + fastNotFoundRecheck(t) + storagePath := "/api/storage/" + testSourceRelativePath() + var storageInfoRequests int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == storagePath && r.URL.RawQuery == "": + storageInfoRequests++ + if storageInfoRequests == 1 { + w.WriteHeader(http.StatusOK) + require.NoError(t, json.NewEncoder(w).Encode(map[string]any{"size": "11"})) + return + } + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"errors":[{"status":404,"message":"Unable to find item"}]}`)) + case r.URL.Path == storagePath && r.URL.RawQuery == "properties": + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"errors":[{"status":404,"message":"Element nicht gefunden"}]}`)) + case r.URL.Path == testRootPath: + _, _ = w.Write([]byte(testRootBody)) + default: + http.Error(w, "unexpected request", http.StatusInternalServerError) + } + })) + defer server.Close() + + client, err := NewSourceClient(context.Background(), newTestSourceServerDetails(server.URL)) + require.NoError(t, err) + + metadata, err := client.GetFileMetadata(context.Background(), testSourceFile()) + assert.Nil(t, metadata) + assert.True(t, IsSourceItemGone(err)) +} + +// TestGetFileMetadata_propertiesNotFoundNonArtifactoryBody_itemStillExists_fails: a 404 that is +// not an Artifactory JSON error (proxy HTML page, plain text, empty body, JSON without a 404 +// status) could be anything, so it is retried (bounded) and then fails the item rather than +// silently dropping the properties. +func TestGetFileMetadata_propertiesNotFoundNonArtifactoryBody_itemStillExists_fails(t *testing.T) { + bodies := map[string]string{ + "nginx html": "404 Not Found

404 Not Found

nginx
", + "plain text": "404 page not found", + "empty": "", + "json no status": `{"errors":[{"message":"Keine Eigenschaften gefunden"}]}`, + "json other shape": `{"message":"Keine Eigenschaften gefunden"}`, + } + for name, body := range bodies { + t.Run(name, func(t *testing.T) { + fastNotFoundRecheck(t) + storagePath := "/api/storage/" + testSourceRelativePath() + var propertiesRequests int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == storagePath && r.URL.RawQuery == "": + w.WriteHeader(http.StatusOK) + require.NoError(t, json.NewEncoder(w).Encode(map[string]any{"size": "11"})) + case r.URL.Path == storagePath && r.URL.RawQuery == "properties": + propertiesRequests++ + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(body)) + case r.URL.Path == storagePath && r.URL.RawQuery == "stats": + w.WriteHeader(http.StatusNotFound) + case r.URL.Path == testRootPath: + _, _ = w.Write([]byte(testRootBody)) + default: + http.Error(w, "unexpected request", http.StatusInternalServerError) + } + })) + defer server.Close() + + client, err := NewSourceClient(context.Background(), newTestSourceServerDetails(server.URL)) + require.NoError(t, err) + + metadata, err := client.GetFileMetadata(context.Background(), testSourceFile()) + assert.Nil(t, metadata, "properties must not be silently dropped") + require.Error(t, err) + assert.False(t, IsSourceItemGone(err)) + assert.Equal(t, notFoundRecheckAttempts, propertiesRequests) + }) + } +} + +// TestGetFileMetadata_propertiesNotFoundOtherBody_recoversOnRetry verifies that a transient +// ambiguous properties-404 followed by a proper answer yields the real properties. +func TestGetFileMetadata_propertiesNotFoundOtherBody_recoversOnRetry(t *testing.T) { + fastNotFoundRecheck(t) + storagePath := "/api/storage/" + testSourceRelativePath() + var propertiesRequests int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == storagePath && r.URL.RawQuery == "": + w.WriteHeader(http.StatusOK) + require.NoError(t, json.NewEncoder(w).Encode(map[string]any{"size": "11"})) + case r.URL.Path == storagePath && r.URL.RawQuery == "properties": + propertiesRequests++ + if propertiesRequests == 1 { + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`404 page not found`)) + return + } + w.WriteHeader(http.StatusOK) + require.NoError(t, json.NewEncoder(w).Encode(map[string]any{ + "properties": map[string][]string{"k": {"v"}}, + })) + case r.URL.Path == storagePath && r.URL.RawQuery == "stats": + w.WriteHeader(http.StatusNotFound) + case r.URL.Path == testRootPath: + _, _ = w.Write([]byte(testRootBody)) + default: + http.Error(w, "unexpected request", http.StatusInternalServerError) + } + })) + defer server.Close() + + client, err := NewSourceClient(context.Background(), newTestSourceServerDetails(server.URL)) + require.NoError(t, err) + + metadata, err := client.GetFileMetadata(context.Background(), testSourceFile()) + require.NoError(t, err) + assert.Equal(t, map[string][]string{"k": {"v"}}, metadata.Properties) + assert.Equal(t, 2, propertiesRequests) +} + +// TestGetFileMetadata_propertiesNotFoundProxyThenArtifactoryJSON404: a transient proxy 404 on the +// first properties request followed by Artifactory's own (non-English) JSON 404 resolves to +// "no properties" instead of exhausting the retries. +func TestGetFileMetadata_propertiesNotFoundProxyThenArtifactoryJSON404(t *testing.T) { + fastNotFoundRecheck(t) + storagePath := "/api/storage/" + testSourceRelativePath() + var propertiesRequests int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == storagePath && r.URL.RawQuery == "": + w.WriteHeader(http.StatusOK) + require.NoError(t, json.NewEncoder(w).Encode(map[string]any{"size": "11"})) + case r.URL.Path == storagePath && r.URL.RawQuery == "properties": + propertiesRequests++ + w.WriteHeader(http.StatusNotFound) + if propertiesRequests == 1 { + _, _ = w.Write([]byte(`404 page not found`)) + return + } + _, _ = w.Write([]byte(`{"errors":[{"status":404,"message":"Keine Eigenschaften gefunden"}]}`)) + case r.URL.Path == storagePath && r.URL.RawQuery == "stats": + w.WriteHeader(http.StatusNotFound) + case r.URL.Path == testRootPath: + _, _ = w.Write([]byte(testRootBody)) + default: + http.Error(w, "unexpected request", http.StatusInternalServerError) + } + })) + defer server.Close() + + client, err := NewSourceClient(context.Background(), newTestSourceServerDetails(server.URL)) + require.NoError(t, err) + + metadata, err := client.GetFileMetadata(context.Background(), testSourceFile()) + require.NoError(t, err) + assert.Nil(t, metadata.Properties) + assert.Equal(t, 2, propertiesRequests) +} + +// TestGetFileMetadata_propertiesNotFoundEnglishBody_noRetryNoCorroboration pins the existing +// behavior: the recognized English "No properties could be found" body means an empty +// property set, immediately, without extra requests. +func TestGetFileMetadata_propertiesNotFoundEnglishBody_noRetryNoCorroboration(t *testing.T) { + fastNotFoundRecheck(t) + storagePath := "/api/storage/" + testSourceRelativePath() + var storageInfoRequests, propertiesRequests int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == storagePath && r.URL.RawQuery == "": + storageInfoRequests++ + w.WriteHeader(http.StatusOK) + require.NoError(t, json.NewEncoder(w).Encode(map[string]any{"size": "11"})) + case r.URL.Path == storagePath && r.URL.RawQuery == "properties": + propertiesRequests++ + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"errors":[{"message":"No properties could be found."}]}`)) + case r.URL.Path == storagePath && r.URL.RawQuery == "stats": + w.WriteHeader(http.StatusNotFound) + default: + http.Error(w, "unexpected request", http.StatusInternalServerError) + } + })) + defer server.Close() + + client, err := NewSourceClient(context.Background(), newTestSourceServerDetails(server.URL)) + require.NoError(t, err) + + metadata, err := client.GetFileMetadata(context.Background(), testSourceFile()) + require.NoError(t, err) + assert.Nil(t, metadata.Properties) + assert.Equal(t, 1, propertiesRequests) + assert.Equal(t, 1, storageInfoRequests) +} + +// TestGetFileMetadata_propertiesNotFoundOtherBody_itemActuallyGone verifies that when the +// corroborating storage-info re-check also 404s, the item is correctly classified as gone. +func TestGetFileMetadata_propertiesNotFoundOtherBody_itemActuallyGone(t *testing.T) { + fastNotFoundRecheck(t) + storagePath := "/api/storage/" + testSourceRelativePath() + storageInfoCall := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == storagePath && r.URL.RawQuery == "": + storageInfoCall++ + if storageInfoCall == 1 { + w.WriteHeader(http.StatusOK) + require.NoError(t, json.NewEncoder(w).Encode(map[string]any{ + "size": "11", + "checksums": map[string]string{ + "sha1": "sha1-value", + }, + })) + return + } + // Corroborating re-check: the item is gone by the time properties are fetched. + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"errors":[{"status":404,"message":"Unable to find item"}]}`)) + case r.URL.Path == storagePath && r.URL.RawQuery == "properties": + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"errors":[{"status":404,"message":"Unable to find item"}]}`)) + case r.URL.Path == testRootPath: + _, _ = w.Write([]byte(testRootBody)) + default: + t.Errorf("unexpected request: %s", r.URL.String()) + http.Error(w, "unexpected request", http.StatusInternalServerError) + } + })) + defer server.Close() + + client, err := NewSourceClient(context.Background(), newTestSourceServerDetails(server.URL)) + require.NoError(t, err) + + metadata, err := client.GetFileMetadata(context.Background(), testSourceFile()) + assert.Nil(t, metadata) + assert.True(t, IsSourceItemGone(err)) +} + +// TestGetFileMetadata_notFound_confirmedByCorroboratingCheck verifies that a storage-info 404 +// is classified as ErrSourceItemGone only after the control probe (repository root 200) and a +// second item check confirm it: item 404, root 200, item 404. +func TestGetFileMetadata_notFound_confirmedByCorroboratingCheck(t *testing.T) { + var itemRequests, rootRequests int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch storageRequestKind(r) { + case "root": + rootRequests++ + _, _ = w.Write([]byte(testRootBody)) + case "item": + itemRequests++ + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"errors":[{"status":404,"message":"Unable to find item"}]}`)) + default: + t.Errorf("unexpected request: %s", r.URL.String()) + http.Error(w, "unexpected request", http.StatusInternalServerError) + } + })) + defer server.Close() + + client, err := NewSourceClient(context.Background(), newTestSourceServerDetails(server.URL)) + require.NoError(t, err) + + metadata, err := client.GetFileMetadata(context.Background(), testSourceFile()) + assert.Nil(t, metadata) + assert.True(t, IsSourceItemGone(err)) + assert.Equal(t, 2, itemRequests, "the initial storage-info 404 plus one confirming item check") + assert.Equal(t, 1, rootRequests, "one control probe") +} + +// TestGetFileMetadata_notFound_transient_notGone verifies that a storage-info 404 which does +// NOT hold up on corroborating re-check (e.g. a transient proxy hiccup) is surfaced as a plain +// error, not silently classified as a deletion. +func TestGetFileMetadata_notFound_transient_notGone(t *testing.T) { + storagePath := "/api/storage/" + testSourceRelativePath() + requestCount := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == testRootPath { + _, _ = w.Write([]byte(testRootBody)) + return + } + if r.URL.Path != storagePath || r.URL.RawQuery != "" { + t.Errorf("unexpected request: %s", r.URL.String()) + http.Error(w, "unexpected request", http.StatusInternalServerError) + return + } + requestCount++ + if requestCount == 1 { + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"errors":[{"status":404,"message":"Unable to find item"}]}`)) + return + } + w.WriteHeader(http.StatusOK) + require.NoError(t, json.NewEncoder(w).Encode(map[string]any{ + "size": "11", + "checksums": map[string]string{ + "sha1": "sha1-value", + }, + })) + })) + defer server.Close() + + client, err := NewSourceClient(context.Background(), newTestSourceServerDetails(server.URL)) + require.NoError(t, err) + + metadata, err := client.GetFileMetadata(context.Background(), testSourceFile()) + assert.Nil(t, metadata) + require.Error(t, err) + assert.False(t, IsSourceItemGone(err), "a 404 that doesn't hold up on corroboration must not be classified as a deletion") +} + +func TestGetFileReader_success(t *testing.T) { + fileContent := "streaming-bytes" + downloadPath := "/" + testSourceRelativePath() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != downloadPath { + t.Errorf("unexpected request: %s", r.URL.String()) + http.Error(w, "unexpected request", http.StatusInternalServerError) + return + } + _, _ = w.Write([]byte(fileContent)) + })) + defer server.Close() + + client, err := NewSourceClient(context.Background(), newTestSourceServerDetails(server.URL)) + require.NoError(t, err) + + reader, err := client.GetFileReader(context.Background(), testSourceFile()) + require.NoError(t, err) + require.NotNil(t, reader) + defer func() { + assert.NoError(t, reader.Close()) + }() + + got, err := io.ReadAll(reader) + require.NoError(t, err) + assert.Equal(t, fileContent, string(got)) +} + +// closeTrackingTransport wraps a RoundTripper and records, per request path, whether the +// response body handed to the caller was closed. +type closeTrackingTransport struct { + base http.RoundTripper + mu sync.Mutex + // closed maps request path -> body closed. + closed map[string]bool +} + +type closeTrackingBody struct { + io.ReadCloser + onClose func() +} + +func (b *closeTrackingBody) Close() error { + b.onClose() + return b.ReadCloser.Close() +} + +func (c *closeTrackingTransport) RoundTrip(req *http.Request) (*http.Response, error) { + resp, err := c.base.RoundTrip(req) + if err != nil { + return resp, err + } + reqPath := req.URL.Path + c.mu.Lock() + c.closed[reqPath] = false + c.mu.Unlock() + resp.Body = &closeTrackingBody{ReadCloser: resp.Body, onClose: func() { + c.mu.Lock() + c.closed[reqPath] = true + c.mu.Unlock() + }} + return resp, nil +} + +func (c *closeTrackingTransport) isClosed(reqPath string) (closed, seen bool) { + c.mu.Lock() + defer c.mu.Unlock() + closed, seen = c.closed[reqPath] + return +} + +// trackStreamBodies makes the client's content-GET (stream) transport record body closes. +func trackStreamBodies(client *SourceClient) *closeTrackingTransport { + httpClient := client.streamServiceManager.Client().GetHttpClient().GetClient() + tracker := &closeTrackingTransport{base: httpClient.Transport, closed: map[string]bool{}} + httpClient.Transport = tracker + return tracker +} + +// TestGetFileReader_notFound_itemGone_closesBody: a content-GET 404 corroborated by the control +// probe (root 200) and a storage-info 404 is a real deletion (F-09/B-13 "skipped, logged"), and +// the 404 response body of the content GET must be closed. +func TestGetFileReader_notFound_itemGone_closesBody(t *testing.T) { + downloadPath := "/" + testSourceRelativePath() + var itemRequests, rootRequests int + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch storageRequestKind(r) { + case "content": + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"errors":[{"status":404,"message":"Unable to find item"}]}`)) + case "root": + rootRequests++ + _, _ = w.Write([]byte(testRootBody)) + case "item": + itemRequests++ + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"errors":[{"status":404,"message":"Unable to find item"}]}`)) + default: + t.Errorf("unexpected request: %s", r.URL.String()) + http.Error(w, "unexpected request", http.StatusInternalServerError) + } + })) + defer server.Close() + + client, err := NewSourceClient(context.Background(), newTestSourceServerDetails(server.URL)) + require.NoError(t, err) + tracker := trackStreamBodies(client) + + reader, err := client.GetFileReader(context.Background(), testSourceFile()) + assert.Nil(t, reader) + assert.True(t, IsSourceItemGone(err)) + assert.Equal(t, 1, rootRequests) + assert.Equal(t, 1, itemRequests) + closed, seen := tracker.isClosed(downloadPath) + assert.True(t, seen, "content GET was not observed") + assert.True(t, closed, "the 404 response body of the content GET must be closed") +} + +// TestGetFileReader_notFound_storageInfoExists_notGone covers B-13/B-38: a content-GET 404 while +// storage-info says the item exists (transient 404, blacked-out repo, ...) must be a retryable +// error so the file lands in the errors CSV, never a silent "gone" skip. The error text must +// name the persistent causes instead of promising that it is transient (M3), and the content 404 +// body must be closed. +func TestGetFileReader_notFound_storageInfoExists_notGone(t *testing.T) { + downloadPath := "/" + testSourceRelativePath() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch storageRequestKind(r) { + case "content": + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"errors":[{"status":404,"message":"Unable to find item"}]}`)) + case "root": + _, _ = w.Write([]byte(testRootBody)) + case "item": + w.WriteHeader(http.StatusOK) + require.NoError(t, json.NewEncoder(w).Encode(map[string]any{"size": "11"})) + default: + t.Errorf("unexpected request: %s", r.URL.String()) + http.Error(w, "unexpected request", http.StatusInternalServerError) + } + })) + defer server.Close() + + client, err := NewSourceClient(context.Background(), newTestSourceServerDetails(server.URL)) + require.NoError(t, err) + tracker := trackStreamBodies(client) + + reader, err := client.GetFileReader(context.Background(), testSourceFile()) + assert.Nil(t, reader) + require.Error(t, err) + assert.False(t, IsSourceItemGone(err), "content 404 with an existing item must not be classified as a deletion") + for _, cause := range []string{"blacked out", "include/exclude", "filestore"} { + assert.Contains(t, err.Error(), cause, "the error must name the persistent causes of a content 404 on an existing item") + } + assert.NotContains(t, err.Error(), "treating this as a transient error") + closed, _ := tracker.isClosed(downloadPath) + assert.True(t, closed, "the 404 response body must be closed") +} + +// TestGetFileReader_notFound_confirmError_isRetryableNotGone: when the corroboration itself fails +// (here the control probe is forbidden) the content 404 must surface as an error saying it could +// not be corroborated, never as "gone", and the response body must still be closed. +func TestGetFileReader_notFound_confirmError_isRetryableNotGone(t *testing.T) { + downloadPath := "/" + testSourceRelativePath() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch storageRequestKind(r) { + case "content": + w.WriteHeader(http.StatusNotFound) + case "root": + w.WriteHeader(http.StatusForbidden) + default: + t.Errorf("unexpected request: %s", r.URL.String()) + http.Error(w, "unexpected request", http.StatusInternalServerError) + } + })) + defer server.Close() + + client, err := NewSourceClient(context.Background(), newTestSourceServerDetails(server.URL)) + require.NoError(t, err) + tracker := trackStreamBodies(client) + + reader, err := client.GetFileReader(context.Background(), testSourceFile()) + assert.Nil(t, reader) + require.Error(t, err) + assert.False(t, IsSourceItemGone(err)) + assert.Contains(t, err.Error(), "could not corroborate") + closed, _ := tracker.isClosed(downloadPath) + assert.True(t, closed, "the 404 response body must be closed") +} + +func TestGetFileReader_cancellation_closesBody(t *testing.T) { + downloadPath := "/" + testSourceRelativePath() + requestCancelled := make(chan struct{}, 1) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != downloadPath { + t.Errorf("unexpected request: %s", r.URL.String()) + http.Error(w, "unexpected request", http.StatusInternalServerError) + return + } + w.WriteHeader(http.StatusOK) + flusher, ok := w.(http.Flusher) + require.True(t, ok) + for { + select { + case <-r.Context().Done(): + requestCancelled <- struct{}{} + return + default: + _, _ = w.Write([]byte("x")) + flusher.Flush() + time.Sleep(5 * time.Millisecond) + } + } + })) + defer server.Close() + + client, err := NewSourceClient(context.Background(), newTestSourceServerDetails(server.URL)) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + reader, err := client.GetFileReader(ctx, testSourceFile()) + require.NoError(t, err) + require.NotNil(t, reader) + + buf := make([]byte, 1) + _, err = reader.Read(buf) + require.NoError(t, err) + + cancel() + + _, err = reader.Read(buf) + assert.ErrorIs(t, err, context.Canceled) + assert.NoError(t, reader.Close()) + + select { + case <-requestCancelled: + case <-time.After(time.Second): + t.Fatal("expected server request to observe cancellation") + } +} + +func newTestSourceServerDetails(serverURL string) *config.ServerDetails { + return &config.ServerDetails{ + Url: serverURL + "/", + ArtifactoryUrl: serverURL + "/", + } +} + +func TestNewSourceClient_streamServiceManagerHasNoOverallTimeoutOrRetries(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + client, err := NewSourceClient(context.Background(), newTestSourceServerDetails(server.URL)) + require.NoError(t, err) + + streamConfig := client.streamServiceManager.GetConfig() + assert.Zero(t, streamConfig.GetOverallRequestTimeout()) + assert.Zero(t, streamConfig.GetHttpRetries()) + + metadataConfig := client.metadataServiceManager.GetConfig() + assert.Equal(t, time.Minute, metadataConfig.GetOverallRequestTimeout()) + assert.Equal(t, metadataTransferRetries, metadataConfig.GetHttpRetries()) + assert.Equal(t, metadataRetryWaitMilliSecs, metadataConfig.GetHttpRetryWaitMilliSecs()) +} + +func TestGetFileReader_unrelatedErrorContaining404_notSourceGone(t *testing.T) { + downloadPath := "/" + testSourceRelativePath() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != downloadPath { + t.Errorf("unexpected request: %s", r.URL.String()) + http.Error(w, "unexpected request", http.StatusInternalServerError) + return + } + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(`{"errors":[{"message":"upstream returned 404 from cache"}]}`)) + })) + defer server.Close() + + client, err := NewSourceClient(context.Background(), newTestSourceServerDetails(server.URL)) + require.NoError(t, err) + + reader, err := client.GetFileReader(context.Background(), testSourceFile()) + assert.Nil(t, reader) + require.Error(t, err) + assert.False(t, IsSourceItemGone(err)) + // The source's explanation must reach the error (and so the errors CSV), not just the status line. + assert.Contains(t, err.Error(), "upstream returned 404 from cache") +} + +func TestGetFileReader_streamSurvivesSlowResponse(t *testing.T) { + downloadPath := "/" + testSourceRelativePath() + fileContent := "delayed-stream" + streamDelay := 300 * time.Millisecond + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != downloadPath { + t.Errorf("unexpected request: %s", r.URL.String()) + http.Error(w, "unexpected request", http.StatusInternalServerError) + return + } + time.Sleep(streamDelay) + _, _ = w.Write([]byte(fileContent)) + })) + defer server.Close() + + client, err := NewSourceClient(context.Background(), newTestSourceServerDetails(server.URL)) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), streamDelay*3) + defer cancel() + + reader, err := client.GetFileReader(ctx, testSourceFile()) + require.NoError(t, err) + require.NotNil(t, reader) + defer func() { + assert.NoError(t, reader.Close()) + }() + + got, err := io.ReadAll(reader) + require.NoError(t, err) + assert.Equal(t, fileContent, string(got)) +} + +// TestGetFileReader_nonRespondingPeer_boundedByResponseHeaderTimeout reproduces B-17(b): a peer +// that accepts the TCP connection but never writes any response (unlike a slow-but-responding +// server, or a connection refused/reset). Without a ResponseHeaderTimeout on the streaming +// transport, GetFileReader would block forever on such a peer, since the streaming service +// manager's overall http.Client.Timeout is intentionally 0 (see +// TestNewSourceClient_streamServiceManagerHasNoOverallTimeoutOrRetries) to allow large file +// bodies to stream for as long as they need. streamResponseHeaderTimeout is shrunk here so the +// test doesn't have to wait out the multi-minute production value. +func TestGetFileReader_nonRespondingPeer_boundedByResponseHeaderTimeout(t *testing.T) { + origTimeout := streamResponseHeaderTimeout + streamResponseHeaderTimeout = 200 * time.Millisecond + defer func() { streamResponseHeaderTimeout = origTimeout }() + + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + defer func() { _ = ln.Close() }() + go func() { + for { + conn, acceptErr := ln.Accept() + if acceptErr != nil { + return + } + // Accept the connection and drain whatever the client sends, but never write a + // response and never close the connection - simulating a hung peer. + go func(c net.Conn) { + buf := make([]byte, 4096) + for { + if _, readErr := c.Read(buf); readErr != nil { + return + } + } + }(conn) + } + }() + + client, err := NewSourceClient(context.Background(), newTestSourceServerDetails("http://"+ln.Addr().String())) + require.NoError(t, err) + + type result struct { + reader io.ReadCloser + err error + } + done := make(chan result, 1) + go func() { + reader, getErr := client.GetFileReader(context.Background(), testSourceFile()) + done <- result{reader, getErr} + }() + + select { + case res := <-done: + require.Error(t, res.err, "a non-responding peer must fail once the response-header timeout elapses, not succeed") + assert.Nil(t, res.reader) + case <-time.After(5 * time.Second): + t.Fatal("GetFileReader hung indefinitely against a non-responding peer instead of being bounded by streamResponseHeaderTimeout") + } +} + +// TestGetFileReader_contextCancel_abortsRoundTrip verifies that cancelling the caller +// context while the server has not yet sent any response headers causes GetFileReader +// to return promptly with context.Canceled, not block until the server decides to reply. +func TestGetFileReader_contextCancel_abortsRoundTrip(t *testing.T) { + downloadPath := "/" + testSourceRelativePath() + serverReceivedRequest := make(chan struct{}) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != downloadPath { + t.Errorf("unexpected request: %s", r.URL.String()) + return + } + // Signal that the server has the request; block until it's cancelled. + close(serverReceivedRequest) + <-r.Context().Done() + })) + defer server.Close() + + client, err := NewSourceClient(context.Background(), newTestSourceServerDetails(server.URL)) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + done := make(chan error, 1) + go func() { + _, err := client.GetFileReader(ctx, testSourceFile()) + done <- err + }() + + // Wait for server to start handling the request, then cancel. + select { + case <-serverReceivedRequest: + case <-time.After(5 * time.Second): + t.Fatal("server did not receive request in time") + } + cancel() + + select { + case err := <-done: + assert.ErrorIs(t, err, context.Canceled) + case <-time.After(2 * time.Second): + t.Fatal("GetFileReader did not return promptly after context cancellation") + } +} + +// TestGetFileMetadata_contextCancel_abortsMetadataRoundTrip verifies that cancelling the +// caller context while fetchFileInfo is waiting for a response causes GetFileMetadata to +// return promptly with context.Canceled. +func TestGetFileMetadata_contextCancel_abortsMetadataRoundTrip(t *testing.T) { + storagePath := "/api/storage/" + testSourceRelativePath() + serverReceivedRequest := make(chan struct{}) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != storagePath { + return + } + close(serverReceivedRequest) + <-r.Context().Done() + })) + defer server.Close() + + client, err := NewSourceClient(context.Background(), newTestSourceServerDetails(server.URL)) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + done := make(chan error, 1) + go func() { + _, err := client.GetFileMetadata(ctx, testSourceFile()) + done <- err + }() + + select { + case <-serverReceivedRequest: + case <-time.After(5 * time.Second): + t.Fatal("server did not receive metadata request in time") + } + cancel() + + select { + case err := <-done: + assert.ErrorIs(t, err, context.Canceled) + case <-time.After(2 * time.Second): + t.Fatal("GetFileMetadata did not return promptly after context cancellation") + } +} + +// TestGetFileMetadata_sizeParseError_noFallback verifies that when the fileinfo size +// field is unparseable and the caller's FileRepresentation carries no positive size, +// GetFileMetadata returns a wrapped, path-specific error rather than silently using zero. +func TestGetFileMetadata_sizeParseError_noFallback(t *testing.T) { + storagePath := "/api/storage/" + testSourceRelativePath() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == storagePath && r.URL.RawQuery == "": + w.WriteHeader(http.StatusOK) + require.NoError(t, json.NewEncoder(w).Encode(map[string]any{ + "size": "not-a-number", + "checksums": map[string]string{ + "sha1": "sha1-value", + }, + })) + default: + // other endpoints won't be reached before error is returned + w.WriteHeader(http.StatusInternalServerError) + } + })) + defer server.Close() + + client, err := NewSourceClient(context.Background(), newTestSourceServerDetails(server.URL)) + require.NoError(t, err) + + // Size == 0: no positive fallback, so a parse error must be returned. + file := testSourceFile() + file.Size = 0 + + metadata, err := client.GetFileMetadata(context.Background(), file) + assert.Nil(t, metadata) + require.Error(t, err) + assert.Contains(t, err.Error(), testSourceRelativePath()) +} + +// TestGetFileMetadata_sizeParseError_fallbackUsed verifies that when the fileinfo size +// is unparseable but the FileRepresentation carries a positive size, the positive size +// is used as a fallback and no error is returned. +func TestGetFileMetadata_sizeParseError_fallbackUsed(t *testing.T) { + storagePath := "/api/storage/" + testSourceRelativePath() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == storagePath && r.URL.RawQuery == "": + w.WriteHeader(http.StatusOK) + require.NoError(t, json.NewEncoder(w).Encode(map[string]any{ + "size": "not-a-number", + "checksums": map[string]string{ + "sha1": "sha1-value", + }, + })) + case r.URL.Path == storagePath && r.URL.RawQuery == "properties": + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"errors":[{"message":"No properties could be found."}]}`)) + case r.URL.Path == storagePath && r.URL.RawQuery == "stats": + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"errors":[{"message":"Unable to find item"}]}`)) + default: + t.Errorf("unexpected request: %s", r.URL.String()) + http.Error(w, "unexpected request", http.StatusInternalServerError) + } + })) + defer server.Close() + + client, err := NewSourceClient(context.Background(), newTestSourceServerDetails(server.URL)) + require.NoError(t, err) + + // testSourceFile() has Size=11 > 0, so the fallback must be used. + metadata, err := client.GetFileMetadata(context.Background(), testSourceFile()) + require.NoError(t, err) + assert.Equal(t, int64(11), metadata.Size) +} + +func TestGetFileMetadata_folderHasNoSizeField(t *testing.T) { + storagePath := "/api/storage/" + testSourceRelativePath() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == storagePath && r.URL.RawQuery == "": + w.WriteHeader(http.StatusOK) + require.NoError(t, json.NewEncoder(w).Encode(map[string]any{ + "repo": testSourceRepo, + "path": testSourcePath + "/" + testSourceName, + "created": "2020-01-01T00:00:00.000Z", + "createdBy": "admin", + "lastModified": "2020-01-02T00:00:00.000Z", + "modifiedBy": "deployer", + "children": []map[string]any{{"uri": "/child", "folder": false}}, + })) + case r.URL.Path == storagePath && r.URL.RawQuery == "properties": + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"errors":[{"message":"No properties could be found."}]}`)) + case r.URL.Path == storagePath && r.URL.RawQuery == "stats": + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"errors":[{"message":"Unable to find item"}]}`)) + default: + t.Errorf("unexpected request: %s", r.URL.String()) + http.Error(w, "unexpected request", http.StatusInternalServerError) + } + })) + defer server.Close() + + client, err := NewSourceClient(context.Background(), newTestSourceServerDetails(server.URL)) + require.NoError(t, err) + + file := testSourceFile() + file.Size = 0 + + metadata, err := client.GetFileMetadata(context.Background(), file) + require.NoError(t, err) + require.NotNil(t, metadata) + assert.Zero(t, metadata.Size) +} + +// TestGetFileMetadata_folderItem_skipsStatsGET verifies that a real folder item (Name == "", +// as produced for AQL folder results) resolves Size 0 and never issues the "stats" GET, since +// download statistics don't apply to folders. +func TestGetFileMetadata_folderItem_skipsStatsGET(t *testing.T) { + folderRelativePath := testSourceRepo + "/" + testSourcePath + storagePath := "/api/storage/" + folderRelativePath + statsRequested := false + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == storagePath && r.URL.RawQuery == "": + w.WriteHeader(http.StatusOK) + require.NoError(t, json.NewEncoder(w).Encode(map[string]any{ + "repo": testSourceRepo, + "path": testSourcePath, + "children": []map[string]any{{"uri": "/child", "folder": false}}, + })) + case r.URL.Path == storagePath && r.URL.RawQuery == "properties": + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"errors":[{"message":"No properties could be found."}]}`)) + case r.URL.Path == storagePath && r.URL.RawQuery == "stats": + statsRequested = true + w.WriteHeader(http.StatusOK) + require.NoError(t, json.NewEncoder(w).Encode(map[string]any{})) + default: + t.Errorf("unexpected request: %s", r.URL.String()) + http.Error(w, "unexpected request", http.StatusInternalServerError) + } + })) + defer server.Close() + + client, err := NewSourceClient(context.Background(), newTestSourceServerDetails(server.URL)) + require.NoError(t, err) + + folder := api.FileRepresentation{Repo: testSourceRepo, Path: testSourcePath, Name: ""} + metadata, err := client.GetFileMetadata(context.Background(), folder) + require.NoError(t, err) + require.NotNil(t, metadata) + assert.Zero(t, metadata.Size) + assert.False(t, statsRequested, "stats GET must be skipped for a folder item") +} + +// TestSourceClient_metadataAndContentGET_usesOnlySourceCredentials exercises both +// metadata GET and content GET against a live source httptest while a distinct +// target httptest must receive zero traffic. +func TestSourceClient_metadataAndContentGET_usesOnlySourceCredentials(t *testing.T) { + const sourceToken = "source-secret-token" + const targetToken = "target-secret-token" + storagePath := "/api/storage/" + testSourceRelativePath() + downloadPath := "/" + testSourceRelativePath() + fileContent := "hello world" + + var sourceAuth []string + var sourcePaths []string + var sawContentGET bool + targetHits := 0 + + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + targetHits++ + t.Errorf("source client contacted target: %s %s auth=%q", r.Method, r.URL.String(), r.Header.Get("Authorization")) + w.WriteHeader(http.StatusForbidden) + })) + defer target.Close() + + source := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + sourcePaths = append(sourcePaths, r.URL.Path) + sourceAuth = append(sourceAuth, r.Header.Get("Authorization")) + switch { + case r.URL.Path == storagePath && r.URL.RawQuery == "": + w.WriteHeader(http.StatusOK) + require.NoError(t, json.NewEncoder(w).Encode(map[string]any{ + "size": strconv.Itoa(len(fileContent)), + "checksums": map[string]string{"sha1": "sha1-value"}, + })) + case r.URL.Path == storagePath && r.URL.RawQuery == "properties": + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"errors":[{"message":"No properties could be found."}]}`)) + case r.URL.Path == storagePath && r.URL.RawQuery == "stats": + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"errors":[{"message":"Unable to find item"}]}`)) + case r.URL.Path == downloadPath: + sawContentGET = true + _, _ = w.Write([]byte(fileContent)) + default: + t.Errorf("unexpected source request: %s", r.URL.String()) + http.Error(w, "unexpected request", http.StatusInternalServerError) + } + })) + defer source.Close() + + sourceDetails := newTestSourceServerDetails(source.URL) + sourceDetails.AccessToken = sourceToken + targetDetails := newTestSourceServerDetails(target.URL) + targetDetails.AccessToken = targetToken + require.NotEqual(t, sourceDetails.ArtifactoryUrl, targetDetails.ArtifactoryUrl) + + client, err := NewSourceClient(context.Background(), sourceDetails) + require.NoError(t, err) + + _, err = client.GetFileMetadata(context.Background(), testSourceFile()) + require.NoError(t, err) + + reader, err := client.GetFileReader(context.Background(), testSourceFile()) + require.NoError(t, err) + require.NotNil(t, reader) + got, err := io.ReadAll(reader) + require.NoError(t, err) + assert.Equal(t, fileContent, string(got)) + assert.NoError(t, reader.Close()) + + require.NotEmpty(t, sourceAuth, "expected source traffic") + require.True(t, sawContentGET, "content GET must be exercised") + for i, auth := range sourceAuth { + assert.Equal(t, "Bearer "+sourceToken, auth, "source request %d", i) + assert.NotContains(t, auth, targetToken) + } + assert.Zero(t, targetHits) + for _, p := range sourcePaths { + assert.NotContains(t, p, "/api/plugins/execute") + } +} + +// --- confirmItemGone control probe (B-20 / JFMIG-93 review blocker B1) --- + +const testRootBody = `{"repo":"` + testSourceRepo + `","path":"/","children":[]}` + +// storageRequestKind classifies a request to the fake source by what it targets. +func storageRequestKind(r *http.Request) string { + rel := testSourceRelativePath() + switch { + case r.URL.Path == "/"+rel: + return "content" + case r.URL.Path == "/api/storage/"+rel && r.URL.RawQuery == "": + return "item" + case r.URL.Path == "/api/storage/"+rel && r.URL.RawQuery == "properties": + return "properties" + case r.URL.Path == "/api/storage/"+rel && r.URL.RawQuery == "stats": + return "stats" + case r.URL.Path == "/api/storage/"+testSourceRepo: + return "root" + } + return "unknown" +} + +// fastConfirmRetry shrinks the interval between retries of the confirmation requests. +func fastConfirmRetry(t *testing.T) { + t.Helper() + old := confirmRetryInterval + confirmRetryInterval = time.Millisecond + t.Cleanup(func() { confirmRetryInterval = old }) +} + +// confirmEntryPoints are the three callers of confirmItemGone, each driven through its public method. +// "setup" prepares the request counters the fake server uses to answer the first, ambiguous request. +var confirmEntryPoints = []struct { + name string + // firstRequestOK: the first item (storage-info) request answers 200 so the flow reaches the + // request that triggers the confirmation (properties 404). + firstRequestOK bool + run func(client *SourceClient) error +}{ + {"storage-info 404", false, func(c *SourceClient) error { + _, err := c.GetFileMetadata(context.Background(), testSourceFile()) + return err + }}, + {"properties 404", true, func(c *SourceClient) error { + _, err := c.GetFileMetadata(context.Background(), testSourceFile()) + return err + }}, + {"content 404", false, func(c *SourceClient) error { + r, err := c.GetFileReader(context.Background(), testSourceFile()) + if r != nil { + _ = r.Close() + } + return err + }}, +} + +// TestConfirmItemGone_outageWindowLongerThanRecheckSpan_isRetryableNotGone is the B-20 acceptance +// test: while the source answers 404 for the item AND for the repository root (a router/Artifactory +// restart window of any length), the item must never be declared gone. The window is held open for +// the whole call, so no number of re-checks can see the end of it; only a control probe can tell +// "outage" from "deleted". Once the window closes the same call succeeds. +func TestConfirmItemGone_outageWindowLongerThanRecheckSpan_isRetryableNotGone(t *testing.T) { + for _, ep := range confirmEntryPoints { + t.Run(ep.name, func(t *testing.T) { + var windowOpen atomic.Bool + windowOpen.Store(true) + var itemRequests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + kind := storageRequestKind(r) + if kind == "item" && ep.firstRequestOK && itemRequests.Add(1) == 1 { + _, _ = w.Write([]byte(`{"size":"11"}`)) + return + } + if windowOpen.Load() && (kind == "item" || kind == "properties" || kind == "root" || kind == "content" || kind == "stats") { + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`404 page not found`)) + return + } + switch kind { + case "root": + _, _ = w.Write([]byte(testRootBody)) + case "item": + _, _ = w.Write([]byte(`{"size":"11"}`)) + case "properties": + _, _ = w.Write([]byte(`{"properties":{"k":["v"]}}`)) + case "content": + _, _ = w.Write([]byte("hello world")) + case "stats": + w.WriteHeader(http.StatusNotFound) + default: + http.Error(w, "unexpected request", http.StatusInternalServerError) + } + })) + defer server.Close() + client, err := NewSourceClient(context.Background(), newTestSourceServerDetails(server.URL)) + require.NoError(t, err) + + start := time.Now() + err = ep.run(client) + elapsed := time.Since(start) + require.Error(t, err, "a 404 while the repository root also 404s is an outage, not a deletion") + assert.False(t, IsSourceItemGone(err), "item must not be declared gone inside an outage window: %v", err) + assert.Less(t, elapsed, 1500*time.Millisecond, "an outage must be reported without sleeping through re-checks") + + // The window closes: the very same call now succeeds (the failure was retryable). + windowOpen.Store(false) + itemRequests.Store(10) + assert.NoError(t, ep.run(client)) + }) + } +} + +// TestConfirmItemGone_rootOK_itemMissing_goneWithoutStall: the repository root answers (200, JSON) +// and the item 404s, so the item really is deleted. This must be decided at once, without the +// multi-second sleeping the old consecutive re-checks cost every deleted file. +func TestConfirmItemGone_rootOK_itemMissing_goneWithoutStall(t *testing.T) { + for _, ep := range confirmEntryPoints { + t.Run(ep.name, func(t *testing.T) { + var rootRequests atomic.Int32 + var itemRequests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch storageRequestKind(r) { + case "root": + rootRequests.Add(1) + _, _ = w.Write([]byte(testRootBody)) + case "item": + if ep.firstRequestOK && itemRequests.Add(1) == 1 { + _, _ = w.Write([]byte(`{"size":"11"}`)) + return + } + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"errors":[{"status":404,"message":"Unable to find item"}]}`)) + case "properties", "content": + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"errors":[{"status":404,"message":"Unable to find item"}]}`)) + default: + http.Error(w, "unexpected request", http.StatusInternalServerError) + } + })) + defer server.Close() + client, err := NewSourceClient(context.Background(), newTestSourceServerDetails(server.URL)) + require.NoError(t, err) + + start := time.Now() + err = ep.run(client) + elapsed := time.Since(start) + assert.True(t, IsSourceItemGone(err), "root 200 + item 404 means deleted, got: %v", err) + assert.Less(t, elapsed, 500*time.Millisecond, "deleted items must not stall the worker") + assert.GreaterOrEqual(t, int(rootRequests.Load()), 1, "the repository root must be probed before declaring gone") + }) + } +} + +// TestConfirmItemGone_rootProbeUnusable_isRetryableNotGone: a root probe that does not prove the +// repository is reachable and healthy (404, 403, 5xx, a 200 that is not Artifactory JSON) must +// never let an item be declared gone. +func TestConfirmItemGone_rootProbeUnusable_isRetryableNotGone(t *testing.T) { + fastConfirmRetry(t) + cases := map[string]func(w http.ResponseWriter){ + "root 404": func(w http.ResponseWriter) { + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`404 page not found`)) + }, + "root 403": func(w http.ResponseWriter) { w.WriteHeader(http.StatusForbidden) }, + "root 503": func(w http.ResponseWriter) { w.WriteHeader(http.StatusServiceUnavailable) }, + "root 200 html": func(w http.ResponseWriter) { _, _ = w.Write([]byte(`Starting up`)) }, + "root 200 empty": func(w http.ResponseWriter) {}, + "root 200 no repo": func(w http.ResponseWriter) { _, _ = w.Write([]byte(`{"errors":[]}`)) }, + } + for name, rootAnswer := range cases { + t.Run(name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if storageRequestKind(r) == "root" { + rootAnswer(w) + return + } + w.WriteHeader(http.StatusNotFound) + })) + defer server.Close() + client, err := NewSourceClient(context.Background(), newTestSourceServerDetails(server.URL)) + require.NoError(t, err) + + _, err = client.GetFileMetadata(context.Background(), testSourceFile()) + require.Error(t, err) + assert.False(t, IsSourceItemGone(err), "unusable root probe must not produce 'gone': %v", err) + }) + } +} + +// TestConfirmItemGone_serverErrorsAreBoundedAndHonourContext: the confirmation requests must not +// inherit the metadata client's 600 x 5s retry (about 50 minutes on a persistent 503), and must stop +// promptly when the context is cancelled. +func TestConfirmItemGone_serverErrorsAreBoundedAndHonourContext(t *testing.T) { + t.Run("bounded", func(t *testing.T) { + fastConfirmRetry(t) + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + requests.Add(1) + w.WriteHeader(http.StatusServiceUnavailable) + })) + defer server.Close() + client, err := NewSourceClient(context.Background(), newTestSourceServerDetails(server.URL)) + require.NoError(t, err) + + // Safety net so a regression fails after a few seconds instead of retrying for ~50 minutes. + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + time.AfterFunc(3*time.Second, cancel) + + gone, err := client.confirmItemGone(ctx, client.metadataServiceManager, testSourceRelativePath()) + assert.False(t, gone) + require.Error(t, err) + assert.NotErrorIs(t, err, context.Canceled, "the retries must give up on their own") + assert.Equal(t, int32(confirmRetryAttempts), requests.Load()) // #nosec G115 -- small constant + }) + t.Run("context cancelled during retry wait", func(t *testing.T) { + old, oldAttempts := confirmRetryInterval, confirmRetryAttempts + confirmRetryInterval, confirmRetryAttempts = time.Hour, 5 + t.Cleanup(func() { confirmRetryInterval, confirmRetryAttempts = old, oldAttempts }) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusServiceUnavailable) + })) + defer server.Close() + client, err := NewSourceClient(context.Background(), newTestSourceServerDetails(server.URL)) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + time.AfterFunc(100*time.Millisecond, cancel) + start := time.Now() + gone, err := client.confirmItemGone(ctx, client.metadataServiceManager, testSourceRelativePath()) + assert.False(t, gone) + assert.ErrorIs(t, err, context.Canceled) + assert.Less(t, time.Since(start), 2*time.Second) + }) +} + +// TestConfirmItemGone_itemNonArtifactory404_isRetryableNotGone (N1): a healthy repo root with a +// plain-text/HTML/empty 404 on the item path is not Artifactory's "Unable to find item" answer, +// so the item must not be declared gone. +func TestConfirmItemGone_itemNonArtifactory404_isRetryableNotGone(t *testing.T) { + for name, body := range map[string]string{"plain": "404 page not found", "html": "404", "empty": ""} { + t.Run(name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if storageRequestKind(r) == "root" { + _, _ = w.Write([]byte(testRootBody)) + return + } + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(body)) + })) + defer server.Close() + client, err := NewSourceClient(context.Background(), newTestSourceServerDetails(server.URL)) + require.NoError(t, err) + _, err = client.GetFileMetadata(context.Background(), testSourceFile()) + require.Error(t, err) + assert.False(t, IsSourceItemGone(err), "non-Artifactory 404 body must not prove deletion: %v", err) + }) + } +} + +// --- N2: the repository-root probe must be cheap (bounded read, cached, deduplicated) --- + +const testItemNotFoundBody = `{"errors":[{"status":404,"message":"Unable to find item"}]}` + +// newRootProbeServer answers the repository root with rootHandler and every other storage request +// with an Artifactory 404 (a deleted item). It counts the root requests. +func newRootProbeServer(t *testing.T, rootRequests *atomic.Int32, rootHandler func(w http.ResponseWriter, r *http.Request)) *SourceClient { + t.Helper() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if storageRequestKind(r) == "root" { + rootRequests.Add(1) + rootHandler(w, r) + return + } + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(testItemNotFoundBody)) + })) + t.Cleanup(server.Close) + client, err := NewSourceClient(context.Background(), newTestSourceServerDetails(server.URL)) + require.NoError(t, err) + return client +} + +func setRootProbeCacheTTL(t *testing.T, ttl time.Duration) { + t.Helper() + old := rootProbeCacheTTL + rootProbeCacheTTL = ttl + t.Cleanup(func() { rootProbeCacheTTL = old }) +} + +// TestConfirmItemGone_rootProbeIsCachedPerRepo: thousands of "gone" decisions in one repo must not +// each cost a root request (a first-level children listing on the server). +func TestConfirmItemGone_rootProbeIsCachedPerRepo(t *testing.T) { + setRootProbeCacheTTL(t, time.Minute) + var rootRequests atomic.Int32 + client := newRootProbeServer(t, &rootRequests, func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(testRootBody)) + }) + for i := 0; i < 20; i++ { + gone, err := client.confirmItemGone(context.Background(), client.metadataServiceManager, testSourceRelativePath()) + require.NoError(t, err) + assert.True(t, gone) + } + assert.Equal(t, int32(1), rootRequests.Load(), "a successful probe must be reused within the TTL") +} + +// TestConfirmItemGone_concurrentRootProbesAreDeduplicated: workers that arrive while a probe is in +// flight share it (singleflight), even when nothing is cached yet. +func TestConfirmItemGone_concurrentRootProbesAreDeduplicated(t *testing.T) { + setRootProbeCacheTTL(t, time.Minute) + var rootRequests atomic.Int32 + client := newRootProbeServer(t, &rootRequests, func(w http.ResponseWriter, _ *http.Request) { + time.Sleep(300 * time.Millisecond) + _, _ = w.Write([]byte(testRootBody)) + }) + const workers = 16 + var wg sync.WaitGroup + results := make([]bool, workers) + errs := make([]error, workers) + for i := 0; i < workers; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + results[i], errs[i] = client.confirmItemGone(context.Background(), client.metadataServiceManager, testSourceRelativePath()) + }(i) + } + wg.Wait() + for i := 0; i < workers; i++ { + require.NoError(t, errs[i]) + assert.True(t, results[i]) + } + assert.Equal(t, int32(1), rootRequests.Load()) +} + +// TestConfirmItemGone_rootProbeCacheExpires: the cached success is only valid for the TTL. +func TestConfirmItemGone_rootProbeCacheExpires(t *testing.T) { + setRootProbeCacheTTL(t, 50*time.Millisecond) + var rootRequests atomic.Int32 + client := newRootProbeServer(t, &rootRequests, func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(testRootBody)) + }) + _, err := client.confirmItemGone(context.Background(), client.metadataServiceManager, testSourceRelativePath()) + require.NoError(t, err) + time.Sleep(120 * time.Millisecond) + _, err = client.confirmItemGone(context.Background(), client.metadataServiceManager, testSourceRelativePath()) + require.NoError(t, err) + assert.Equal(t, int32(2), rootRequests.Load()) +} + +// TestConfirmItemGone_failedRootProbeIsNeverCached: only a successful probe may be reused. After a +// failure the next decision probes again, and once the root is healthy "gone" is reachable. +func TestConfirmItemGone_failedRootProbeIsNeverCached(t *testing.T) { + fastConfirmRetry(t) + setRootProbeCacheTTL(t, time.Minute) + var healthy atomic.Bool + var rootRequests atomic.Int32 + client := newRootProbeServer(t, &rootRequests, func(w http.ResponseWriter, _ *http.Request) { + if !healthy.Load() { + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`404 page not found`)) + return + } + _, _ = w.Write([]byte(testRootBody)) + }) + for i := 0; i < 3; i++ { + gone, err := client.confirmItemGone(context.Background(), client.metadataServiceManager, testSourceRelativePath()) + require.Error(t, err) + assert.False(t, gone) + } + assert.Equal(t, int32(3), rootRequests.Load(), "a failed probe must be repeated, not cached") + + healthy.Store(true) + gone, err := client.confirmItemGone(context.Background(), client.metadataServiceManager, testSourceRelativePath()) + require.NoError(t, err) + assert.True(t, gone) +} + +// TestProbeRepositoryRoot_readsAtMostABoundedPrefix: the root answer is a children listing that can +// be tens of MB. The probe must stop reading once it has the "repo" field instead of buffering it. +func TestProbeRepositoryRoot_readsAtMostABoundedPrefix(t *testing.T) { + const totalBytes = 64 << 20 + var written atomic.Int64 + var rootRequests atomic.Int32 + client := newRootProbeServer(t, &rootRequests, func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + n, _ := w.Write([]byte(`{"repo":"` + testSourceRepo + `","children":[`)) + written.Add(int64(n)) + chunk := []byte(strings.Repeat(`{"uri":"/some-child-entry","folder":false},`, 1500)) + for written.Load() < totalBytes && r.Context().Err() == nil { + n, err := w.Write(chunk) + written.Add(int64(n)) + if err != nil { + return + } + } + _, _ = w.Write([]byte(`{}]}`)) + }) + + err := probeRepositoryRoot(context.Background(), client.metadataServiceManager, testSourceRepo) + require.NoError(t, err) + // Let the server notice the closed connection. + time.Sleep(200 * time.Millisecond) + assert.Less(t, written.Load(), int64(totalBytes/2), "the probe must not download the whole listing") +} + +// TestProbeRepositoryRoot_noRepoFieldWithinBoundedPrefix_fails: when the body has no "repo" field +// in the bounded prefix the probe fails (the safe, retryable side) instead of reading without bound. +func TestProbeRepositoryRoot_noRepoFieldWithinBoundedPrefix_fails(t *testing.T) { + var rootRequests atomic.Int32 + client := newRootProbeServer(t, &rootRequests, func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(`{"children":[`)) + chunk := []byte(strings.Repeat(`{"uri":"/some-child-entry","folder":false},`, 1500)) + for i := 0; i < 600 && r.Context().Err() == nil; i++ { + if _, err := w.Write(chunk); err != nil { + return + } + } + _, _ = w.Write([]byte(`{}],"repo":"` + testSourceRepo + `"}`)) + }) + err := probeRepositoryRoot(context.Background(), client.metadataServiceManager, testSourceRepo) + require.Error(t, err) +} + +// --- N3: probe-before-item ordering --- + +// TestConfirmItemGone_probeRunsBeforeItemCheck: the outage window closes at the very first root +// request (the root 404s once, then everything is healthy and the item answers with an Artifactory +// 404). Probing first sees the outage and returns a retryable error; checking the item first would +// see "healthy deleted item", probe the now healthy root and wrongly declare the item gone. +func TestConfirmItemGone_probeRunsBeforeItemCheck(t *testing.T) { + var mu sync.Mutex + var order []string + var rootRequests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + kind := storageRequestKind(r) + mu.Lock() + order = append(order, kind) + mu.Unlock() + switch kind { + case "root": + if rootRequests.Add(1) == 1 { + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`404 page not found`)) + return + } + _, _ = w.Write([]byte(testRootBody)) + default: + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(testItemNotFoundBody)) + } + })) + defer server.Close() + client, err := NewSourceClient(context.Background(), newTestSourceServerDetails(server.URL)) + require.NoError(t, err) + + gone, err := client.confirmItemGone(context.Background(), client.metadataServiceManager, testSourceRelativePath()) + assert.False(t, gone, "the outage visible at the first root request must win") + require.Error(t, err) + mu.Lock() + defer mu.Unlock() + require.NotEmpty(t, order) + assert.Equal(t, "root", order[0], "the control probe must be the first confirmation request") + assert.NotContains(t, order, "item", "no item check once the probe failed") +} + +// --- N4: test-strength gaps --- + +// TestGetFileMetadata_ambiguousPropertiesWaitHonoursContext: the wait between properties +// re-requests must be cancelled with the context. The backoff is an hour, so only a context-aware +// wait returns; a safety timer makes a regression fail fast instead of hanging until the go test +// timeout. +func TestGetFileMetadata_ambiguousPropertiesWaitHonoursContext(t *testing.T) { + old := notFoundRecheckBackoff + notFoundRecheckBackoff = time.Hour + t.Cleanup(func() { notFoundRecheckBackoff = old }) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch storageRequestKind(r) { + case "root": + _, _ = w.Write([]byte(testRootBody)) + case "item": + _, _ = w.Write([]byte(`{"size":"11"}`)) + default: // properties: an ambiguous, non-Artifactory 404 on an item that exists + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`404 page not found`)) + } + })) + defer server.Close() + client, err := NewSourceClient(context.Background(), newTestSourceServerDetails(server.URL)) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + time.AfterFunc(100*time.Millisecond, cancel) + done := make(chan error, 1) + go func() { + _, callErr := client.GetFileMetadata(ctx, testSourceFile()) + done <- callErr + }() + select { + case callErr := <-done: + assert.ErrorIs(t, callErr, context.Canceled) + case <-time.After(5 * time.Second): + t.Fatal("the properties re-request wait ignored context cancellation") + } +} + +// TestConfirmItemGone_retriesAreSpacedByConfirmRetryInterval: consecutive confirmation attempts +// against a failing source must be at least confirmRetryInterval apart, so a restart is not +// hammered by back-to-back requests. +func TestConfirmItemGone_retriesAreSpacedByConfirmRetryInterval(t *testing.T) { + oldInterval, oldAttempts := confirmRetryInterval, confirmRetryAttempts + confirmRetryInterval, confirmRetryAttempts = 100*time.Millisecond, 3 + t.Cleanup(func() { confirmRetryInterval, confirmRetryAttempts = oldInterval, oldAttempts }) + var mu sync.Mutex + var stamps []time.Time + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + mu.Lock() + stamps = append(stamps, time.Now()) + mu.Unlock() + w.WriteHeader(http.StatusServiceUnavailable) + })) + defer server.Close() + client, err := NewSourceClient(context.Background(), newTestSourceServerDetails(server.URL)) + require.NoError(t, err) + + gone, err := client.confirmItemGone(context.Background(), client.metadataServiceManager, testSourceRelativePath()) + assert.False(t, gone) + require.Error(t, err) + mu.Lock() + defer mu.Unlock() + require.Len(t, stamps, 3) + for i := 1; i < len(stamps); i++ { + assert.GreaterOrEqual(t, stamps[i].Sub(stamps[i-1]), 90*time.Millisecond, "attempt %d came too soon after attempt %d", i+1, i) + } +} diff --git a/artifactory/commands/transferfiles/srcpluginservice.go b/artifactory/commands/transferfiles/srcpluginservice.go deleted file mode 100644 index ab605b680..000000000 --- a/artifactory/commands/transferfiles/srcpluginservice.go +++ /dev/null @@ -1,162 +0,0 @@ -package transferfiles - -import ( - "encoding/json" - "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/api" - commandsUtils "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/utils" - "github.com/jfrog/jfrog-client-go/auth" - "github.com/jfrog/jfrog-client-go/http/jfroghttpclient" - "github.com/jfrog/jfrog-client-go/utils/errorutils" - "net/http" -) - -const ( - syncChunks = "syncChunks" - uploadChunk = "uploadChunk" -) - -type VerifyCompatibilityResponse struct { - Version string `json:"version,omitempty"` - Message string `json:"message,omitempty"` -} - -type srcUserPluginService struct { - client *jfroghttpclient.JfrogHttpClient - artDetails *auth.ServiceDetails -} - -func NewSrcUserPluginService(artDetails auth.ServiceDetails, client *jfroghttpclient.JfrogHttpClient) *srcUserPluginService { - return &srcUserPluginService{artDetails: &artDetails, client: client} -} - -func (sup *srcUserPluginService) GetArtifactoryDetails() auth.ServiceDetails { - return *sup.artDetails -} - -func (sup *srcUserPluginService) GetJfrogHttpClient() *jfroghttpclient.JfrogHttpClient { - return sup.client -} - -func (sup *srcUserPluginService) IsDryRun() bool { - return false -} - -func (sup *srcUserPluginService) syncChunks(ucStatus api.UploadChunksStatusBody) (api.UploadChunksStatusResponse, error) { - content, err := json.Marshal(ucStatus) - if err != nil { - return api.UploadChunksStatusResponse{}, errorutils.CheckError(err) - } - - httpDetails := sup.GetArtifactoryDetails().CreateHttpClientDetails() - httpDetails.SetContentTypeApplicationJson() - resp, body, err := sup.client.SendPost(sup.GetArtifactoryDetails().GetUrl()+commandsUtils.PluginsExecuteRestApi+syncChunks, content, &httpDetails) - if err != nil { - return api.UploadChunksStatusResponse{}, err - } - - if err = errorutils.CheckResponseStatusWithBody(resp, body, http.StatusOK); err != nil { - return api.UploadChunksStatusResponse{}, err - } - - var statusResponse api.UploadChunksStatusResponse - err = json.Unmarshal(body, &statusResponse) - if err != nil { - return api.UploadChunksStatusResponse{}, errorutils.CheckError(err) - } - return statusResponse, nil -} - -// Uploads a chunk of files. -// If no error occurred, returns an uuid token to get chunk status with. -func (sup *srcUserPluginService) uploadChunk(chunk api.UploadChunk) (uploadChunkResponse api.UploadChunkResponse, err error) { - content, err := json.Marshal(chunk) - if err != nil { - return api.UploadChunkResponse{}, errorutils.CheckError(err) - } - - httpDetails := sup.GetArtifactoryDetails().CreateHttpClientDetails() - httpDetails.SetContentTypeApplicationJson() - resp, body, err := sup.client.SendPost(sup.GetArtifactoryDetails().GetUrl()+commandsUtils.PluginsExecuteRestApi+uploadChunk, content, &httpDetails) - if err != nil { - return api.UploadChunkResponse{}, err - } - - if err = errorutils.CheckResponseStatusWithBody(resp, body, http.StatusAccepted); err != nil { - return api.UploadChunkResponse{}, err - } - - var uploadResponse api.UploadChunkResponse - err = json.Unmarshal(body, &uploadResponse) - if err != nil { - return api.UploadChunkResponse{}, errorutils.CheckError(err) - } - if uploadResponse.UuidToken == "" { - return api.UploadChunkResponse{}, errorutils.CheckErrorf("unexpected empty token returned for chunk upload") - } - if uploadResponse.NodeId == "" { - return api.UploadChunkResponse{}, errorutils.CheckErrorf("unexpected empty node id returned for chunk upload") - } - return uploadResponse, nil -} - -func (sup *srcUserPluginService) version() (string, error) { - dataTransferVersionUrl := sup.GetArtifactoryDetails().GetUrl() + commandsUtils.PluginsExecuteRestApi + "dataTransferVersion" - httpDetails := sup.GetArtifactoryDetails().CreateHttpClientDetails() - return commandsUtils.GetTransferPluginVersion(sup.client, dataTransferVersionUrl, "data-transfer", commandsUtils.Source, &httpDetails) -} - -func (sup *srcUserPluginService) verifyCompatibilityRequest() (*VerifyCompatibilityResponse, error) { - httpDetails := sup.GetArtifactoryDetails().CreateHttpClientDetails() - httpDetails.SetContentTypeApplicationJson() - resp, body, err := sup.client.SendPost(sup.GetArtifactoryDetails().GetUrl()+commandsUtils.PluginsExecuteRestApi+"verifyCompatibility", []byte("{}"), &httpDetails) - if err != nil { - return nil, err - } - - err = errorutils.CheckResponseStatusWithBody(resp, body, http.StatusOK) - if err != nil { - return nil, err - } - - var result VerifyCompatibilityResponse - err = json.Unmarshal(body, &result) - if err != nil { - return nil, errorutils.CheckError(err) - } - - return &result, nil -} - -func (sup *srcUserPluginService) verifyConnectivityRequest(targetAuth api.TargetAuth) error { - httpDetails := sup.GetArtifactoryDetails().CreateHttpClientDetails() - httpDetails.SetContentTypeApplicationJson() - content, err := json.Marshal(targetAuth) - if err != nil { - return errorutils.CheckError(err) - } - resp, body, err := sup.client.SendPost(sup.GetArtifactoryDetails().GetUrl()+commandsUtils.PluginsExecuteRestApi+"verifySourceTargetConnectivity", content, &httpDetails) - if err != nil { - return err - } - - return errorutils.CheckResponseStatusWithBody(resp, body, http.StatusOK) -} - -func (sup *srcUserPluginService) stop() (nodeId string, err error) { - httpDetails := sup.GetArtifactoryDetails().CreateHttpClientDetails() - resp, body, err := sup.client.SendPost(sup.GetArtifactoryDetails().GetUrl()+commandsUtils.PluginsExecuteRestApi+"stop", []byte{}, &httpDetails) - if err != nil { - return "", err - } - - if err = errorutils.CheckResponseStatus(resp, http.StatusOK); err != nil { - return "", err - } - - var result api.NodeIdResponse - err = json.Unmarshal(body, &result) - if err != nil { - return "", errorutils.CheckError(err) - } - return result.NodeId, nil -} diff --git a/artifactory/commands/transferfiles/state/runstatus.go b/artifactory/commands/transferfiles/state/runstatus.go index 2c6924f6c..ec49c0ea4 100644 --- a/artifactory/commands/transferfiles/state/runstatus.go +++ b/artifactory/commands/transferfiles/state/runstatus.go @@ -15,7 +15,9 @@ import ( // Can be used to identify when the version of the CLI doesn't support the structure of the transfer directory. const transferRunStatusVersion = 1 -var saveRunStatusMutex sync.Mutex +var saveRunStatusMutex sync.RWMutex +var workingThreadsMutex sync.RWMutex +var runStatusCountersMutex sync.RWMutex type ActionOnStatusFunc func(transferRunStatus *TransferRunStatus) error @@ -41,8 +43,11 @@ type TransferRunStatus struct { VisitedFolders uint64 `json:"visited_folders,omitempty"` DelayedFiles uint64 `json:"delayed_files,omitempty"` TransferFailures uint64 `json:"transfer_failures,omitempty"` + SkippedSourceGone uint64 `json:"skipped_source_gone,omitempty"` TimeEstimationManager `json:"time_estimation,omitempty"` - StaleChunks []StaleChunks `json:"stale_chunks,omitempty"` + // StaleChunks is retained for on-disk run-status.json schema compatibility. + // GET/PUT transfer no longer tracks stale plugin chunks. + StaleChunks []StaleChunks `json:"stale_chunks,omitempty"` } // This structure contains a collection of chunks that have been undergoing processing for over 30 minutes @@ -62,8 +67,10 @@ func (ts *TransferRunStatus) action(action ActionOnStatusFunc) error { return err } - now := time.Now() - if now.Sub(ts.lastSaveTimestamp).Seconds() < float64(stateAndStatusSaveIntervalSecs) { + saveRunStatusMutex.RLock() + sinceLastSave := time.Since(ts.lastSaveTimestamp).Seconds() + saveRunStatusMutex.RUnlock() + if sinceLastSave < float64(stateAndStatusSaveIntervalSecs) { return nil } @@ -72,7 +79,7 @@ func (ts *TransferRunStatus) action(action ActionOnStatusFunc) error { } defer saveRunStatusMutex.Unlock() - ts.lastSaveTimestamp = now + ts.lastSaveTimestamp = time.Now() return ts.persistTransferRunStatus() } @@ -83,7 +90,18 @@ func (ts *TransferRunStatus) persistTransferRunStatus() (err error) { } ts.Version = transferRunStatusVersion - content, err := json.Marshal(ts) + // Lock order: saveRunStatusMutex (held by action) -> workingThreadsMutex -> + // timeEstimationMutex -> runStatusCountersMutex. Time-estimation code must + // not call back into TransferRunStatus.action while locked. + content, err := func() ([]byte, error) { + workingThreadsMutex.RLock() + defer workingThreadsMutex.RUnlock() + timeEstimationMutex.RLock() + defer timeEstimationMutex.RUnlock() + runStatusCountersMutex.RLock() + defer runStatusCountersMutex.RUnlock() + return json.Marshal(ts) + }() if err != nil { return errorutils.CheckError(err) } diff --git a/artifactory/commands/transferfiles/state/runstatus_test.go b/artifactory/commands/transferfiles/state/runstatus_test.go index 3fbd396e3..24ca282ec 100644 --- a/artifactory/commands/transferfiles/state/runstatus_test.go +++ b/artifactory/commands/transferfiles/state/runstatus_test.go @@ -1,8 +1,10 @@ package state import ( + "os" "testing" + "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" "github.com/stretchr/testify/assert" ) @@ -20,3 +22,42 @@ func TestSaveAndLoadRunStatus(t *testing.T) { actualStatus.stateManager = stateManager assert.Equal(t, stateManager.TransferRunStatus, actualStatus) } + +// A run-status.json written by a pre-GET/PUT CLI version (plugin-based transfer) still carries a +// populated "stale_chunks" field. GET/PUT transfer no longer produces or displays this data, but the +// field is kept on TransferRunStatus for schema compatibility, so loading such a file must still +// succeed and round-trip the legacy value rather than erroring out or silently dropping it. +func TestLoadRunStatus_legacyStaleChunksFieldParses(t *testing.T) { + _, cleanUp := InitStateTest(t) + defer cleanUp() + + legacyRunStatus := `{ + "version": 1, + "current_repo": "repo1", + "stale_chunks": [ + { + "node_id": "node-1", + "stale_node_chunks": [ + {"chunk_id": "chunk-1", "files": ["a.txt", "b.txt"], "sent": 1690000000} + ] + } + ] + }` + statusFilePath, err := coreutils.GetJfrogTransferRunStatusFilePath() + assert.NoError(t, err) + assert.NoError(t, os.WriteFile(statusFilePath, []byte(legacyRunStatus), 0600)) + + actualStatus, exists, err := loadTransferRunStatus() + assert.NoError(t, err) + assert.True(t, exists) + assert.Equal(t, transferRunStatusVersion, actualStatus.Version) + assert.Equal(t, "repo1", actualStatus.CurrentRepoKey) + assert.Equal(t, []StaleChunks{ + { + NodeID: "node-1", + Chunks: []StaleChunk{ + {ChunkID: "chunk-1", Files: []string{"a.txt", "b.txt"}, Sent: 1690000000}, + }, + }, + }, actualStatus.StaleChunks) +} diff --git a/artifactory/commands/transferfiles/state/state.go b/artifactory/commands/transferfiles/state/state.go index 02c5a94f4..42306f1d0 100644 --- a/artifactory/commands/transferfiles/state/state.go +++ b/artifactory/commands/transferfiles/state/state.go @@ -12,7 +12,8 @@ import ( "time" ) -var saveStateMutex sync.Mutex +var saveStateMutex sync.RWMutex +var stateCountersMutex sync.RWMutex type ActionOnStateFunc func(state *TransferState) error @@ -67,8 +68,10 @@ func (ts *TransferState) Action(action ActionOnStateFunc) error { return err } - now := time.Now() - if now.Sub(ts.lastSaveTimestamp).Seconds() < float64(stateAndStatusSaveIntervalSecs) { + saveStateMutex.RLock() + sinceLastSave := time.Since(ts.lastSaveTimestamp).Seconds() + saveStateMutex.RUnlock() + if sinceLastSave < float64(stateAndStatusSaveIntervalSecs) { return nil } @@ -77,7 +80,7 @@ func (ts *TransferState) Action(action ActionOnStateFunc) error { } defer saveStateMutex.Unlock() - ts.lastSaveTimestamp = now + ts.lastSaveTimestamp = time.Now() return ts.persistTransferState(false) } @@ -88,7 +91,11 @@ func (ts *TransferState) persistTransferState(snapshot bool) (err error) { return err } - content, err := json.Marshal(ts) + content, err := func() ([]byte, error) { + stateCountersMutex.RLock() + defer stateCountersMutex.RUnlock() + return json.Marshal(ts) + }() if err != nil { return errorutils.CheckError(err) } diff --git a/artifactory/commands/transferfiles/state/state_test.go b/artifactory/commands/transferfiles/state/state_test.go index ce2020848..c5bd10969 100644 --- a/artifactory/commands/transferfiles/state/state_test.go +++ b/artifactory/commands/transferfiles/state/state_test.go @@ -1,6 +1,7 @@ package state import ( + "sync" "testing" "time" @@ -107,6 +108,33 @@ func setAutoSaveSnapshot(interval int) (cleanUp func()) { } } +func TestSnapshotAction_concurrentTimestampAccess(t *testing.T) { + stateManager, cleanUp := InitStateTest(t) + defer cleanUp() + defer setAutoSaveSnapshot(0)() + + assert.NoError(t, stateManager.SetRepoState(repo1Key, 0, 0, false, true)) + stateManager.repoTransferSnapshot.lastSaveTimestamp = time.Time{} + + const callers = 32 + errs := make(chan error, callers) + var wg sync.WaitGroup + wg.Add(callers) + for i := 0; i < callers; i++ { + go func() { + defer wg.Done() + _, err := stateManager.LookUpNode(".") + errs <- err + }() + } + wg.Wait() + close(errs) + + for err := range errs { + assert.NoError(t, err) + } +} + // Testing two scenarios here: // 1. loading (snapshot file and state) from existing snapshot // 2. loading state of fully transferred repository (phase 1 completed) diff --git a/artifactory/commands/transferfiles/state/statemanager.go b/artifactory/commands/transferfiles/state/statemanager.go index 461d7122b..cc2deeea3 100644 --- a/artifactory/commands/transferfiles/state/statemanager.go +++ b/artifactory/commands/transferfiles/state/statemanager.go @@ -112,15 +112,16 @@ func (ts *TransferStateManager) SetRepoState(repoKey string, totalSizeBytes, tot return err } return ts.action(func(transferRunStatus *TransferRunStatus) error { - transferRunStatus.CurrentRepoKey = repoKey - transferRunStatus.BuildInfoRepo = buildInfoRepo - transferRunStatus.VisitedFolders = 0 - - transferRunStatus.OverallTransfer.TransferredUnits += int64(transferredFiles) signedTransferredSizeBytes, err := safeconvert.Uint64ToInt64(transferredSizeBytes) if err != nil { return fmt.Errorf("failed to set transferred size bytes: %w", err) } + runStatusCountersMutex.Lock() + defer runStatusCountersMutex.Unlock() + transferRunStatus.CurrentRepoKey = repoKey + transferRunStatus.BuildInfoRepo = buildInfoRepo + transferRunStatus.VisitedFolders = 0 + transferRunStatus.OverallTransfer.TransferredUnits += int64(transferredFiles) transferRunStatus.OverallTransfer.TransferredSizeBytes += signedTransferredSizeBytes return nil }) @@ -147,19 +148,23 @@ func (ts *TransferStateManager) SetRepoFullTransferCompleted() error { // Increasing Transferred Diff files (modified files) and SizeByBytes value in suitable repository progress state func (ts *TransferStateManager) IncTransferredSizeAndFilesPhase1(chunkTotalFiles, chunkTotalSizeInBytes int64) error { err := ts.Action(func(state *TransferState) error { - atomicallyAddInt64(&state.CurrentRepo.Phase1Info.TransferredSizeBytes, chunkTotalSizeInBytes) - atomicallyAddInt64(&state.CurrentRepo.Phase1Info.TransferredUnits, chunkTotalFiles) + stateCountersMutex.Lock() + defer stateCountersMutex.Unlock() + state.CurrentRepo.Phase1Info.TransferredSizeBytes += chunkTotalSizeInBytes + state.CurrentRepo.Phase1Info.TransferredUnits += chunkTotalFiles return nil }) if err != nil { return err } return ts.action(func(transferRunStatus *TransferRunStatus) error { - atomicallyAddInt64(&transferRunStatus.OverallTransfer.TransferredSizeBytes, chunkTotalSizeInBytes) - atomicallyAddInt64(&transferRunStatus.OverallTransfer.TransferredUnits, chunkTotalFiles) + runStatusCountersMutex.Lock() + defer runStatusCountersMutex.Unlock() + transferRunStatus.OverallTransfer.TransferredSizeBytes += chunkTotalSizeInBytes + transferRunStatus.OverallTransfer.TransferredUnits += chunkTotalFiles if transferRunStatus.BuildInfoRepo { - atomicallyAddInt64(&transferRunStatus.OverallBiFiles.TransferredUnits, chunkTotalFiles) + transferRunStatus.OverallBiFiles.TransferredUnits += chunkTotalFiles } return nil }) @@ -167,16 +172,20 @@ func (ts *TransferStateManager) IncTransferredSizeAndFilesPhase1(chunkTotalFiles func (ts *TransferStateManager) IncTransferredSizeAndFilesPhase2(chunkTotalFiles, chunkTotalSizeInBytes int64) error { return ts.Action(func(state *TransferState) error { - atomicallyAddInt64(&state.CurrentRepo.Phase2Info.TransferredSizeBytes, chunkTotalSizeInBytes) - atomicallyAddInt64(&state.CurrentRepo.Phase2Info.TransferredUnits, chunkTotalFiles) + stateCountersMutex.Lock() + defer stateCountersMutex.Unlock() + state.CurrentRepo.Phase2Info.TransferredSizeBytes += chunkTotalSizeInBytes + state.CurrentRepo.Phase2Info.TransferredUnits += chunkTotalFiles return nil }) } func (ts *TransferStateManager) IncTotalSizeAndFilesPhase2(filesNumber, totalSize int64) error { return ts.Action(func(state *TransferState) error { - atomicallyAddInt64(&state.CurrentRepo.Phase2Info.TotalSizeBytes, totalSize) - atomicallyAddInt64(&state.CurrentRepo.Phase2Info.TotalUnits, filesNumber) + stateCountersMutex.Lock() + defer stateCountersMutex.Unlock() + state.CurrentRepo.Phase2Info.TotalSizeBytes += totalSize + state.CurrentRepo.Phase2Info.TotalUnits += filesNumber return nil }) } @@ -184,10 +193,12 @@ func (ts *TransferStateManager) IncTotalSizeAndFilesPhase2(filesNumber, totalSiz // Set relevant information of files and storage we need to transfer in phase3 func (ts *TransferStateManager) SetTotalSizeAndFilesPhase3(filesNumber, totalSize int64) error { return ts.Action(func(state *TransferState) error { + stateCountersMutex.Lock() + defer stateCountersMutex.Unlock() state.CurrentRepo.Phase3Info.TransferredUnits = 0 state.CurrentRepo.Phase3Info.TransferredSizeBytes = 0 - atomicallyAddInt64(&state.CurrentRepo.Phase3Info.TotalSizeBytes, totalSize) - atomicallyAddInt64(&state.CurrentRepo.Phase3Info.TotalUnits, filesNumber) + state.CurrentRepo.Phase3Info.TotalSizeBytes += totalSize + state.CurrentRepo.Phase3Info.TotalUnits += filesNumber return nil }) } @@ -195,8 +206,10 @@ func (ts *TransferStateManager) SetTotalSizeAndFilesPhase3(filesNumber, totalSiz // Increase transferred storage and files in phase 3 func (ts *TransferStateManager) IncTransferredSizeAndFilesPhase3(chunkTotalFiles, chunkTotalSizeInBytes int64) error { return ts.Action(func(state *TransferState) error { - atomicallyAddInt64(&state.CurrentRepo.Phase3Info.TransferredSizeBytes, chunkTotalSizeInBytes) - atomicallyAddInt64(&state.CurrentRepo.Phase3Info.TransferredUnits, chunkTotalFiles) + stateCountersMutex.Lock() + defer stateCountersMutex.Unlock() + state.CurrentRepo.Phase3Info.TransferredSizeBytes += chunkTotalSizeInBytes + state.CurrentRepo.Phase3Info.TransferredUnits += chunkTotalFiles return nil }) } @@ -302,27 +315,53 @@ func (ts *TransferStateManager) GetDiffHandlingRange() (start, end time.Time, er func (ts *TransferStateManager) IncVisitedFolders() error { return ts.action(func(transferRunStatus *TransferRunStatus) error { - atomicallyAddUint64(&transferRunStatus.VisitedFolders, 1, true) + runStatusCountersMutex.Lock() + defer runStatusCountersMutex.Unlock() + transferRunStatus.VisitedFolders++ return nil }) } func (ts *TransferStateManager) ChangeDelayedFilesCountBy(count uint64, increase bool) error { return ts.action(func(transferRunStatus *TransferRunStatus) error { - atomicallyAddUint64(&transferRunStatus.DelayedFiles, count, increase) + runStatusCountersMutex.Lock() + defer runStatusCountersMutex.Unlock() + if increase { + transferRunStatus.DelayedFiles += count + } else { + transferRunStatus.DelayedFiles -= count + } return nil }) } func (ts *TransferStateManager) ChangeTransferFailureCountBy(count uint64, increase bool) error { return ts.action(func(transferRunStatus *TransferRunStatus) error { - atomicallyAddUint64(&transferRunStatus.TransferFailures, count, increase) + runStatusCountersMutex.Lock() + defer runStatusCountersMutex.Unlock() + if increase { + transferRunStatus.TransferFailures += count + } else { + transferRunStatus.TransferFailures -= count + } + return nil + }) +} + +// IncSkippedSourceGone counts items skipped because they no longer exist in the source. +func (ts *TransferStateManager) IncSkippedSourceGone(count uint64) error { + return ts.action(func(transferRunStatus *TransferRunStatus) error { + runStatusCountersMutex.Lock() + defer runStatusCountersMutex.Unlock() + transferRunStatus.SkippedSourceGone += count return nil }) } func (ts *TransferStateManager) IncRepositoriesTransferred() error { return ts.action(func(transferRunStatus *TransferRunStatus) error { + runStatusCountersMutex.Lock() + defer runStatusCountersMutex.Unlock() transferRunStatus.TotalRepositories.TransferredUnits++ return nil }) @@ -330,6 +369,8 @@ func (ts *TransferStateManager) IncRepositoriesTransferred() error { func (ts *TransferStateManager) SetRepoPhase(phaseId int) error { return ts.action(func(transferRunStatus *TransferRunStatus) error { + runStatusCountersMutex.Lock() + defer runStatusCountersMutex.Unlock() transferRunStatus.CurrentRepoPhase = phaseId return nil }) @@ -337,34 +378,23 @@ func (ts *TransferStateManager) SetRepoPhase(phaseId int) error { func (ts *TransferStateManager) SetWorkingThreads(workingThreads int) error { return ts.action(func(transferRunStatus *TransferRunStatus) error { + workingThreadsMutex.Lock() + defer workingThreadsMutex.Unlock() transferRunStatus.WorkingThreads = workingThreads return nil }) } -func (ts *TransferStateManager) GetWorkingThreads() (workingThreads int, err error) { - return workingThreads, ts.action(func(transferRunStatus *TransferRunStatus) error { - workingThreads = transferRunStatus.WorkingThreads - return nil - }) -} - -func (ts *TransferStateManager) SetStaleChunks(staleChunks []StaleChunks) error { - return ts.action(func(transferRunStatus *TransferRunStatus) error { - transferRunStatus.StaleChunks = staleChunks - return nil - }) -} - -func (ts *TransferStateManager) GetStaleChunks() (staleChunks []StaleChunks, err error) { - return staleChunks, ts.action(func(transferRunStatus *TransferRunStatus) error { - staleChunks = transferRunStatus.StaleChunks - return nil - }) +func (ts *TransferStateManager) GetWorkingThreads() (int, error) { + workingThreadsMutex.RLock() + defer workingThreadsMutex.RUnlock() + return ts.WorkingThreads, nil } func (ts *TransferStateManager) SaveStateAndSnapshots() error { + saveStateMutex.Lock() ts.TransferState.lastSaveTimestamp = time.Now() + saveStateMutex.Unlock() if err := ts.persistTransferState(false); err != nil { return err } @@ -372,7 +402,9 @@ func (ts *TransferStateManager) SaveStateAndSnapshots() error { if ts.repoTransferSnapshot == nil { return nil } + saveRepoSnapshotMutex.Lock() ts.repoTransferSnapshot.lastSaveTimestamp = time.Now() + saveRepoSnapshotMutex.Unlock() if err := ts.repoTransferSnapshot.snapshotManager.PersistRepoSnapshot(); err != nil { return err } @@ -446,13 +478,26 @@ func (ts *TransferStateManager) GetRunningTimeString() (runningTime string) { func UpdateChunkInState(stateManager *TransferStateManager, chunk *api.ChunkStatus) (err error) { var chunkTotalSizeInBytes int64 = 0 var chunkTotalFiles int64 = 0 + var chunkSkippedGone uint64 for _, file := range chunk.Files { + if file.Status == api.SkippedSourceItemGone { + // The item vanished from the source: it was not transferred, count it separately. + if file.Name != "" { + chunkSkippedGone++ + } + continue + } if file.Status != api.Fail && file.Name != "" { // Count only successfully transferred files chunkTotalSizeInBytes += file.SizeBytes chunkTotalFiles++ } } + if chunkSkippedGone > 0 { + if err = stateManager.IncSkippedSourceGone(chunkSkippedGone); err != nil { + return err + } + } switch stateManager.CurrentRepoPhase { case api.Phase1: err = stateManager.IncTransferredSizeAndFilesPhase1(chunkTotalFiles, chunkTotalSizeInBytes) diff --git a/artifactory/commands/transferfiles/state/statemanager_test.go b/artifactory/commands/transferfiles/state/statemanager_test.go index ba83815ad..4e0d833a9 100644 --- a/artifactory/commands/transferfiles/state/statemanager_test.go +++ b/artifactory/commands/transferfiles/state/statemanager_test.go @@ -2,6 +2,7 @@ package state import ( "github.com/jfrog/gofrog/safeconvert" + "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/api" "sync" "testing" "time" @@ -53,6 +54,29 @@ func TestPhase2CompletedPersistedAfterSaveStateAndSnapshots(t *testing.T) { assert.True(t, loaded.CurrentRepo.Diffs[0].Completed, "Phase 2 completed=true must be persisted to disk after SaveStateAndSnapshots") } +// TestUpdateChunkInState_sourceGoneIsNotCountedAsTransferred: items skipped because they no longer +// exist in the source must not inflate the transferred-units count (which made 39 lost files out +// of 60 show up as 60/60); they are counted separately. +func TestUpdateChunkInState_sourceGoneIsNotCountedAsTransferred(t *testing.T) { + stateManager, cleanUp := InitStateTest(t) + defer cleanUp() + assert.NoError(t, stateManager.SetRepoState(repo1Key, 0, 3, false, true)) + assert.NoError(t, stateManager.SetRepoPhase(api.Phase1)) + stateManager.CurrentRepoPhase = api.Phase1 + + chunk := api.ChunkStatus{Files: []api.FileUploadStatusResponse{ + {FileRepresentation: api.FileRepresentation{Repo: repo1Key, Path: "a", Name: "ok.bin"}, Status: api.Success, SizeBytes: 10}, + {FileRepresentation: api.FileRepresentation{Repo: repo1Key, Path: "a", Name: "gone1.bin"}, Status: api.SkippedSourceItemGone}, + {FileRepresentation: api.FileRepresentation{Repo: repo1Key, Path: "a", Name: "gone2.bin"}, Status: api.SkippedSourceItemGone}, + }} + assert.NoError(t, UpdateChunkInState(stateManager, &chunk)) + + assert.Equal(t, int64(1), stateManager.CurrentRepo.Phase1Info.TransferredUnits) + assert.Equal(t, int64(10), stateManager.CurrentRepo.Phase1Info.TransferredSizeBytes) + assert.Equal(t, int64(1), stateManager.OverallTransfer.TransferredUnits) + assert.Equal(t, uint64(2), stateManager.SkippedSourceGone) +} + func assertRepoTransferred(t *testing.T, stateManager *TransferStateManager, expected bool) { transferred, err := stateManager.IsRepoTransferred() assert.NoError(t, err) diff --git a/artifactory/commands/transferfiles/state/timeestimation.go b/artifactory/commands/transferfiles/state/timeestimation.go index 9e17c01d6..4a9d1fe2c 100644 --- a/artifactory/commands/transferfiles/state/timeestimation.go +++ b/artifactory/commands/transferfiles/state/timeestimation.go @@ -3,9 +3,10 @@ package state import ( "errors" "fmt" - "github.com/jfrog/gofrog/safeconvert" + "sync" "time" + "github.com/jfrog/gofrog/safeconvert" "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/api" "github.com/jfrog/jfrog-client-go/artifactory/services/utils" @@ -27,6 +28,7 @@ const ( ) var numOfSpeedsToKeepPerWorkingThread = 10 +var timeEstimationMutex sync.RWMutex type TimeEstimationManager struct { // Speeds of the last done chunks, in bytes/ms @@ -41,39 +43,65 @@ type TimeEstimationManager struct { stateManager *TransferStateManager } +func (tem *TimeEstimationManager) ResetCurrentTotalTransferredBytes() { + timeEstimationMutex.Lock() + defer timeEstimationMutex.Unlock() + + tem.CurrentTotalTransferredBytes = 0 +} + func (tem *TimeEstimationManager) AddChunkStatus(chunkStatus api.ChunkStatus, durationMillis int64) error { - if durationMillis == 0 { + chunkSizeBytes := chunkSizeBytesForSpeed(chunkStatus) + var workingThreads int + var err error + if durationMillis != 0 && chunkSizeBytes != 0 { + // Read working threads before taking timeEstimationMutex. Status persistence + // takes saveRunStatusMutex -> workingThreadsMutex -> timeEstimationMutex. + workingThreads, err = tem.stateManager.GetWorkingThreads() + if err != nil { + log.Error("Couldn't calculate time estimation:", err.Error()) + return err + } + } + + timeEstimationMutex.Lock() + defer timeEstimationMutex.Unlock() + + if err := tem.addTransferredBytesFromChunk(chunkStatus); err != nil { + return err + } + if durationMillis == 0 || chunkSizeBytes == 0 { return nil } + tem.addSpeedSampleFromChunk(workingThreads, chunkSizeBytes, durationMillis) + return nil +} - return tem.addDataChunkStatus(chunkStatus, durationMillis) +func (tem *TimeEstimationManager) addTransferredBytesFromChunk(chunkStatus api.ChunkStatus) error { + for _, file := range chunkStatus.Files { + if file.Status == api.Fail { + continue + } + unsignedSizeBytes, err := safeconvert.Int64ToUint64(file.SizeBytes) + if err != nil { + return fmt.Errorf("failed to calculate the estimated remaining time: %w", err) + } + tem.CurrentTotalTransferredBytes += unsignedSizeBytes + } + return nil } -func (tem *TimeEstimationManager) addDataChunkStatus(chunkStatus api.ChunkStatus, durationMillis int64) error { +func chunkSizeBytesForSpeed(chunkStatus api.ChunkStatus) int64 { var chunkSizeBytes int64 for _, file := range chunkStatus.Files { - if file.Status != api.Fail { - unsignedSizeBytes, err := safeconvert.Int64ToUint64(file.SizeBytes) - if err != nil { - return fmt.Errorf("failed to calculate the estimated remaining time: %w", err) - } - tem.CurrentTotalTransferredBytes += unsignedSizeBytes - } if (file.Status == api.Success || file.Status == api.SkippedLargeProps) && !file.ChecksumDeployed { chunkSizeBytes += file.SizeBytes } } + return chunkSizeBytes +} - // If no files were uploaded regularly (with no errors and not checksum-deployed), don't use this chunk for the time estimation calculation. - if chunkSizeBytes == 0 { - return nil - } - - workingThreads, err := tem.stateManager.GetWorkingThreads() - if err != nil { - log.Error("Couldn't calculate time estimation:", err.Error()) - return err - } +func (tem *TimeEstimationManager) addSpeedSampleFromChunk(workingThreads int, chunkSizeBytes, durationMillis int64) { speed := calculateChunkSpeed(workingThreads, chunkSizeBytes, durationMillis) tem.LastSpeeds = append(tem.LastSpeeds, speed) tem.LastSpeedsSum += speed @@ -85,11 +113,10 @@ func (tem *TimeEstimationManager) addDataChunkStatus(chunkStatus api.ChunkStatus } if len(tem.LastSpeeds) == 0 { tem.SpeedsAverage = 0 - return err + return } // Calculate speed in bytes/ms tem.SpeedsAverage = tem.LastSpeedsSum / float64(len(tem.LastSpeeds)) - return nil } func calculateChunkSpeed(workingThreads int, chunkSizeSum, chunkDuration int64) float64 { @@ -104,6 +131,9 @@ func (tem *TimeEstimationManager) getSpeed() float64 { // GetSpeedString gets the transfer speed as an easy-to-read string. func (tem *TimeEstimationManager) GetSpeedString() string { + timeEstimationMutex.RLock() + defer timeEstimationMutex.RUnlock() + if len(tem.LastSpeeds) == 0 { return "Not available yet" } @@ -116,6 +146,9 @@ func (tem *TimeEstimationManager) GetSpeedString() string { // 2. No files transferred // 3. The transfer speed is less than 1 byte per second func (tem *TimeEstimationManager) GetEstimatedRemainingTimeString() (string, error) { + timeEstimationMutex.RLock() + defer timeEstimationMutex.RUnlock() + remainingTimeSec, err := tem.getEstimatedRemainingSeconds() if remainingTimeSec == 0 || err != nil { return "Not available yet", err @@ -144,7 +177,9 @@ func (tem *TimeEstimationManager) getEstimatedRemainingSeconds() (uint64, error) // Less than 1 byte per second return 0, nil } + runStatusCountersMutex.RLock() remainingBytes := tem.stateManager.OverallTransfer.TotalSizeBytes - tem.stateManager.OverallTransfer.TransferredSizeBytes + runStatusCountersMutex.RUnlock() unsignedRemainingBytes, err := safeconvert.Int64ToUint64(remainingBytes) if err != nil { return 0, fmt.Errorf("failed to calculate the estimated remaining time: %w", err) diff --git a/artifactory/commands/transferfiles/state/timeestimation_test.go b/artifactory/commands/transferfiles/state/timeestimation_test.go index b9b23d8fc..fe1627e08 100644 --- a/artifactory/commands/transferfiles/state/timeestimation_test.go +++ b/artifactory/commands/transferfiles/state/timeestimation_test.go @@ -1,11 +1,11 @@ package state import ( - "github.com/jfrog/gofrog/safeconvert" "testing" "time" "github.com/jfrog/build-info-go/utils" + "github.com/jfrog/gofrog/safeconvert" rtServicesUtils "github.com/jfrog/jfrog-client-go/artifactory/services/utils" "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" @@ -232,6 +232,38 @@ func TestTransferredSizeInState(t *testing.T) { assertTransferredSizes(t, timeEstMng.stateManager, chunkStatus1.Files[0].SizeBytes+chunkStatus1.Files[1].SizeBytes, chunkStatus3.Files[0].SizeBytes+chunkStatus4.Files[0].SizeBytes) } +func TestAddChunkStatus_zeroDurationChecksumHitCountsBytes(t *testing.T) { + timeEstMng, cleanUp := initTimeEstimationDataTest(t) + defer cleanUp() + + const fileSize = 8 * rtServicesUtils.SizeMiB + chunkStatus := api.ChunkStatus{ + Files: []api.FileUploadStatusResponse{ + createFileUploadStatusResponse(repo1Key, fileSize, true, api.Success), + }, + } + assert.NoError(t, timeEstMng.AddChunkStatus(chunkStatus, 0)) + assert.Equal(t, uint64(fileSize), timeEstMng.CurrentTotalTransferredBytes) + assert.Empty(t, timeEstMng.LastSpeeds) +} + +func TestTimeEstimationManagerResetCurrentTotalTransferredBytes(t *testing.T) { + timeEstMng, cleanUp := initTimeEstimationDataTest(t) + defer cleanUp() + + timeEstMng.LastSpeeds = []float64{1} + timeEstMng.LastSpeedsSum = 1 + timeEstMng.SpeedsAverage = 1 + timeEstMng.CurrentTotalTransferredBytes = 1 + + timeEstMng.ResetCurrentTotalTransferredBytes() + + assert.Equal(t, []float64{1}, timeEstMng.LastSpeeds) + assert.Equal(t, float64(1), timeEstMng.LastSpeedsSum) + assert.Equal(t, float64(1), timeEstMng.SpeedsAverage) + assert.Zero(t, timeEstMng.CurrentTotalTransferredBytes) +} + func addChunkStatus(t *testing.T, timeEstMng *TimeEstimationManager, chunkStatus api.ChunkStatus, workingThreads int, includedInTotalSize bool, durationMillis int64) { if includedInTotalSize { err := UpdateChunkInState(timeEstMng.stateManager, &chunkStatus) diff --git a/artifactory/commands/transferfiles/state/transfersnapshot.go b/artifactory/commands/transferfiles/state/transfersnapshot.go index cb1a66a85..0970d43bd 100644 --- a/artifactory/commands/transferfiles/state/transfersnapshot.go +++ b/artifactory/commands/transferfiles/state/transfersnapshot.go @@ -35,16 +35,15 @@ func (ts *TransferStateManager) snapshotAction(action SnapshotActionFunc) (err e return err } - now := time.Now() - if now.Sub(ts.repoTransferSnapshot.lastSaveTimestamp).Minutes() < float64(snapshotSaveIntervalMin) { - return nil - } - if !saveRepoSnapshotMutex.TryLock() { return nil } defer saveRepoSnapshotMutex.Unlock() + now := time.Now() + if now.Sub(ts.repoTransferSnapshot.lastSaveTimestamp).Minutes() < float64(snapshotSaveIntervalMin) { + return nil + } ts.repoTransferSnapshot.lastSaveTimestamp = now if err = ts.repoTransferSnapshot.snapshotManager.PersistRepoSnapshot(); err != nil { return err diff --git a/artifactory/commands/transferfiles/state/utils.go b/artifactory/commands/transferfiles/state/utils.go index 2cb401536..146d35dd6 100644 --- a/artifactory/commands/transferfiles/state/utils.go +++ b/artifactory/commands/transferfiles/state/utils.go @@ -6,7 +6,6 @@ import ( "path/filepath" "strconv" "strings" - "sync/atomic" "time" "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" @@ -122,22 +121,3 @@ func GetOldTransferDirectoryStructureError() error { } return errorutils.CheckErrorf(oldTransferDirectoryStructureErrorFormat, transferDir) } - -// Atomically add to an int64 variable. -// addr - Pointer to int64 variable -// delta - The change to do -func atomicallyAddInt64(addr *int64, delta int64) { - atomic.AddInt64(addr, delta) -} - -// Atomically add to an uint64 variable. -// addr - Pointer to uint64 variable -// delta - The change to do -// increase - True to increment, false to decrement -func atomicallyAddUint64(addr *uint64, delta uint64, increase bool) { - if increase { - atomic.AddUint64(addr, delta) - } else { - atomic.AddUint64(addr, ^(delta - 1)) - } -} diff --git a/artifactory/commands/transferfiles/status.go b/artifactory/commands/transferfiles/status.go index 50c8069d0..aa47f9809 100644 --- a/artifactory/commands/transferfiles/status.go +++ b/artifactory/commands/transferfiles/status.go @@ -5,7 +5,6 @@ import ( "path/filepath" "strconv" "strings" - "time" "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/api" "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/state" @@ -59,7 +58,6 @@ func ShowStatus() error { output.WriteString("\n") setRepositoryStatus(stateManager, &output) } - addStaleChunks(stateManager, &output) log.Output(output.String()) return nil } @@ -91,6 +89,9 @@ func addOverallStatus(stateManager *state.TransferStateManager, output *strings. failureTxt += " (" + progressbar.RetryFailureContentNote + ")" } addString(output, "❌", "Transfer failures", failureTxt, 2) + if stateManager.SkippedSourceGone > 0 { + addString(output, "⏭ ", "Skipped (deleted at source)", strconv.FormatUint(stateManager.SkippedSourceGone, 10)+" (items that no longer exist in the source; listed in the errors summary CSV, not retried)", 1) + } return nil } @@ -127,27 +128,6 @@ func setRepositoryStatus(stateManager *state.TransferStateManager, output *strin addString(output, "✋", "Delayed files", delayedTxt, 2) } -func addStaleChunks(stateManager *state.TransferStateManager, output *strings.Builder) { - if len(stateManager.StaleChunks) == 0 { - return - } - output.WriteString("\n") - addTitle(output, "File Chunks in Transit for More than 30 Minutes") - - for _, nodeStaleChunks := range stateManager.StaleChunks { - addString(output, "🏷️ ", "Node ID", nodeStaleChunks.NodeID, 1) - for _, staleChunks := range nodeStaleChunks.Chunks { - addString(output, " 🏷️ ", "Chunk ID", staleChunks.ChunkID, 1) - sent := time.Unix(staleChunks.Sent, 0) - runningSecs := int64(time.Since(sent).Seconds()) - addString(output, " ⏱️ ", "Sent", sent.Format(time.DateTime)+" ("+state.SecondsToLiteralTime(runningSecs, "")+")", 1) - for _, file := range staleChunks.Files { - output.WriteString("\t\t📄 " + file + "\n") - } - } - } -} - func addTitle(output *strings.Builder, title string) { output.WriteString(coreutils.PrintBoldTitle(title + "\n")) } diff --git a/artifactory/commands/transferfiles/status_test.go b/artifactory/commands/transferfiles/status_test.go index 38df54b4b..9692e2b43 100644 --- a/artifactory/commands/transferfiles/status_test.go +++ b/artifactory/commands/transferfiles/status_test.go @@ -3,7 +3,6 @@ package transferfiles import ( "bytes" "testing" - "time" "github.com/jfrog/build-info-go/utils" "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/api" @@ -54,7 +53,7 @@ func TestShowStatus(t *testing.T) { defer cleanUp() // Create state manager and persist to file system - createStateManager(t, api.Phase1, false, false) + createStateManager(t, api.Phase1, false) // Run show status and check output assert.NoError(t, ShowStatus()) @@ -86,7 +85,7 @@ func TestShowStatusDiffPhase(t *testing.T) { defer cleanUp() // Create state manager and persist to file system - createStateManager(t, api.Phase2, false, false) + createStateManager(t, api.Phase2, false) // Run show status and check output assert.NoError(t, ShowStatus()) @@ -102,6 +101,7 @@ func TestShowStatusDiffPhase(t *testing.T) { assert.Contains(t, results, "Transfer speed: 0.011 MB/s") assert.Contains(t, results, "Estimated time remaining: Not available yet") assert.Contains(t, results, "Transfer failures: 223 (In Phase 3 and in subsequent executions, we'll retry transferring the failed files)") + assert.NotContains(t, results, "Skipped (deleted at source)", "shown only when there are such items") // Check repository status assert.Contains(t, results, "Current Repository Status") @@ -113,12 +113,25 @@ func TestShowStatusDiffPhase(t *testing.T) { assert.NotContains(t, results, "Files: 500 / 10000 (5.0%)") } +// TestShowStatus_skippedSourceGone verifies --status surfaces items skipped because they no longer +// exist in the source, and stays silent when there are none. +func TestShowStatus_skippedSourceGone(t *testing.T) { + buffer, cleanUp := initStatusTest(t) + defer cleanUp() + createStateManagerWithSkippedGone(t, api.Phase1, false, 39) + + assert.NoError(t, ShowStatus()) + results := buffer.String() + assert.Contains(t, results, "Skipped (deleted at source):") + assert.Contains(t, results, "39") +} + func TestShowBuildInfoRepo(t *testing.T) { buffer, cleanUp := initStatusTest(t) defer cleanUp() // Create state manager and persist to file system - createStateManager(t, api.Phase3, true, false) + createStateManager(t, api.Phase3, true) // Run show status and check output assert.NoError(t, ShowStatus()) @@ -145,30 +158,16 @@ func TestShowBuildInfoRepo(t *testing.T) { assert.NotContains(t, results, "Visited folders") } -func TestShowStaleChunks(t *testing.T) { - buffer, cleanUp := initStatusTest(t) - defer cleanUp() - - // Create state manager and persist to file system - createStateManager(t, api.Phase1, false, true) - - // Run show status and check output - assert.NoError(t, ShowStatus()) - results := buffer.String() - - // Check stale chunks - assert.Contains(t, results, "File Chunks in Transit for More than 30 Minutes") - assert.Contains(t, results, "Node ID:\tnode-id-1") - assert.Contains(t, results, "Sent:\t") - assert.Contains(t, results, "(31 minutes)") - assert.Contains(t, results, "a/b/c") - assert.Contains(t, results, "d/e/f") -} - // Create state manager and persist in the file system. // t - The testing object // phase - Phase ID -func createStateManager(t *testing.T, phase int, buildInfoRepo bool, staleChunks bool) { +func createStateManager(t *testing.T, phase int, buildInfoRepo bool) { + createStateManagerWithSkippedGone(t, phase, buildInfoRepo, 0) +} + +// createStateManagerWithSkippedGone is createStateManager with a given number of items skipped +// because they no longer exist in the source. +func createStateManagerWithSkippedGone(t *testing.T, phase int, buildInfoRepo bool, skippedGone uint64) { stateManager, err := state.NewTransferStateManager(false) assert.NoError(t, err) assert.NoError(t, stateManager.TryLockTransferStateManager()) @@ -184,24 +183,12 @@ func createStateManager(t *testing.T, phase int, buildInfoRepo bool, staleChunks stateManager.VisitedFolders = 15 stateManager.DelayedFiles = 20 stateManager.TransferFailures = 223 + stateManager.SkippedSourceGone = skippedGone stateManager.LastSpeeds = []float64{12} stateManager.LastSpeedsSum = 12 stateManager.SpeedsAverage = 12 - if staleChunks { - stateManager.StaleChunks = append(stateManager.StaleChunks, state.StaleChunks{ - NodeID: staleChunksNodeIdOne, - Chunks: []state.StaleChunk{ - { - ChunkID: staleChunksChunkId, - Sent: time.Now().Add(-time.Minute * 31).Unix(), - Files: []string{"a/b/c", "d/e/f"}, - }, - }, - }) - } - // Increment transferred size and files. This action also persists the run status. assert.NoError(t, stateManager.IncTransferredSizeAndFilesPhase1(500, 5000)) diff --git a/artifactory/commands/transferfiles/streamtransfer.go b/artifactory/commands/transferfiles/streamtransfer.go new file mode 100644 index 000000000..a32046911 --- /dev/null +++ b/artifactory/commands/transferfiles/streamtransfer.go @@ -0,0 +1,219 @@ +package transferfiles + +import ( + "context" + "errors" + "fmt" + "io" + "sync/atomic" + "time" +) + +const defaultStreamBufferSize = 32 * 1024 + +// streamIdleTimeout fails a stream when no bytes move from the source to the target for this long +// while the body is being copied. The streaming clients have no overall timeout (see +// streamResponseHeaderTimeout), so without it a source that stops sending, or a target that stops +// reading, mid-body while keeping its connection open would block the worker until the whole +// transfer is cancelled. It does not apply once the body is fully copied: the wait for the target's +// response is bounded by streamResponseHeaderTimeout. +// +// A var (not a const) so tests can shrink it. +var streamIdleTimeout = 5 * time.Minute + +var ErrStreamSizeMismatch = errors.New("stream body size mismatch") + +var ErrStreamIdle = errors.New("stream stalled") + +func IsStreamSizeMismatch(err error) bool { + return errors.Is(err, ErrStreamSizeMismatch) +} + +type putFunc func(ctx context.Context, reader io.Reader) error + +// StreamGetToPut pipes source GET bytes into target PUT using a bounded buffer. +// expectedSize must match the exact number of bytes read from source. +func StreamGetToPut(ctx context.Context, expectedSize int64, source io.ReadCloser, put putFunc, bufferSize int) (int64, error) { + if bufferSize <= 0 { + bufferSize = defaultStreamBufferSize + } + if source == nil { + return 0, fmt.Errorf("source reader is nil") + } + + // put runs under its own context so a stalled stream can abort a target PUT that is blocked + // writing to a target which stopped reading. + putCtx, cancelPut := context.WithCancelCause(ctx) + defer cancelPut(nil) + + pr, pw := io.Pipe() + progress := &progressWriter{w: pw} + progress.touch() + copyDone := make(chan streamCopyResult, 1) + copyFinished := make(chan struct{}) + buf := make([]byte, bufferSize) + + go func() { + var result streamCopyResult + defer func() { + close(copyFinished) + copyDone <- result + }() + defer closePipeWriter(pw, &result.err) + + result.bytesCopied, result.err = copyWithExactSize(ctx, progress, source, expectedSize, buf) + }() + + // put(ctx, pr) can block on pr.Read(), which only unblocks once the copy goroutine above + // writes to (or closes) pw. If source.Read() is itself blocked and ctx-blind, that copy + // goroutine never reaches pw, so a ctx cancellation would otherwise hang here forever: + // nothing closes source or pw until after put returns, and put can't return until one of + // them is closed. This watcher breaks that deadlock by closing both as soon as ctx is done, + // without waiting for put to return first. It does the same, and also cancels put, when no + // bytes have moved for streamIdleTimeout while the body is still being copied. + idleTimeout := streamIdleTimeout + watchDone := make(chan struct{}) + go func() { + idleCheck := time.NewTicker(max(idleTimeout/4, time.Millisecond)) + defer idleCheck.Stop() + idleTicks := idleCheck.C + finished := copyFinished + for { + select { + case <-ctx.Done(): + closeReadCloser(source) + _ = pw.CloseWithError(ctx.Err()) + return + case <-watchDone: + return + case <-finished: + finished, idleTicks = nil, nil + case <-idleTicks: + if progress.idleFor() < idleTimeout { + continue + } + select { + case <-finished: + finished, idleTicks = nil, nil + continue + default: + } + idleErr := fmt.Errorf("%w: no bytes were transferred for %s", ErrStreamIdle, idleTimeout) + cancelPut(idleErr) + closeReadCloser(source) + _ = pw.CloseWithError(idleErr) + return + } + } + }() + + putErr := func() (err error) { + defer func() { + if recovered := recover(); recovered != nil { + err = fmt.Errorf("panic in put: %v", recovered) + } + }() + return put(putCtx, pr) + }() + close(watchDone) + closeReadCloser(pr) + + sourceClosed := false + if putErr != nil { + // Closing the pipe reader only unblocks writes to the pipe. Close the + // source too so an in-flight source Read cannot keep this call hung. + closeReadCloser(source) + sourceClosed = true + } + copyResult := <-copyDone + if !sourceClosed { + closeReadCloser(source) + } + // The put and copy errors of a stalled stream only say that the pipe or request was closed. + if idleErr := context.Cause(putCtx); errors.Is(idleErr, ErrStreamIdle) { + return copyResult.bytesCopied, idleErr + } + if putErr != nil { + return copyResult.bytesCopied, errors.Join(putErr, copyResult.err) + } + if copyResult.err != nil { + return copyResult.bytesCopied, copyResult.err + } + return copyResult.bytesCopied, nil +} + +type streamCopyResult struct { + bytesCopied int64 + err error +} + +// progressWriter records when bytes last moved through it, for the idle check in StreamGetToPut. +type progressWriter struct { + w io.Writer + lastProgress atomic.Int64 // UnixNano +} + +func (p *progressWriter) Write(b []byte) (int, error) { + n, err := p.w.Write(b) + if n > 0 { + p.touch() + } + return n, err +} + +func (p *progressWriter) touch() { + p.lastProgress.Store(time.Now().UnixNano()) +} + +func (p *progressWriter) idleFor() time.Duration { + return time.Since(time.Unix(0, p.lastProgress.Load())) +} + +func copyWithExactSize(ctx context.Context, dst io.Writer, src io.Reader, expectedSize int64, buf []byte) (int64, error) { + var copied int64 + for { + if err := ctx.Err(); err != nil { + return copied, err + } + n, readErr := src.Read(buf) + if err := ctx.Err(); err != nil { + return copied, err + } + if n > 0 { + if copied+int64(n) > expectedSize { + return copied, fmt.Errorf("%w: expected %d bytes, read at least %d", ErrStreamSizeMismatch, expectedSize, copied+int64(n)) + } + written, writeErr := dst.Write(buf[:n]) + if err := ctx.Err(); err != nil { + return copied + int64(written), errors.Join(writeErr, err) + } + if writeErr != nil { + return copied + int64(written), writeErr + } + if written != n { + return copied + int64(written), io.ErrShortWrite + } + copied += int64(n) + } + if readErr == io.EOF { + if copied != expectedSize { + return copied, fmt.Errorf("%w: expected %d bytes, got %d", ErrStreamSizeMismatch, expectedSize, copied) + } + return copied, nil + } + if readErr != nil { + return copied, readErr + } + } +} + +func closePipeWriter(pw *io.PipeWriter, copyErr *error) { + if pw == nil { + return + } + if *copyErr != nil { + _ = pw.CloseWithError(*copyErr) + return + } + _ = pw.Close() +} diff --git a/artifactory/commands/transferfiles/streamtransfer_test.go b/artifactory/commands/transferfiles/streamtransfer_test.go new file mode 100644 index 000000000..f79ec63f8 --- /dev/null +++ b/artifactory/commands/transferfiles/streamtransfer_test.go @@ -0,0 +1,490 @@ +package transferfiles + +import ( + "context" + "errors" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestStreamGetToPut_success_exactSize(t *testing.T) { + payload := strings.Repeat("a", 100) + source := &closeTrackingReadCloser{ReadCloser: io.NopCloser(strings.NewReader(payload))} + var received []byte + + n, err := StreamGetToPut(context.Background(), int64(len(payload)), source, func(_ context.Context, reader io.Reader) error { + var readErr error + received, readErr = readAllWithBuffer(reader, defaultStreamBufferSize) + return readErr + }, defaultStreamBufferSize) + require.NoError(t, err) + assert.Equal(t, int64(len(payload)), n) + assert.Equal(t, payload, string(received)) + assert.Equal(t, int32(1), source.closeCalls.Load()) +} + +func TestStreamGetToPut_shortBody_fails(t *testing.T) { + source := io.NopCloser(strings.NewReader("short")) + _, err := StreamGetToPut(context.Background(), 100, source, func(_ context.Context, reader io.Reader) error { + _, copyErr := io.Copy(io.Discard, reader) + return copyErr + }, defaultStreamBufferSize) + require.Error(t, err) + assert.True(t, IsStreamSizeMismatch(err)) +} + +func TestStreamGetToPut_longBody_fails(t *testing.T) { + source := io.NopCloser(strings.NewReader(strings.Repeat("x", 101))) + _, err := StreamGetToPut(context.Background(), 100, source, func(_ context.Context, reader io.Reader) error { + _, copyErr := io.Copy(io.Discard, reader) + return copyErr + }, defaultStreamBufferSize) + require.Error(t, err) + assert.True(t, IsStreamSizeMismatch(err)) +} + +func TestStreamGetToPut_putError_closesSource(t *testing.T) { + sourceClosed := false + source := &closeTrackingReadCloser{ + ReadCloser: io.NopCloser(strings.NewReader("payload")), + onClose: func() { + sourceClosed = true + }, + } + + putErr := errors.New("put failed") + _, err := StreamGetToPut(context.Background(), 7, source, func(_ context.Context, _ io.Reader) error { + return putErr + }, defaultStreamBufferSize) + require.ErrorIs(t, err, putErr) + assert.True(t, sourceClosed) + assert.Equal(t, int32(1), source.closeCalls.Load()) +} + +func TestStreamGetToPut_putError_unblocksSourceRead(t *testing.T) { + source := newCloseUnblocksReadCloser() + putErr := errors.New("put failed") + result := make(chan error, 1) + + go func() { + _, err := StreamGetToPut(context.Background(), 1, source, func(_ context.Context, _ io.Reader) error { + <-source.readStarted + return putErr + }, defaultStreamBufferSize) + result <- err + }() + + select { + case err := <-result: + require.ErrorIs(t, err, putErr) + assert.True(t, source.closed.Load()) + assert.Equal(t, int32(1), source.closeCalls.Load()) + case <-time.After(time.Second): + t.Fatal("stream transfer remained blocked after PUT failure") + } +} + +func TestStreamGetToPut_cancel_closesBothLegs(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + blockRead := make(chan struct{}) + + source := &blockingReadCloser{ + data: strings.Repeat("z", 1024), + block: blockRead, + } + var putReader io.Reader + putStarted := make(chan struct{}) + + go func() { + <-putStarted + cancel() + close(blockRead) + }() + + _, err := StreamGetToPut(ctx, 1024, source, func(_ context.Context, reader io.Reader) error { + putReader = reader + close(putStarted) + _, readErr := reader.Read(make([]byte, 1)) + return readErr + }, defaultStreamBufferSize) + require.ErrorIs(t, err, context.Canceled) + assert.True(t, source.closed) + require.NotNil(t, putReader) +} + +// TestStreamGetToPut_cancel_withCtxBlindSourceAndRealTargetPut reproduces the deadlock the +// fake-put cancel tests above cannot detect: a source reader that is genuinely ctx-blind (a real +// network response body, not a type that selects on ctx internally) feeding a real +// TargetClient.Put over the wire. Before the fix, ctx cancellation only closed source/pw after +// put returned, but put could never return because its writeLoop was blocked reading pr, which +// only unblocks once the copy goroutine writes to pw, which never happens while source.Read is +// itself stuck. This must complete well within the 2s bound instead of hanging forever. +func TestStreamGetToPut_cancel_withCtxBlindSourceAndRealTargetPut(t *testing.T) { + // source.Read blocks on a channel and is ctx-blind: nothing but an explicit Close unblocks + // it, exactly like a stalled real network reader. + source := newCloseUnblocksReadCloser() + + targetServer := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { + <-r.Context().Done() + })) + defer targetServer.Close() + + client, err := NewTargetClient(context.Background(), newTestTargetServerDetails(targetServer.URL), nil) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + putStarted := make(chan struct{}) + go func() { + <-putStarted + <-source.readStarted + cancel() + }() + + metadata := testTargetMetadata() + metadata.Size = 1024 + + start := time.Now() + _, err = StreamGetToPut(ctx, metadata.Size, source, func(putCtx context.Context, reader io.Reader) error { + close(putStarted) + return client.Put(putCtx, metadata, reader, defaultTargetDeployOptions()) + }, defaultStreamBufferSize) + elapsed := time.Since(start) + require.Error(t, err) + assert.True(t, source.closed.Load()) + assert.Less(t, elapsed, 2*time.Second, + "StreamGetToPut must not hang when ctx cancels while a ctx-blind source Read and the real target PUT's write are both mid-flight") +} + +func TestStreamGetToPut_usesBoundedBuffer_notReadAll(t *testing.T) { + payload := strings.Repeat("b", defaultStreamBufferSize*3+17) + source := io.NopCloser(strings.NewReader(payload)) + var maxRead int + + n, err := StreamGetToPut(context.Background(), int64(len(payload)), source, func(_ context.Context, reader io.Reader) error { + buf := make([]byte, defaultStreamBufferSize) + for { + readN, readErr := reader.Read(buf) + if readN > maxRead { + maxRead = readN + } + if readErr == io.EOF { + return nil + } + if readErr != nil { + return readErr + } + } + }, defaultStreamBufferSize) + require.NoError(t, err) + assert.Equal(t, int64(len(payload)), n) + assert.LessOrEqual(t, maxRead, defaultStreamBufferSize) +} + +func TestStreamGetToPut_putPanic_closesSourceAndUnblocksCopy(t *testing.T) { + source := newCloseUnblocksReadCloser() + result := make(chan error, 1) + + go func() { + _, err := StreamGetToPut(context.Background(), 1, source, func(_ context.Context, _ io.Reader) error { + <-source.readStarted + panic("put exploded") + }, defaultStreamBufferSize) + result <- err + }() + + select { + case err := <-result: + require.Error(t, err) + assert.Contains(t, err.Error(), "panic in put") + assert.True(t, source.closed.Load()) + assert.Equal(t, int32(1), source.closeCalls.Load()) + case <-time.After(time.Second): + t.Fatal("stream transfer remained blocked after put panic") + } +} + +func setStreamIdleTimeout(t *testing.T, timeout time.Duration) { + orig := streamIdleTimeout + streamIdleTimeout = timeout + t.Cleanup(func() { streamIdleTimeout = orig }) +} + +func TestStreamGetToPut_sourceStallsMidBody_failsWithIdleError(t *testing.T) { + setStreamIdleTimeout(t, 100*time.Millisecond) + source := newCloseUnblocksReadCloser() + result := make(chan error, 1) + + go func() { + _, err := StreamGetToPut(context.Background(), 1, source, func(_ context.Context, reader io.Reader) error { + _, copyErr := io.Copy(io.Discard, reader) + return copyErr + }, defaultStreamBufferSize) + result <- err + }() + + select { + case err := <-result: + require.ErrorIs(t, err, ErrStreamIdle) + assert.True(t, source.closed.Load()) + case <-time.After(2 * time.Second): + t.Fatal("stream transfer remained blocked on a stalled source") + } +} + +func TestStreamGetToPut_targetStopsReading_failsWithIdleError(t *testing.T) { + setStreamIdleTimeout(t, 100*time.Millisecond) + payload := strings.Repeat("t", 4*defaultStreamBufferSize) + source := &closeTrackingReadCloser{ReadCloser: io.NopCloser(strings.NewReader(payload))} + result := make(chan error, 1) + + go func() { + _, err := StreamGetToPut(context.Background(), int64(len(payload)), source, func(putCtx context.Context, reader io.Reader) error { + if _, readErr := reader.Read(make([]byte, 1)); readErr != nil { + return readErr + } + // A target that stops reading the body blocks the HTTP client until its context ends. + <-putCtx.Done() + return putCtx.Err() + }, defaultStreamBufferSize) + result <- err + }() + + select { + case err := <-result: + require.ErrorIs(t, err, ErrStreamIdle) + assert.True(t, source.closed.Load()) + case <-time.After(2 * time.Second): + t.Fatal("stream transfer remained blocked on a target that stopped reading") + } +} + +func TestStreamGetToPut_slowButProgressingStream_succeeds(t *testing.T) { + setStreamIdleTimeout(t, 150*time.Millisecond) + payload := "slow" + source := io.NopCloser(&trickleReader{data: payload, delay: 50 * time.Millisecond}) + + n, err := StreamGetToPut(context.Background(), int64(len(payload)), source, func(_ context.Context, reader io.Reader) error { + _, copyErr := io.Copy(io.Discard, reader) + return copyErr + }, defaultStreamBufferSize) + require.NoError(t, err) + assert.Equal(t, int64(len(payload)), n) +} + +func TestStreamGetToPut_slowTargetResponseAfterBody_isNotIdle(t *testing.T) { + setStreamIdleTimeout(t, 50*time.Millisecond) + payload := "payload" + source := io.NopCloser(strings.NewReader(payload)) + + n, err := StreamGetToPut(context.Background(), int64(len(payload)), source, func(putCtx context.Context, reader io.Reader) error { + if _, copyErr := io.Copy(io.Discard, reader); copyErr != nil { + return copyErr + } + // The whole body was sent; the target may take long to answer (e.g. checksumming a large + // file). Waiting for the response is bounded by ResponseHeaderTimeout, not the idle timeout. + time.Sleep(200 * time.Millisecond) + return putCtx.Err() + }, defaultStreamBufferSize) + require.NoError(t, err) + assert.Equal(t, int64(len(payload)), n) +} + +// trickleReader returns one byte per Read, after waiting delay. +type trickleReader struct { + data string + delay time.Duration +} + +func (r *trickleReader) Read(p []byte) (int, error) { + if len(r.data) == 0 { + return 0, io.EOF + } + time.Sleep(r.delay) + n := copy(p[:1], r.data) + r.data = r.data[n:] + return n, nil +} + +func TestStreamGetToPut_largeGeneratedStream_remainsBufferBounded(t *testing.T) { + const streamSize = int64(64 * 1024 * 1024) + source := &generatedReadCloser{remaining: streamSize} + + n, err := StreamGetToPut(context.Background(), streamSize, source, func(_ context.Context, reader io.Reader) error { + _, copyErr := io.CopyBuffer(io.Discard, reader, make([]byte, defaultStreamBufferSize)) + return copyErr + }, defaultStreamBufferSize) + + require.NoError(t, err) + assert.Equal(t, streamSize, n) + assert.LessOrEqual(t, source.maxReadRequest, defaultStreamBufferSize) + assert.True(t, source.closed.Load()) +} + +func TestStreamGetToPut_retryIssuesFreshReader(t *testing.T) { + var getCount atomic.Int32 + payload := strings.Repeat("retry-me", defaultStreamBufferSize) + sources := make([]*closeTrackingReadCloser, 0, 2) + + get := func() io.ReadCloser { + getCount.Add(1) + source := &closeTrackingReadCloser{ReadCloser: io.NopCloser(strings.NewReader(payload))} + sources = append(sources, source) + return source + } + + putErr := errors.New("mid-stream PUT failure") + var committed string + firstPut := func(_ context.Context, reader io.Reader) error { + partial := make([]byte, 17) + _, err := io.ReadFull(reader, partial) + require.NoError(t, err) + return putErr + } + + _, err := StreamGetToPut(context.Background(), int64(len(payload)), get(), firstPut, defaultStreamBufferSize) + require.ErrorIs(t, err, putErr) + require.Len(t, sources, 1) + assert.True(t, sources[0].closed.Load()) + assert.Empty(t, committed) + + _, err = StreamGetToPut(context.Background(), int64(len(payload)), get(), func(_ context.Context, reader io.Reader) error { + content, readErr := io.ReadAll(reader) + if readErr == nil { + committed = string(content) + } + return readErr + }, defaultStreamBufferSize) + require.NoError(t, err) + assert.Equal(t, int32(2), getCount.Load()) + assert.Equal(t, payload, committed) +} + +type generatedReadCloser struct { + remaining int64 + maxReadRequest int + closed atomic.Bool +} + +func (r *generatedReadCloser) Read(p []byte) (int, error) { + if len(p) > r.maxReadRequest { + r.maxReadRequest = len(p) + } + if r.remaining == 0 { + return 0, io.EOF + } + n := len(p) + if int64(n) > r.remaining { + n = int(r.remaining) + } + for i := 0; i < n; i++ { + p[i] = 'x' + } + r.remaining -= int64(n) + return n, nil +} + +func (r *generatedReadCloser) Close() error { + r.closed.Store(true) + return nil +} + +type closeTrackingReadCloser struct { + io.ReadCloser + onClose func() + closed atomic.Bool + closeCalls atomic.Int32 +} + +func (c *closeTrackingReadCloser) Close() error { + c.closeCalls.Add(1) + c.closed.Store(true) + if c.onClose != nil { + c.onClose() + } + if c.ReadCloser != nil { + return c.ReadCloser.Close() + } + return nil +} + +type closeUnblocksReadCloser struct { + readStarted chan struct{} + unblockRead chan struct{} + closeOnce sync.Once + closed atomic.Bool + closeCalls atomic.Int32 +} + +func newCloseUnblocksReadCloser() *closeUnblocksReadCloser { + return &closeUnblocksReadCloser{ + readStarted: make(chan struct{}), + unblockRead: make(chan struct{}), + } +} + +func (c *closeUnblocksReadCloser) Read(_ []byte) (int, error) { + select { + case <-c.readStarted: + default: + close(c.readStarted) + } + <-c.unblockRead + return 0, io.ErrClosedPipe +} + +func (c *closeUnblocksReadCloser) Close() error { + c.closeCalls.Add(1) + c.closeOnce.Do(func() { + c.closed.Store(true) + close(c.unblockRead) + }) + return nil +} + +type blockingReadCloser struct { + data string + offset int + block chan struct{} + closed bool +} + +func (b *blockingReadCloser) Read(p []byte) (int, error) { + <-b.block + if b.offset >= len(b.data) { + return 0, io.EOF + } + n := copy(p, b.data[b.offset:]) + b.offset += n + return n, nil +} + +func (b *blockingReadCloser) Close() error { + b.closed = true + return nil +} + +func readAllWithBuffer(r io.Reader, bufSize int) ([]byte, error) { + buf := make([]byte, bufSize) + var out []byte + for { + n, err := r.Read(buf) + if n > 0 { + out = append(out, buf[:n]...) + } + if err == io.EOF { + return out, nil + } + if err != nil { + return out, err + } + } +} diff --git a/artifactory/commands/transferfiles/targetclient.go b/artifactory/commands/transferfiles/targetclient.go new file mode 100644 index 000000000..7abf6259d --- /dev/null +++ b/artifactory/commands/transferfiles/targetclient.go @@ -0,0 +1,705 @@ +package transferfiles + +import ( + "bytes" + "context" + "encoding/json" + "encoding/xml" + "errors" + "fmt" + "io" + "net/http" + "path" + "strconv" + "strings" + "sync" + "time" + + "github.com/jfrog/jfrog-cli-core/v2/utils/config" + "github.com/jfrog/jfrog-client-go/artifactory" + artifactoryutils "github.com/jfrog/jfrog-client-go/artifactory/services/utils" + "github.com/jfrog/jfrog-client-go/auth" + clientutils "github.com/jfrog/jfrog-client-go/utils" + "github.com/jfrog/jfrog-client-go/utils/errorutils" + "github.com/jfrog/jfrog-client-go/utils/io/httputils" + "github.com/jfrog/jfrog-client-go/utils/log" +) + +const ( + maxPropertyValueLength = 2400 + // Properties whose encoded matrix string exceeds 4000 chars are sent via PATCH + // (/api/metadata) instead of the deploy URL matrix params, which would make the URL too long. + maxPropertyEncodedStringLength = 4000 + // itemStatisticsSuffix is Artifactory's internal PUT target for writing download + // statistics onto an existing item (path + ":statistics"). The target must accept + // application/xml artifactory.stats bodies on that suffix. + itemStatisticsSuffix = ":statistics" + // statsRetries/statsRetryWaitMilliSecs bound the best-effort ApplyStatistics PUT to a short + // budget instead of inheriting the transfer manager's full retries/wait policy (600 x 5s), + // which could stall a single item for tens of minutes against a target returning 5xx. + statsRetries = 5 + statsRetryWaitMilliSecs = 1000 +) + +// ErrPermanentTarget marks a target response classified as non-retryable (a 4xx other than +// 429). Wrapped onto the error returned by permanentOrRetryableResponseError so callers can use +// errors.Is instead of matching the "permanent target HTTP" message prefix. +var ErrPermanentTarget = errors.New("permanent target error") + +type artifactoryStatsXML struct { + XMLName xml.Name `xml:"artifactory.stats"` + DownloadCount int64 `xml:"downloadCount"` + LastDownloaded int64 `xml:"lastDownloaded"` + LastDownloadedBy string `xml:"lastDownloadedBy"` +} + +type updateItemPropertiesBody struct { + Props map[string][]string `json:"props"` +} + +type TargetDeployOptions struct { + BuildInfoRepo bool + MinChecksumDeploySize int64 + CheckExistenceInFilestore bool + PackageType string +} + +type ChecksumDeployOutcome int + +const ( + ChecksumDeployMiss ChecksumDeployOutcome = iota + ChecksumDeployHit +) + +type eligiblePropertiesCacheKey struct { + metadata *SourceFileMetadata + packageType string +} + +type eligiblePropertiesCacheValue struct { + properties *artifactoryutils.Properties + skippedLarge bool + // viaPatch/encoded memoize propertyDelivery's decision for this item, computed once + // alongside properties instead of being re-derived by each of TryChecksumDeploy, Put, + // CreateFolder, and ApplyProperties. + viaPatch bool + encoded string +} + +type TargetClient struct { + serverDetails *config.ServerDetails + metadataServiceManager artifactory.ArtifactoryServicesManager + streamServiceManager artifactory.ArtifactoryServicesManager + + // eligibleCache retains one result per file/folder transfer so concurrent workers + // cannot evict each other's result between target operations. + eligibleMu sync.Mutex + eligibleCache map[eligiblePropertiesCacheKey]eligiblePropertiesCacheValue + filterEligibleProperties func(map[string][]string, TargetDeployOptions) (*artifactoryutils.Properties, bool) +} + +func NewTargetClient(ctx context.Context, serverDetails *config.ServerDetails, proxyTransport http.RoundTripper) (*TargetClient, error) { + metadataServiceManager, err := createMetadataTransferServiceManager(ctx, serverDetails, httpClientWithTransport(proxyTransport, time.Minute)) + if err != nil { + return nil, err + } + streamTransport := proxyTransport + if streamTransport == nil { + // No --proxy-key configured: build the same bounded-header-wait transport used for the + // source streaming client instead of falling through to client-go's default transport, + // which has no ResponseHeaderTimeout and would hang indefinitely against a non-responding + // target peer. + streamTransport, err = newDefaultStreamingTransport(serverDetails) + if err != nil { + return nil, err + } + } + streamServiceManager, err := createStreamingTransferServiceManager(ctx, serverDetails, httpClientWithTransport(streamTransport, 0)) + if err != nil { + return nil, err + } + return &TargetClient{ + serverDetails: serverDetails, + metadataServiceManager: metadataServiceManager, + streamServiceManager: streamServiceManager, + eligibleCache: make(map[eligiblePropertiesCacheKey]eligiblePropertiesCacheValue), + filterEligibleProperties: filterEligibleProperties, + }, nil +} + +// doManagerRequest sends a fixed-content request bound to ctx (so a per-call ctx cancellation +// actually aborts the in-flight request, unlike the service manager's own long-lived, +// construction-time context) and retries retryable statuses using the manager's configured +// retry budget. content may be nil for a bodyless request; it is safe to resend on retry since +// it isn't consumed like a streaming reader. +func doManagerRequest(ctx context.Context, manager artifactory.ArtifactoryServicesManager, method, fullURL string, content []byte, httpClientsDetails httputils.HttpClientDetails) (*http.Response, []byte, error) { + cfg := manager.GetConfig() + return doManagerRequestWithRetry(ctx, manager, method, fullURL, content, httpClientsDetails, cfg.GetHttpRetries(), cfg.GetHttpRetryWaitMilliSecs()) +} + +// doManagerRequestWithRetry is doManagerRequest with an explicit retry budget, for call sites +// (like ApplyStatistics) that must not inherit the transfer manager's full retry policy. +func doManagerRequestWithRetry(ctx context.Context, manager artifactory.ArtifactoryServicesManager, method, fullURL string, content []byte, httpClientsDetails httputils.HttpClientDetails, maxRetries, retryWaitMilliSecs int) (*http.Response, []byte, error) { + httpCli := manager.Client().GetHttpClient() + + var resp *http.Response + var body []byte + retryExecutor := clientutils.RetryExecutor{ + Context: ctx, + MaxRetries: maxRetries, + RetriesIntervalMilliSecs: retryWaitMilliSecs, + ErrorMessage: fmt.Sprintf("Failure occurred while sending %s request to %s", method, fullURL), + ExecutionHandler: func() (bool, error) { + var bodyReader io.Reader + if content != nil { + bodyReader = bytes.NewReader(content) + } + // Refresh an expiring access token before every attempt, as client-go's own Send* does. + if preErr := manager.GetConfig().GetServiceDetails().RunPreRequestFunctions(&httpClientsDetails); preErr != nil { + return false, preErr + } + req, reqErr := http.NewRequestWithContext(ctx, method, fullURL, bodyReader) + if reqErr != nil { + return !isContextDoneError(ctx, reqErr), reqErr + } + if content != nil { + req.ContentLength = int64(len(content)) + } + applyHttpClientDetails(req, httpClientsDetails) + var doErr error + resp, doErr = httpCli.GetClient().Do(req) + if doErr != nil { + if isContextDoneError(ctx, doErr) { + return false, doErr + } + return true, doErr + } + if resp == nil { + return false, errorutils.CheckErrorf("received empty response from target server") + } + if shouldRetryHTTPStatus(resp.StatusCode) { + drainAndClose(resp.Body) + return true, nil + } + defer drainAndClose(resp.Body) + var readErr error + body, readErr = io.ReadAll(resp.Body) + if readErr != nil { + return false, readErr + } + return false, nil + }, + } + err := retryExecutor.Execute() + return resp, body, err +} + +func doManagerPut(ctx context.Context, manager artifactory.ArtifactoryServicesManager, fullURL string, content []byte, httpClientsDetails httputils.HttpClientDetails) (*http.Response, []byte, error) { + return doManagerRequest(ctx, manager, http.MethodPut, fullURL, content, httpClientsDetails) +} + +func doManagerPatch(ctx context.Context, manager artifactory.ArtifactoryServicesManager, fullURL string, content []byte, httpClientsDetails httputils.HttpClientDetails) (*http.Response, []byte, error) { + return doManagerRequest(ctx, manager, http.MethodPatch, fullURL, content, httpClientsDetails) +} + +// doManagerPutStream sends a single-attempt streaming PUT bound to ctx. Unlike doManagerPut, it +// never retries: reader is typically the read side of an io.Pipe fed once from a source GET, and +// can't be rewound for a second attempt. +// +// Unlike doManagerPut/doManagerGet (which go through the ArtifactoryServicesManager's own +// JfrogHttpClient.Send* methods), this call issues the request via the raw *http.Client so the +// streaming body can be piped through directly. JfrogHttpClient.Send* runs client-go's +// pre-request interceptors (which proactively refresh an expiring access/refresh token) before +// every request; a raw *http.Client.Do call bypasses that entirely, so a long-running transfer +// whose token expires mid-stream would otherwise fail every subsequent PUT with 401. Run the +// same interceptors here, against the same httpClientsDetails, before building the request. +func doManagerPutStream(ctx context.Context, manager artifactory.ArtifactoryServicesManager, fullURL string, reader io.Reader, size int64, httpClientsDetails httputils.HttpClientDetails) (*http.Response, []byte, error) { + if err := manager.GetConfig().GetServiceDetails().RunPreRequestFunctions(&httpClientsDetails); err != nil { + return nil, nil, err + } + reqBody := reader + if size == 0 { + // A non-nil Body of unknown length makes Go emit Transfer-Encoding: chunked even when + // ContentLength is explicitly set to 0, since the zero value can't be told apart from + // "unset". http.NoBody signals "no body" so the request goes out with Content-Length: 0 + // instead, which zero-byte items (they skip checksum-deploy) require on the target. + reqBody = http.NoBody + } + req, err := http.NewRequestWithContext(ctx, http.MethodPut, fullURL, reqBody) + if err != nil { + return nil, nil, err + } + req.ContentLength = size + applyHttpClientDetails(req, httpClientsDetails) + resp, err := manager.Client().GetHttpClient().GetClient().Do(req) + if err != nil { + return nil, nil, err + } + if resp == nil { + return nil, nil, nil + } + defer drainAndClose(resp.Body) + if isSuccessfulDeployStatusCode(resp.StatusCode) { + return resp, nil, nil + } + body, err := io.ReadAll(resp.Body) + if err != nil { + return resp, nil, err + } + return resp, body, nil +} + +func (tc *TargetClient) Ping(ctx context.Context) error { + if err := ctx.Err(); err != nil { + return err + } + // Reimplements jfrog-client-go's PingService.Ping bound to ctx: the service manager's + // Ping() runs under its own construction-time context, so a per-call ctx cancellation + // would not abort it. + pingURL, err := clientutils.BuildUrl(tc.metadataServiceManager.GetConfig().GetServiceDetails().GetUrl(), "api/system/ping", map[string]string{}) + if err != nil { + return err + } + resp, body, err := doManagerGet(ctx, tc.metadataServiceManager, pingURL, true) + if err != nil { + return err + } + return errorutils.CheckResponseStatusWithBody(resp, body, http.StatusOK) +} + +// TryChecksumDeploy does not itself guarantee the eligibleCache entry is released: on +// ChecksumDeployMiss the entry must stay cached for the Put call that follows. Callers must +// still eventually call ApplyProperties (which always releases via defer, see +// ReleaseEligibleProperties) even after a ChecksumDeployHit or a permanent/retryable error here. +func (tc *TargetClient) TryChecksumDeploy(ctx context.Context, metadata *SourceFileMetadata, options TargetDeployOptions) (ChecksumDeployOutcome, error) { + if err := ctx.Err(); err != nil { + return ChecksumDeployMiss, err + } + if !shouldTryChecksumDeploy(metadata, options) { + return ChecksumDeployMiss, nil + } + deployURL, err := tc.buildDeployURL(targetRelativePath(metadata)) + if err != nil { + return ChecksumDeployMiss, err + } + deployURL += tc.propertyMatrixSuffix(metadata, options) + httpClientsDetails := tc.metadataServiceManager.GetConfig().GetServiceDetails().CreateHttpClientDetails() + addDeployHeaders(&httpClientsDetails, tc.metadataServiceManager.GetConfig().GetServiceDetails(), metadata, options, true) + resp, body, err := doManagerPut(ctx, tc.metadataServiceManager, deployURL, nil, httpClientsDetails) + if err != nil { + tc.ReleaseEligibleProperties(metadata, options) + return ChecksumDeployMiss, err + } + if isChecksumDeployHit(resp.StatusCode) { + // No further target call needs the cached eligible properties for this item: the + // matrix params on this PUT already carried them. Release now rather than waiting on + // ApplyProperties's deferred release. + tc.ReleaseEligibleProperties(metadata, options) + return ChecksumDeployHit, nil + } + if isChecksumDeployMiss(resp.StatusCode) { + return ChecksumDeployMiss, nil + } + if err = permanentOrRetryableResponseError(resp, body, http.StatusOK, http.StatusCreated, http.StatusAccepted); err != nil { + tc.ReleaseEligibleProperties(metadata, options) + return ChecksumDeployMiss, err + } + return ChecksumDeployMiss, nil +} + +// Put does not itself release the eligibleCache entry on success: the caller must still call +// ApplyProperties afterward, which always releases it via defer (see +// ReleaseEligibleProperties). On error, Put releases it directly since ApplyProperties is not +// expected to run. +func (tc *TargetClient) Put(ctx context.Context, metadata *SourceFileMetadata, reader io.Reader, options TargetDeployOptions) error { + if err := ctx.Err(); err != nil { + return err + } + deployURL, err := tc.buildDeployURL(targetRelativePath(metadata)) + if err != nil { + return err + } + deployURL += tc.propertyMatrixSuffix(metadata, options) + httpClientsDetails := tc.streamServiceManager.GetConfig().GetServiceDetails().CreateHttpClientDetails() + addDeployHeaders(&httpClientsDetails, tc.streamServiceManager.GetConfig().GetServiceDetails(), metadata, options, false) + // Skip artifactoryutils.UploadFileFromReader: it always AddChecksumHeaders including + // X-Checksum-Md5 (even when Md5 is empty). Plain PUT must omit that header; it is + // reserved for CheckExistenceInFilestore checksum-deploy. Use doManagerPutStream instead of + // SendPut/UploadFileFromReader so the request is bound to this call's ctx (not just the + // service manager's construction-time context) and can actually be aborted mid-flight. + resp, body, err := doManagerPutStream(ctx, tc.streamServiceManager, deployURL, reader, metadata.Size, httpClientsDetails) + if err != nil { + tc.ReleaseEligibleProperties(metadata, options) + return err + } + if resp == nil { + tc.ReleaseEligibleProperties(metadata, options) + return errorutils.CheckErrorf("received empty response from target deploy") + } + if isSuccessfulDeployStatusCode(resp.StatusCode) { + return nil + } + tc.ReleaseEligibleProperties(metadata, options) + return permanentOrRetryableResponseError(resp, body, http.StatusOK, http.StatusCreated, http.StatusAccepted) +} + +func (tc *TargetClient) CreateFolder(ctx context.Context, metadata *SourceFileMetadata, options TargetDeployOptions) error { + if err := ctx.Err(); err != nil { + return err + } + deployURL, err := tc.buildDeployURL(targetRelativePath(metadata)) + if err != nil { + return err + } + deployURL += tc.propertyMatrixSuffix(metadata, options) + httpClientsDetails := tc.metadataServiceManager.GetConfig().GetServiceDetails().CreateHttpClientDetails() + addIdentityHeaders(&httpClientsDetails, metadata) + resp, body, err := doManagerPut(ctx, tc.metadataServiceManager, deployURL, nil, httpClientsDetails) + if err != nil { + tc.ReleaseEligibleProperties(metadata, options) + return err + } + if isSuccessfulDeployStatusCode(resp.StatusCode) { + return nil + } + tc.ReleaseEligibleProperties(metadata, options) + return permanentOrRetryableResponseError(resp, body, http.StatusOK, http.StatusCreated, http.StatusAccepted) +} + +func (tc *TargetClient) ApplyProperties(ctx context.Context, metadata *SourceFileMetadata, options TargetDeployOptions) (skippedLargeProps bool, err error) { + if err := ctx.Err(); err != nil { + return false, err + } + entry := tc.eligiblePropertiesEntry(metadata, options) + defer tc.ReleaseEligibleProperties(metadata, options) + if !entry.viaPatch { + // Short, path-safe maps are applied as matrix params on checksum/full PUT. + return entry.skippedLarge, nil + } + eligibleProps := entry.properties + relativePath := strings.TrimSuffix(targetRelativePath(metadata), "/") + return entry.skippedLarge, tc.applyPropertiesViaPatch(ctx, relativePath, eligibleProps) +} + +func (tc *TargetClient) applyPropertiesViaPatch(ctx context.Context, relativePath string, eligibleProps *artifactoryutils.Properties) error { + setPropertiesURL, err := clientutils.BuildUrl(tc.metadataServiceManager.GetConfig().GetServiceDetails().GetUrl(), path.Join("api", "metadata", relativePath), map[string]string{}) + if err != nil { + return err + } + requestBody, err := json.Marshal(updateItemPropertiesBody{Props: eligibleProps.ToMap()}) + if err != nil { + return errorutils.CheckError(err) + } + httpClientsDetails := tc.metadataServiceManager.GetConfig().GetServiceDetails().CreateHttpClientDetails() + httpClientsDetails.SetContentTypeApplicationJson() + resp, body, err := doManagerPatch(ctx, tc.metadataServiceManager, setPropertiesURL, requestBody, httpClientsDetails) + if err != nil { + return err + } + return permanentOrRetryableResponseError(resp, body, http.StatusNoContent) +} + +func (tc *TargetClient) ApplyStatistics(ctx context.Context, metadata *SourceFileMetadata, options TargetDeployOptions) error { + if err := ctx.Err(); err != nil { + return err + } + if skipsDownloadStatistics(options) || metadata.Name == "" || !hasDownloadStatistics(metadata) { + return nil + } + relativePath := strings.TrimSuffix(targetRelativePath(metadata), "/") + deployURL, err := tc.buildDeployURL(relativePath) + if err != nil { + log.Warn("Couldn't set stats for", relativePath+". Reason:", err.Error()) + return nil + } + statsXML, err := xml.Marshal(artifactoryStatsXML{ + DownloadCount: metadata.DownloadCount, + LastDownloaded: metadata.LastDownloaded, + LastDownloadedBy: metadata.LastDownloadedBy, + }) + if err != nil { + log.Warn("Couldn't set stats for", relativePath+". Reason:", err.Error()) + return nil + } + httpClientsDetails := tc.metadataServiceManager.GetConfig().GetServiceDetails().CreateHttpClientDetails() + httpClientsDetails.AddHeader("Content-Type", "application/xml") + resp, body, err := doManagerRequestWithRetry( + ctx, tc.metadataServiceManager, http.MethodPut, deployURL+itemStatisticsSuffix, statsXML, httpClientsDetails, + statsRetries, statsRetryWaitMilliSecs, + ) + if err != nil { + log.Warn("Couldn't set stats for", relativePath+". Reason:", err.Error()) + return nil + } + if err = permanentOrRetryableResponseError(resp, body, http.StatusOK, http.StatusCreated, http.StatusNoContent); err != nil { + log.Warn("Couldn't set stats for", relativePath+". Reason:", err.Error()) + return nil + } + log.Debug("Applied download statistics for", targetRelativePath(metadata)) + return nil +} + +// eligiblePropertiesEntry returns the full memoized entry (filtered properties plus the +// propertyDelivery decision) for one file/folder transfer, computing both only once. The +// metadata pointer is the transfer identity shared by all target operations. +func (tc *TargetClient) eligiblePropertiesEntry(metadata *SourceFileMetadata, options TargetDeployOptions) eligiblePropertiesCacheValue { + cacheKey := eligiblePropertiesCacheKey{metadata: metadata, packageType: options.PackageType} + tc.eligibleMu.Lock() + if cached, ok := tc.eligibleCache[cacheKey]; ok { + tc.eligibleMu.Unlock() + return cached + } + tc.eligibleMu.Unlock() + + eligible, skipped := tc.filterEligibleProperties(metadata.Properties, options) + viaPatch, encoded := propertyDelivery(eligible) + entry := eligiblePropertiesCacheValue{properties: eligible, skippedLarge: skipped, viaPatch: viaPatch, encoded: encoded} + + tc.eligibleMu.Lock() + defer tc.eligibleMu.Unlock() + if cached, ok := tc.eligibleCache[cacheKey]; ok { + return cached + } + tc.eligibleCache[cacheKey] = entry + return entry +} + +// eligibleProperties returns the filtered property set for one file/folder transfer. +func (tc *TargetClient) eligibleProperties(metadata *SourceFileMetadata, options TargetDeployOptions) (*artifactoryutils.Properties, bool) { + entry := tc.eligiblePropertiesEntry(metadata, options) + return entry.properties, entry.skippedLarge +} + +// HasEligibleProperties reports whether metadata has at least one property eligible for +// transfer, reusing the same memoized filter result as ApplyProperties instead of re-running +// filterEligibleProperties. +func (tc *TargetClient) HasEligibleProperties(metadata *SourceFileMetadata, options TargetDeployOptions) bool { + eligible, _ := tc.eligibleProperties(metadata, options) + return eligible.KeysLen() > 0 +} + +// ReleaseEligibleProperties drops the cached eligible properties for one transfer item. +// Callers that never reach ApplyProperties should defer this after the metadata pointer is known. +func (tc *TargetClient) ReleaseEligibleProperties(metadata *SourceFileMetadata, options TargetDeployOptions) { + if metadata == nil { + return + } + tc.eligibleMu.Lock() + defer tc.eligibleMu.Unlock() + delete(tc.eligibleCache, eligiblePropertiesCacheKey{metadata: metadata, packageType: options.PackageType}) +} + +func hasDownloadStatistics(metadata *SourceFileMetadata) bool { + return metadata.DownloadCount > 0 || metadata.LastDownloaded > 0 || metadata.LastDownloadedBy != "" +} + +// skipsDownloadStatistics matches Artifactory info repos (BuildInfo, PipeInfo). Those +// stores ingest JSON metadata and reject PUT path:statistics with application/xml (415). +// Distribution, Support, and ReleaseBundles accept statistics (lab 201) and are not skipped. +func skipsDownloadStatistics(options TargetDeployOptions) bool { + if options.BuildInfoRepo { + return true + } + switch strings.ToLower(options.PackageType) { + case "buildinfo", "pipeinfo": + return true + default: + return false + } +} + +func (tc *TargetClient) buildDeployURL(relativePath string) (string, error) { + return clientutils.BuildUrl(tc.metadataServiceManager.GetConfig().GetServiceDetails().GetUrl(), relativePath, map[string]string{}) +} + +func targetRelativePath(metadata *SourceFileMetadata) string { + if metadata.Name == "" { + folderPath := path.Join(metadata.Repo, metadata.Path) + if !strings.HasSuffix(folderPath, "/") { + folderPath += "/" + } + return folderPath + } + if metadata.Path == "." { + return path.Join(metadata.Repo, metadata.Name) + } + return path.Join(metadata.Repo, metadata.Path, metadata.Name) +} + +func shouldTryChecksumDeploy(metadata *SourceFileMetadata, options TargetDeployOptions) bool { + if options.BuildInfoRepo || metadata.Name == "" { + return false + } + if metadata.Sha1 == "" { + return false + } + return metadata.Size >= options.MinChecksumDeploySize +} + +func addDeployHeaders(httpClientsDetails *httputils.HttpClientDetails, serviceDetails auth.ServiceDetails, metadata *SourceFileMetadata, options TargetDeployOptions, checksumDeploy bool) { + addIdentityHeaders(httpClientsDetails, metadata) + if httpClientsDetails.Headers == nil { + httpClientsDetails.Headers = make(map[string]string) + } + // Always send sha1/sha256 on deploy. X-Checksum-Md5 is reserved for the + // CheckExistenceInFilestore checksum-deploy path only (below). + if metadata.Sha1 != "" { + httpClientsDetails.Headers["X-Checksum-Sha1"] = metadata.Sha1 + } + if metadata.Sha256 != "" { + httpClientsDetails.Headers["X-Checksum"] = metadata.Sha256 + } + artifactoryutils.AddAuthHeaders(httpClientsDetails.Headers, serviceDetails) + if checksumDeploy { + httpClientsDetails.AddHeader("X-Checksum-Deploy", "true") + if options.CheckExistenceInFilestore { + httpClientsDetails.AddHeader("X-Check-Binary-Existence-In-Filestore", "true") + httpClientsDetails.AddHeader("X-Checksum-Md5", metadata.Md5) + httpClientsDetails.AddHeader("X-Binary-Size", fmt.Sprintf("%d", metadata.Size)) + } + } +} + +func addIdentityHeaders(httpClientsDetails *httputils.HttpClientDetails, metadata *SourceFileMetadata) { + if created, ok := artifactoryTimestampHeaderValue(metadata.Created); ok { + httpClientsDetails.AddHeader("X-Artifactory-Created", created) + } + if metadata.CreatedBy != "" { + httpClientsDetails.AddHeader("X-Artifactory-Created-By", metadata.CreatedBy) + } + if lastModified, ok := artifactoryTimestampHeaderValue(metadata.LastModified); ok { + httpClientsDetails.AddHeader("X-Artifactory-Last-Modified", lastModified) + } + if metadata.ModifiedBy != "" { + httpClientsDetails.AddHeader("X-Artifactory-Modified-By", metadata.ModifiedBy) + } +} + +func artifactoryTimestampHeaderValue(timestamp string) (string, bool) { + if timestamp == "" { + return "", false + } + if _, err := strconv.ParseInt(timestamp, 10, 64); err == nil { + return timestamp, true + } + parsed, err := time.Parse(time.RFC3339, timestamp) + if err != nil { + return "", false + } + return strconv.FormatInt(parsed.UnixMilli(), 10), true +} + +func filterEligibleProperties(properties map[string][]string, options TargetDeployOptions) (*artifactoryutils.Properties, bool) { + eligibleProps := artifactoryutils.NewProperties() + skippedLargeProps := false + for key, values := range properties { + if isGeneratedPropertyKey(key, options.PackageType) { + continue + } + for _, value := range values { + if len(value) > maxPropertyValueLength { + skippedLargeProps = true + continue + } + eligibleProps.AddProperty(key, value) + } + } + return eligibleProps, skippedLargeProps +} + +// isPathUnsafePropertyValue reports characters that Properties.ToEncodedString cannot make +// matrix-param-safe even after url.QueryEscape: a newline/CR breaks the HTTP request line, "#" +// starts a URL fragment, "," and "\" are the multi-value/escape separators ToEncodedString +// itself parses back out, and "|" is unsafe for some reverse proxies in front of Artifactory. +// ";" and "=" are matrix-param separators too, but ToEncodedString escapes them via +// url.QueryEscape like any other value byte (-> %3B / %3D), so they round-trip safely and are +// deliberately not included here. +func isPathUnsafePropertyValue(value string) bool { + return strings.ContainsAny(value, "\n\r#,\\|") +} + +func containsPathUnsafePropertyValue(properties *artifactoryutils.Properties) bool { + for key, values := range properties.ToMap() { + if isPathUnsafePropertyValue(key) { + return true + } + for _, value := range values { + if isPathUnsafePropertyValue(value) { + return true + } + } + } + return false +} + +// propertyDelivery is the single decision for matrix-param vs PATCH. Put, +// checksum-deploy, and ApplyProperties must not re-derive these conditions. +func propertyDelivery(eligibleProps *artifactoryutils.Properties) (viaPatch bool, encoded string) { + if eligibleProps == nil || eligibleProps.KeysLen() == 0 { + return false, "" + } + encoded = eligibleProps.ToEncodedString(false) + if encoded == "" { + return false, "" + } + if len(encoded) > maxPropertyEncodedStringLength || containsPathUnsafePropertyValue(eligibleProps) { + return true, encoded + } + return false, encoded +} + +func (tc *TargetClient) propertyMatrixSuffix(metadata *SourceFileMetadata, options TargetDeployOptions) string { + if metadata == nil { + return "" + } + entry := tc.eligiblePropertiesEntry(metadata, options) + return formatPropertyMatrixSuffix(entry.viaPatch, entry.encoded) +} + +// propertyMatrixSuffixFromEligible re-derives the propertyDelivery decision from a Properties +// set that was not obtained via TargetClient.eligiblePropertiesEntry, so its result is not +// memoized. Production call sites go through the TargetClient method above instead. +func propertyMatrixSuffixFromEligible(eligibleProps *artifactoryutils.Properties) string { + viaPatch, encoded := propertyDelivery(eligibleProps) + return formatPropertyMatrixSuffix(viaPatch, encoded) +} + +func formatPropertyMatrixSuffix(viaPatch bool, encoded string) string { + if viaPatch || encoded == "" { + return "" + } + return ";" + encoded +} + +func isChecksumDeployHit(statusCode int) bool { + return statusCode == http.StatusOK || statusCode == http.StatusCreated || statusCode == http.StatusAccepted +} + +// Only a missing blob (404) falls through to a full-body PUT. +// HTTP 409 on checksum-deploy means the target already has a different checksum at +// this path — this is a hard failure; we do not overwrite. +func isChecksumDeployMiss(statusCode int) bool { + return statusCode == http.StatusNotFound +} + +func isSuccessfulDeployStatusCode(statusCode int) bool { + return statusCode == http.StatusOK || statusCode == http.StatusCreated || statusCode == http.StatusAccepted +} + +// isPermanentHTTPStatus reports 4xx responses other than 429 as non-retryable. +// The shared HTTP client already skips retries for these codes; this helper surfaces +// that classification at the transfer-files call site. +func isPermanentHTTPStatus(statusCode int) bool { + return statusCode >= 400 && statusCode < 500 && statusCode != http.StatusTooManyRequests +} + +func permanentOrRetryableResponseError(resp *http.Response, body []byte, expectedStatusCodes ...int) error { + err := errorutils.CheckResponseStatusWithBody(resp, body, expectedStatusCodes...) + if err == nil { + return nil + } + if resp != nil && isPermanentHTTPStatus(resp.StatusCode) { + return fmt.Errorf("permanent target HTTP %d: %w: %w", resp.StatusCode, ErrPermanentTarget, err) + } + return err +} diff --git a/artifactory/commands/transferfiles/targetclient_test.go b/artifactory/commands/transferfiles/targetclient_test.go new file mode 100644 index 000000000..39ba0bbc2 --- /dev/null +++ b/artifactory/commands/transferfiles/targetclient_test.go @@ -0,0 +1,1627 @@ +package transferfiles + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + transferutils "github.com/jfrog/jfrog-cli-core/v2/artifactory/utils" + "github.com/jfrog/jfrog-cli-core/v2/utils/config" + artifactoryutils "github.com/jfrog/jfrog-client-go/artifactory/services/utils" + "github.com/jfrog/jfrog-client-go/auth" + "github.com/jfrog/jfrog-client-go/utils/io/httputils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + testTargetRepo = testSourceRepo + canonicalUbuntuDescription = "The Ubuntu container image maintained by Canonical\n\n" + + "Ubuntu is a Debian-based Linux operating system that runs from the desktop to the cloud, " + + "to all your internet connected things.\n" + + "It is the world's most popular operating system across public clouds and OpenStack clouds.\n" + + "It is the number one platform for containers; from Docker to Kubernetes to LXD, " + + "Ubuntu can run your containers at scale.\n" + + "Fast, secure and simple, Ubuntu powers millions of PCs worldwide.\n" +) + +func propertyMatrixSuffix(metadata *SourceFileMetadata, options TargetDeployOptions) string { + eligibleProps, _ := filterEligibleProperties(metadata.Properties, options) + return propertyMatrixSuffixFromEligible(eligibleProps) +} + +func testTargetMetadata() *SourceFileMetadata { + return &SourceFileMetadata{ + Repo: testTargetRepo, + Path: testSourcePath, + Name: testSourceName, + Size: 11, + Sha1: "sha1-value", + Sha256: "sha256-value", + Md5: "md5-value", + Created: "2020-01-01T00:00:00.000Z", + CreatedBy: "admin", + LastModified: "2020-01-02T00:00:00.000Z", + ModifiedBy: "deployer", + Properties: map[string][]string{ + "build.name": {"app"}, + "env": {"prod", "release"}, + }, + } +} + +func testTargetRelativePath() string { + return testSourceRelativePath() +} + +func defaultTargetDeployOptions() TargetDeployOptions { + return TargetDeployOptions{ + MinChecksumDeploySize: 10, + } +} + +func newTestTargetServerDetails(serverURL string) *config.ServerDetails { + return &config.ServerDetails{ + Url: serverURL + "/", + ArtifactoryUrl: serverURL + "/", + } +} + +func TestTargetClient_Ping_usesSystemPing(t *testing.T) { + var requestPath string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestPath = r.URL.Path + _, _ = w.Write([]byte("OK")) + })) + defer server.Close() + + client, err := NewTargetClient(context.Background(), newTestTargetServerDetails(server.URL), nil) + require.NoError(t, err) + + err = client.Ping(context.Background()) + require.NoError(t, err) + assert.Equal(t, "/api/system/ping", requestPath) +} + +func TestTargetClient_TryChecksumDeploy_hit_sendsNoBody(t *testing.T) { + deployPath := "/" + testTargetRelativePath() + var requestMethod string + var requestPath string + var checksumDeployHeader string + var sha1Header string + var bodyBytes []byte + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestMethod = r.Method + requestPath = r.URL.Path + checksumDeployHeader = r.Header.Get("X-Checksum-Deploy") + sha1Header = r.Header.Get("X-Checksum-Sha1") + if r.Body != nil { + bodyBytes, _ = io.ReadAll(r.Body) + } + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + client, err := NewTargetClient(context.Background(), newTestTargetServerDetails(server.URL), nil) + require.NoError(t, err) + + outcome, err := client.TryChecksumDeploy(context.Background(), testTargetMetadata(), defaultTargetDeployOptions()) + require.NoError(t, err) + assert.Equal(t, ChecksumDeployHit, outcome) + assert.Equal(t, http.MethodPut, requestMethod) + assert.True(t, strings.HasPrefix(requestPath, deployPath)) + assert.Contains(t, requestPath, ";build.name=") + assert.Equal(t, "true", checksumDeployHeader) + assert.Empty(t, bodyBytes) + assert.Equal(t, "sha1-value", sha1Header) +} + +func TestTargetClient_TryChecksumDeploy_missOn404(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer server.Close() + + client, err := NewTargetClient(context.Background(), newTestTargetServerDetails(server.URL), nil) + require.NoError(t, err) + + outcome, err := client.TryChecksumDeploy(context.Background(), testTargetMetadata(), defaultTargetDeployOptions()) + require.NoError(t, err) + assert.Equal(t, ChecksumDeployMiss, outcome) +} + +func TestTargetClient_TryChecksumDeploy_conflictDoesNotFallThrough(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusConflict) + _, _ = w.Write([]byte(`{"errors":[{"message":"checksum conflict"}]}`)) + })) + defer server.Close() + + client, err := NewTargetClient(context.Background(), newTestTargetServerDetails(server.URL), nil) + require.NoError(t, err) + + outcome, err := client.TryChecksumDeploy(context.Background(), testTargetMetadata(), defaultTargetDeployOptions()) + require.Error(t, err) + assert.Equal(t, ChecksumDeployMiss, outcome) +} + +func TestTargetClient_TryChecksumDeploy_skipsBuildInfoRepo(t *testing.T) { + requestCount := 0 + server := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) { + requestCount++ + })) + defer server.Close() + + client, err := NewTargetClient(context.Background(), newTestTargetServerDetails(server.URL), nil) + require.NoError(t, err) + + options := defaultTargetDeployOptions() + options.BuildInfoRepo = true + outcome, err := client.TryChecksumDeploy(context.Background(), testTargetMetadata(), options) + require.NoError(t, err) + assert.Equal(t, ChecksumDeployMiss, outcome) + assert.Zero(t, requestCount) +} + +func TestTargetClient_TryChecksumDeploy_filestoreOptionHeaders(t *testing.T) { + var binaryExistenceHeader string + var md5Header string + var md5Present bool + var binarySizeHeader string + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + binaryExistenceHeader = r.Header.Get("X-Check-Binary-Existence-In-Filestore") + _, md5Present = r.Header["X-Checksum-Md5"] + md5Header = r.Header.Get("X-Checksum-Md5") + binarySizeHeader = r.Header.Get("X-Binary-Size") + w.WriteHeader(http.StatusAccepted) + })) + defer server.Close() + + client, err := NewTargetClient(context.Background(), newTestTargetServerDetails(server.URL), nil) + require.NoError(t, err) + + options := defaultTargetDeployOptions() + options.CheckExistenceInFilestore = true + outcome, err := client.TryChecksumDeploy(context.Background(), testTargetMetadata(), options) + require.NoError(t, err) + assert.Equal(t, ChecksumDeployHit, outcome) + assert.Equal(t, "true", binaryExistenceHeader) + assert.True(t, md5Present, "X-Checksum-Md5 must be present on filestore-existence checksum-deploy") + assert.Equal(t, "md5-value", md5Header) + assert.Equal(t, strconv.FormatInt(testTargetMetadata().Size, 10), binarySizeHeader) +} + +func TestTargetClient_Put_setsContentLengthAndChecksumHeaders(t *testing.T) { + deployPath := "/" + testTargetRelativePath() + payload := "hello world" + var contentLength string + var sha1Header string + var sha256Header string + var md5Present bool + var bodyBytes []byte + var requestPath string + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestPath = r.URL.Path + contentLength = r.Header.Get("Content-Length") + sha1Header = r.Header.Get("X-Checksum-Sha1") + sha256Header = r.Header.Get("X-Checksum") + _, md5Present = r.Header["X-Checksum-Md5"] + bodyBytes, _ = io.ReadAll(r.Body) + w.WriteHeader(http.StatusCreated) + })) + defer server.Close() + + client, err := NewTargetClient(context.Background(), newTestTargetServerDetails(server.URL), nil) + require.NoError(t, err) + + metadata := testTargetMetadata() + err = client.Put(context.Background(), metadata, strings.NewReader(payload), defaultTargetDeployOptions()) + require.NoError(t, err) + assert.True(t, strings.HasPrefix(requestPath, deployPath)) + assert.Contains(t, requestPath, ";build.name=") + assert.Equal(t, strconv.FormatInt(metadata.Size, 10), contentLength) + assert.Equal(t, metadata.Sha1, sha1Header) + assert.Equal(t, metadata.Sha256, sha256Header) + assert.False(t, md5Present, "X-Checksum-Md5 must be absent on plain Put") + assert.Equal(t, payload, string(bodyBytes)) +} + +func TestTargetClient_ApplyProperties_setsEligibleProperties(t *testing.T) { + requestCount := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + requestCount++ + w.WriteHeader(http.StatusNoContent) + })) + defer server.Close() + + client, err := NewTargetClient(context.Background(), newTestTargetServerDetails(server.URL), nil) + require.NoError(t, err) + + skippedLargeProps, err := client.ApplyProperties(context.Background(), testTargetMetadata(), defaultTargetDeployOptions()) + require.NoError(t, err) + assert.False(t, skippedLargeProps) + assert.Zero(t, requestCount, "short property maps are applied as deploy matrix params, not a follow-up properties PUT") +} + +func TestTargetClient_ApplyProperties_omitsValuesLongerThan2400(t *testing.T) { + longValue := strings.Repeat("x", 2401) + metadata := testTargetMetadata() + metadata.Properties = map[string][]string{ + "short": {"ok"}, + "long": {longValue}, + } + + requestCount := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + requestCount++ + w.WriteHeader(http.StatusNoContent) + })) + defer server.Close() + + client, err := NewTargetClient(context.Background(), newTestTargetServerDetails(server.URL), nil) + require.NoError(t, err) + + skippedLargeProps, err := client.ApplyProperties(context.Background(), metadata, defaultTargetDeployOptions()) + require.NoError(t, err) + assert.True(t, skippedLargeProps) + assert.Zero(t, requestCount) + assert.Contains(t, client.propertyMatrixSuffix(metadata, defaultTargetDeployOptions()), ";short=") + assert.NotContains(t, client.propertyMatrixSuffix(metadata, defaultTargetDeployOptions()), "long") +} + +func TestPropertyMatrixSuffix_omitsPathUnsafeValues(t *testing.T) { + metadata := testTargetMetadata() + metadata.Properties = map[string][]string{ + "docker.label.org.opencontainers.image.description": { + canonicalUbuntuDescription, + }, + "note": {"see docs#install"}, + "docker.os": {"linux"}, + } + + suffix := propertyMatrixSuffix(metadata, defaultTargetDeployOptions()) + assert.Empty(t, suffix, "one unsafe value routes the entire item through PATCH") + assert.NotContains(t, suffix, "docker.label.org.opencontainers.image.description") + assert.NotContains(t, suffix, "note=") + assert.NotContains(t, suffix, "%0A") +} + +func Test_propertyDelivery_agreesForMatrixAndPatch(t *testing.T) { + tests := []struct { + name string + props map[string][]string + viaPatch bool + }{ + { + name: "path-safe short map uses matrix", + props: map[string][]string{"docker.os": {"linux"}}, + viaPatch: false, + }, + { + name: "unsafe value forces PATCH", + props: map[string][]string{ + "note": {"docs#install"}, + "docker.os": {"linux"}, + }, + viaPatch: true, + }, + { + name: "oversized encoded payload forces PATCH", + props: largeEligiblePropertiesMap(t), + viaPatch: true, + }, + { + // ";" and "=" are matrix-param separators, but ToEncodedString escapes them via + // url.QueryEscape like any other byte, so they round-trip through the matrix path + // instead of forcing PATCH. See isPathUnsafePropertyValue's doc comment. + name: "semicolon and equals values still use matrix", + props: map[string][]string{"note": {"a;b=c"}}, + viaPatch: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + eligible, _ := filterEligibleProperties(tt.props, TargetDeployOptions{}) + viaPatch, encoded := propertyDelivery(eligible) + assert.Equal(t, tt.viaPatch, viaPatch) + suffix := propertyMatrixSuffixFromEligible(eligible) + if viaPatch { + assert.Empty(t, suffix) + assert.NotEmpty(t, encoded) + return + } + assert.Equal(t, ";"+encoded, suffix) + if tt.name == "semicolon and equals values still use matrix" { + assert.NotContains(t, encoded, "a;b=c", "raw ; and = must not appear unescaped in the matrix suffix") + assert.Contains(t, encoded, "a%3Bb%3Dc") + } + }) + } +} + +func Test_isPathUnsafePropertyValue(t *testing.T) { + tests := []struct { + name string + value string + unsafe bool + }{ + {name: "newline", value: "first\nsecond", unsafe: true}, + {name: "carriage return", value: "first\rsecond", unsafe: true}, + {name: "hash", value: "docs#install", unsafe: true}, + {name: "comma", value: "one,two", unsafe: true}, + {name: "backslash", value: `one\two`, unsafe: true}, + {name: "pipe", value: "one|two", unsafe: true}, + {name: "semicolon", value: "one;two", unsafe: false}, + {name: "equals", value: "one=two", unsafe: false}, + {name: "safe", value: "letters numbers 123-_./:", unsafe: false}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.unsafe, isPathUnsafePropertyValue(test.value)) + }) + } +} + +func TestTargetClient_ApplyProperties_patchesPathUnsafeValues(t *testing.T) { + metadata := testTargetMetadata() + metadata.Properties = map[string][]string{ + "docker.label.org.opencontainers.image.description": {canonicalUbuntuDescription}, + "docker.os": {"linux"}, + } + + var requestMethod string + var bodyBytes []byte + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestMethod = r.Method + bodyBytes, _ = io.ReadAll(r.Body) + w.WriteHeader(http.StatusNoContent) + })) + defer server.Close() + + client, err := NewTargetClient(context.Background(), newTestTargetServerDetails(server.URL), nil) + require.NoError(t, err) + + skippedLargeProps, err := client.ApplyProperties(context.Background(), metadata, defaultTargetDeployOptions()) + require.NoError(t, err) + assert.False(t, skippedLargeProps) + assert.Equal(t, http.MethodPatch, requestMethod) + + var patchBody updateItemPropertiesBody + require.NoError(t, json.Unmarshal(bodyBytes, &patchBody)) + assert.Equal(t, map[string][]string{ + "docker.label.org.opencontainers.image.description": {canonicalUbuntuDescription}, + "docker.os": {"linux"}, + }, patchBody.Props) + assert.Empty(t, propertyMatrixSuffix(metadata, defaultTargetDeployOptions())) +} + +func TestTargetClient_ApplyProperties_patchesPathUnsafeKeys(t *testing.T) { + metadata := testTargetMetadata() + metadata.Properties = map[string][]string{ + "safe": {"value"}, + "unsafe|key": {"preserved"}, + } + + var requestMethod string + var requestPath string + var bodyBytes []byte + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestMethod = r.Method + requestPath = r.URL.Path + bodyBytes, _ = io.ReadAll(r.Body) + w.WriteHeader(http.StatusNoContent) + })) + defer server.Close() + + client, err := NewTargetClient(context.Background(), newTestTargetServerDetails(server.URL), nil) + require.NoError(t, err) + + skippedLargeProps, err := client.ApplyProperties(context.Background(), metadata, defaultTargetDeployOptions()) + require.NoError(t, err) + assert.False(t, skippedLargeProps) + assert.Equal(t, http.MethodPatch, requestMethod) + assert.Equal(t, "/api/metadata/"+testTargetRelativePath(), requestPath) + assert.Empty(t, propertyMatrixSuffix(metadata, defaultTargetDeployOptions()), + "unsafe property keys must not be encoded as deploy matrix parameters") + + var patchBody updateItemPropertiesBody + require.NoError(t, json.Unmarshal(bodyBytes, &patchBody)) + assert.Equal(t, metadata.Properties, patchBody.Props) +} + +func TestTargetClient_ApplyProperties_usesPatchWhenEncodedPayloadExceeds4000(t *testing.T) { + metadata := testTargetMetadata() + metadata.Properties = largeEligiblePropertiesMap(t) + + var requestMethod string + var requestPath string + var rawQuery string + var contentType string + var bodyBytes []byte + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestMethod = r.Method + requestPath = r.URL.Path + rawQuery = r.URL.RawQuery + contentType = r.Header.Get("Content-Type") + bodyBytes, _ = io.ReadAll(r.Body) + w.WriteHeader(http.StatusNoContent) + })) + defer server.Close() + + client, err := NewTargetClient(context.Background(), newTestTargetServerDetails(server.URL), nil) + require.NoError(t, err) + + skippedLargeProps, err := client.ApplyProperties(context.Background(), metadata, defaultTargetDeployOptions()) + require.NoError(t, err) + assert.False(t, skippedLargeProps) + assert.Equal(t, http.MethodPatch, requestMethod) + assert.Equal(t, "/api/metadata/"+testTargetRelativePath(), requestPath) + assert.Empty(t, rawQuery) + assert.Contains(t, contentType, "application/json") + + var patchBody updateItemPropertiesBody + require.NoError(t, json.Unmarshal(bodyBytes, &patchBody)) + assert.NotEmpty(t, patchBody.Props) + assert.Contains(t, patchBody.Props, "transfer.prop.0000") +} + +func TestTargetClient_ApplyProperties_usesPutQueryWhenEncodedPayloadAtMost4000(t *testing.T) { + requestCount := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + requestCount++ + w.WriteHeader(http.StatusNoContent) + })) + defer server.Close() + + client, err := NewTargetClient(context.Background(), newTestTargetServerDetails(server.URL), nil) + require.NoError(t, err) + + metadata := testTargetMetadata() + suffix := client.propertyMatrixSuffix(metadata, defaultTargetDeployOptions()) + assert.True(t, strings.HasPrefix(suffix, ";")) + assert.Contains(t, suffix, "build.name=") + assert.LessOrEqual(t, len(strings.TrimPrefix(suffix, ";")), maxPropertyEncodedStringLength) + + _, err = client.ApplyProperties(context.Background(), metadata, defaultTargetDeployOptions()) + require.NoError(t, err) + assert.Zero(t, requestCount) +} + +func TestTargetClient_ApplyProperties_stripsOnlyPackageGeneratedPropertyKeys(t *testing.T) { + metadata := testTargetMetadata() + metadata.Properties = map[string][]string{ + "build.name": {"app"}, + "npm.name": {"package"}, + "npm.version": {"1.0.0"}, + "artifactory.licenses": {"MIT"}, + "artifactory.metadata.exclude": {"*"}, + "package.lowercase": {"true"}, + "ruby": {"x86_64-linux"}, + "baseUrl": {"http://example.com"}, + "conan.settings.os": {"Linux"}, + "conan.settings.compiler": {"gcc"}, + } + + requestCount := 0 + server := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { + requestCount++ + })) + defer server.Close() + + client, err := NewTargetClient(context.Background(), newTestTargetServerDetails(server.URL), nil) + require.NoError(t, err) + + options := defaultTargetDeployOptions() + options.PackageType = "npm" + suffix := client.propertyMatrixSuffix(metadata, options) + assert.Contains(t, suffix, "build.name=") + assert.NotContains(t, suffix, "npm.name") + assert.NotContains(t, suffix, "npm.version") + assert.NotContains(t, suffix, "artifactory.metadata.exclude") + assert.Contains(t, suffix, "artifactory.licenses") + assert.Contains(t, suffix, "package.lowercase") + assert.Contains(t, suffix, "baseUrl") + assert.Contains(t, suffix, "conan.settings.os") + + skippedLargeProps, err := client.ApplyProperties(context.Background(), metadata, options) + require.NoError(t, err) + assert.False(t, skippedLargeProps) + assert.Zero(t, requestCount) +} + +func TestTargetClient_ApplyProperties_stripsPackageScopedGeneratedKeys(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})) + defer server.Close() + + client, err := NewTargetClient(context.Background(), newTestTargetServerDetails(server.URL), nil) + require.NoError(t, err) + + gemsMetadata := testTargetMetadata() + gemsMetadata.Properties = map[string][]string{ + "build.name": {"app"}, + "ruby": {"x86_64-linux"}, + } + gemsOptions := defaultTargetDeployOptions() + gemsOptions.PackageType = "gems" + suffix := client.propertyMatrixSuffix(gemsMetadata, gemsOptions) + assert.Contains(t, suffix, "build.name=") + assert.NotContains(t, suffix, "ruby=") + + pubMetadata := testTargetMetadata() + pubMetadata.Properties = map[string][]string{ + "build.name": {"app"}, + "baseUrl": {"http://example.com"}, + } + pubOptions := defaultTargetDeployOptions() + pubOptions.PackageType = "pub" + suffix = client.propertyMatrixSuffix(pubMetadata, pubOptions) + assert.Contains(t, suffix, "build.name=") + assert.NotContains(t, suffix, "baseUrl=") +} + +func TestTargetClient_ApplyProperties_preservesMultiValues(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNoContent) + })) + defer server.Close() + + client, err := NewTargetClient(context.Background(), newTestTargetServerDetails(server.URL), nil) + require.NoError(t, err) + + metadata := testTargetMetadata() + metadata.Properties = map[string][]string{ + "env": {"prod", "release"}, + } + suffix := client.propertyMatrixSuffix(metadata, defaultTargetDeployOptions()) + assert.Contains(t, suffix, "env=prod") + assert.Contains(t, suffix, "env=release") +} + +func TestTargetClient_CreateFolder_putsZeroByteBody(t *testing.T) { + folderMetadata := &SourceFileMetadata{ + Repo: testTargetRepo, + Path: "empty-dir", + Name: "", + Created: "2020-01-01T00:00:00.000Z", + CreatedBy: "admin", + LastModified: "2020-01-02T00:00:00.000Z", + ModifiedBy: "deployer", + } + deployPath := "/" + testTargetRepo + "/empty-dir/" + var contentLength string + var bodyBytes []byte + var requestPath string + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestPath = r.URL.Path + contentLength = r.Header.Get("Content-Length") + bodyBytes, _ = io.ReadAll(r.Body) + w.WriteHeader(http.StatusCreated) + })) + defer server.Close() + + client, err := NewTargetClient(context.Background(), newTestTargetServerDetails(server.URL), nil) + require.NoError(t, err) + + err = client.CreateFolder(context.Background(), folderMetadata, defaultTargetDeployOptions()) + require.NoError(t, err) + assert.Equal(t, deployPath, requestPath) + assert.Equal(t, "0", contentLength) + assert.Empty(t, bodyBytes) +} + +func TestTargetClient_noPluginExecuteURLs(t *testing.T) { + var seenPaths []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + seenPaths = append(seenPaths, r.URL.Path) + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + client, err := NewTargetClient(context.Background(), newTestTargetServerDetails(server.URL), nil) + require.NoError(t, err) + + require.NoError(t, client.Ping(context.Background())) + _, err = client.TryChecksumDeploy(context.Background(), testTargetMetadata(), defaultTargetDeployOptions()) + require.NoError(t, err) + + for _, path := range seenPaths { + assert.NotContains(t, path, "/api/plugins/execute") + } +} + +func TestTargetClient_checksumDeploy409IsHardFailure(t *testing.T) { + deployPath := "/" + testTargetRelativePath() + checksumDeployCalls := 0 + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !strings.HasPrefix(r.URL.Path, deployPath) || r.Method != http.MethodPut { + w.WriteHeader(http.StatusNotFound) + return + } + if r.Header.Get("X-Checksum-Deploy") == "true" { + checksumDeployCalls++ + w.WriteHeader(http.StatusConflict) + _, _ = w.Write([]byte(`{"errors":[{"message":"checksum conflict"}]}`)) + return + } + t.Errorf("unexpected non-checksum-deploy PUT to %s", r.URL.Path) + w.WriteHeader(http.StatusCreated) + })) + defer server.Close() + + client, err := NewTargetClient(context.Background(), newTestTargetServerDetails(server.URL), nil) + require.NoError(t, err) + + outcome, err := client.TryChecksumDeploy(context.Background(), testTargetMetadata(), defaultTargetDeployOptions()) + assert.Equal(t, ChecksumDeployMiss, outcome) + require.Error(t, err) + assert.Contains(t, err.Error(), "permanent target HTTP 409") + assert.Equal(t, 1, checksumDeployCalls) +} + +func largeEligiblePropertiesMap(t *testing.T) map[string][]string { + t.Helper() + props := make(map[string][]string) + for i := 0; ; i++ { + key := fmt.Sprintf("transfer.prop.%04d", i) + props[key] = []string{"value"} + if encodedPropertiesLength(props) > maxPropertyEncodedStringLength { + break + } + } + require.Greater(t, encodedPropertiesLength(props), maxPropertyEncodedStringLength) + return props +} + +func encodedPropertiesLength(props map[string][]string) int { + eligibleProps, _ := filterEligibleProperties(props, TargetDeployOptions{}) + return len(eligibleProps.ToEncodedString(false)) +} + +func TestNewTargetClient_streamServiceManagerHasNoOverallTimeoutOrRetries(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + client, err := NewTargetClient(context.Background(), newTestTargetServerDetails(server.URL), nil) + require.NoError(t, err) + + streamConfig := client.streamServiceManager.GetConfig() + assert.Zero(t, streamConfig.GetOverallRequestTimeout()) + assert.Zero(t, streamConfig.GetHttpRetries()) + + metadataConfig := client.metadataServiceManager.GetConfig() + assert.Equal(t, time.Minute, metadataConfig.GetOverallRequestTimeout()) + assert.Equal(t, metadataTransferRetries, metadataConfig.GetHttpRetries()) +} + +func TestTargetClient_TryChecksumDeploy_skipsWhenSha1Empty(t *testing.T) { + requestCount := 0 + server := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) { + requestCount++ + })) + defer server.Close() + + client, err := NewTargetClient(context.Background(), newTestTargetServerDetails(server.URL), nil) + require.NoError(t, err) + + metadata := testTargetMetadata() + metadata.Sha1 = "" + outcome, err := client.TryChecksumDeploy(context.Background(), metadata, defaultTargetDeployOptions()) + require.NoError(t, err) + assert.Equal(t, ChecksumDeployMiss, outcome) + assert.Zero(t, requestCount) +} + +func TestTargetClient_emptySha1UsesPutStreamPath(t *testing.T) { + deployPath := "/" + testTargetRelativePath() + payload := "hello world" + checksumDeployCalls := 0 + fullPutCalls := 0 + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !strings.HasPrefix(r.URL.Path, deployPath) || r.Method != http.MethodPut { + w.WriteHeader(http.StatusNotFound) + return + } + if r.Header.Get("X-Checksum-Deploy") == "true" { + checksumDeployCalls++ + w.WriteHeader(http.StatusOK) + return + } + fullPutCalls++ + w.WriteHeader(http.StatusCreated) + })) + defer server.Close() + + client, err := NewTargetClient(context.Background(), newTestTargetServerDetails(server.URL), nil) + require.NoError(t, err) + + metadata := testTargetMetadata() + metadata.Sha1 = "" + outcome, err := client.TryChecksumDeploy(context.Background(), metadata, defaultTargetDeployOptions()) + require.NoError(t, err) + assert.Equal(t, ChecksumDeployMiss, outcome) + assert.Zero(t, checksumDeployCalls) + + err = client.Put(context.Background(), metadata, strings.NewReader(payload), defaultTargetDeployOptions()) + require.NoError(t, err) + assert.Equal(t, 1, fullPutCalls) +} + +func Test_addIdentityHeaders_convertsIsoTimestampsToEpochMilliseconds(t *testing.T) { + var createdHeader string + var lastModifiedHeader string + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + createdHeader = r.Header.Get("X-Artifactory-Created") + lastModifiedHeader = r.Header.Get("X-Artifactory-Last-Modified") + w.WriteHeader(http.StatusCreated) + })) + defer server.Close() + + client, err := NewTargetClient(context.Background(), newTestTargetServerDetails(server.URL), nil) + require.NoError(t, err) + + err = client.Put(context.Background(), testTargetMetadata(), strings.NewReader("hello world"), defaultTargetDeployOptions()) + require.NoError(t, err) + assert.Equal(t, "1577836800000", createdHeader) + assert.Equal(t, "1577923200000", lastModifiedHeader) +} + +func Test_addIdentityHeaders_preservesNumericTimestamps(t *testing.T) { + var createdHeader string + var lastModifiedHeader string + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + createdHeader = r.Header.Get("X-Artifactory-Created") + lastModifiedHeader = r.Header.Get("X-Artifactory-Last-Modified") + w.WriteHeader(http.StatusCreated) + })) + defer server.Close() + + client, err := NewTargetClient(context.Background(), newTestTargetServerDetails(server.URL), nil) + require.NoError(t, err) + + metadata := testTargetMetadata() + metadata.Created = "1670000000000" + metadata.LastModified = "1680000000000" + err = client.Put(context.Background(), metadata, strings.NewReader("hello world"), defaultTargetDeployOptions()) + require.NoError(t, err) + assert.Equal(t, "1670000000000", createdHeader) + assert.Equal(t, "1680000000000", lastModifiedHeader) +} + +func Test_addIdentityHeaders_omitsInvalidTimestamps(t *testing.T) { + var createdHeader string + var lastModifiedHeader string + var createdByHeader string + var modifiedByHeader string + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + createdHeader = r.Header.Get("X-Artifactory-Created") + lastModifiedHeader = r.Header.Get("X-Artifactory-Last-Modified") + createdByHeader = r.Header.Get("X-Artifactory-Created-By") + modifiedByHeader = r.Header.Get("X-Artifactory-Modified-By") + w.WriteHeader(http.StatusCreated) + })) + defer server.Close() + + client, err := NewTargetClient(context.Background(), newTestTargetServerDetails(server.URL), nil) + require.NoError(t, err) + + metadata := testTargetMetadata() + metadata.Created = "not-a-timestamp" + metadata.LastModified = "also-invalid" + err = client.Put(context.Background(), metadata, strings.NewReader("hello world"), defaultTargetDeployOptions()) + require.NoError(t, err) + assert.Empty(t, createdHeader) + assert.Empty(t, lastModifiedHeader) + assert.Equal(t, "admin", createdByHeader) + assert.Equal(t, "deployer", modifiedByHeader) +} + +func TestTargetClient_ApplyStatistics_putsItemStatisticsXML(t *testing.T) { + var ( + requestMethod string + requestPath string + requestBody string + contentType string + ) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestMethod = r.Method + requestPath = r.URL.Path + contentType = r.Header.Get("Content-Type") + body, err := io.ReadAll(r.Body) + if err != nil { + t.Errorf("read body: %v", err) + w.WriteHeader(http.StatusInternalServerError) + return + } + requestBody = string(body) + w.WriteHeader(http.StatusCreated) + })) + defer server.Close() + + client, err := NewTargetClient(context.Background(), newTestTargetServerDetails(server.URL), nil) + require.NoError(t, err) + + metadata := testTargetMetadata() + metadata.DownloadCount = 5 + metadata.LastDownloaded = 1788945284685 + metadata.LastDownloadedBy = "admin" + err = client.ApplyStatistics(context.Background(), metadata, defaultTargetDeployOptions()) + require.NoError(t, err) + assert.Equal(t, http.MethodPut, requestMethod) + assert.Equal(t, "/"+testTargetRelativePath()+":statistics", requestPath) + assert.Equal(t, "application/xml", contentType) + assert.Contains(t, requestBody, "") + assert.Contains(t, requestBody, "5") + assert.Contains(t, requestBody, "1788945284685") + assert.Contains(t, requestBody, "admin") +} + +func TestTargetClient_ApplyStatistics_rejectedStatus_doesNotFail(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.True(t, strings.HasSuffix(r.URL.Path, ":statistics")) + w.WriteHeader(http.StatusConflict) + _, _ = w.Write([]byte(`{"errors":[{"message":"statistics rejected"}]}`)) + })) + defer server.Close() + + client, err := NewTargetClient(context.Background(), newTestTargetServerDetails(server.URL), nil) + require.NoError(t, err) + + metadata := testTargetMetadata() + metadata.DownloadCount = 5 + metadata.LastDownloadedBy = "admin" + err = client.ApplyStatistics(context.Background(), metadata, defaultTargetDeployOptions()) + require.NoError(t, err) +} + +func TestTargetClient_ApplyStatistics_skipsWhenEmpty(t *testing.T) { + requestCount := 0 + server := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) { + requestCount++ + })) + defer server.Close() + + client, err := NewTargetClient(context.Background(), newTestTargetServerDetails(server.URL), nil) + require.NoError(t, err) + + err = client.ApplyStatistics(context.Background(), testTargetMetadata(), defaultTargetDeployOptions()) + require.NoError(t, err) + assert.Zero(t, requestCount) +} + +func TestTargetClient_ApplyStatistics_skipsFolders(t *testing.T) { + requestCount := 0 + server := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) { + requestCount++ + })) + defer server.Close() + + client, err := NewTargetClient(context.Background(), newTestTargetServerDetails(server.URL), nil) + require.NoError(t, err) + + err = client.ApplyStatistics(context.Background(), &SourceFileMetadata{ + Repo: testTargetRepo, + Path: "empty-dir", + DownloadCount: 3, + LastDownloadedBy: "admin", + }, defaultTargetDeployOptions()) + require.NoError(t, err) + assert.Zero(t, requestCount) +} + +func TestTargetClient_sendsTargetCredentials(t *testing.T) { + var receivedAuth []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + receivedAuth = append(receivedAuth, r.Header.Get("Authorization")) + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + details := newTestTargetServerDetails(server.URL) + details.AccessToken = "target-secret-token" + client, err := NewTargetClient(context.Background(), details, nil) + require.NoError(t, err) + + require.NoError(t, client.Ping(context.Background())) + assert.NotEmpty(t, receivedAuth) + for i, auth := range receivedAuth { + assert.Equal(t, "Bearer target-secret-token", auth, "request %d", i) + } +} + +// TestTargetClient_Put_expiredTokenRejection_returnsPromptlyNoHang exercises the real HTTP path +// (doManagerPutStream's direct manager.Client().GetHttpClient().GetClient().Do(req) call, which +// bypasses client-go's JfrogHttpClient.Send* wrapper and therefore its token-refresh +// pre-request interceptor) to settle the B-22 "expiring token mid-stream deadlocks the copy/PUT +// goroutines" theory. The target here simulates an Artifactory rejecting a stale, un-refreshed +// bearer token: it reads none of the request body and immediately responds 401. The reader fed +// to Put blocks forever on all but its first Read (as a real in-flight GET-stream reader would +// while more file content is still arriving), so if the PUT path ever waited for the full body to +// be written before observing the response, this test would hang until its own timeout. +func TestTargetClient_Put_expiredTokenRejection_returnsPromptlyNoHang(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Deliberately do not read r.Body: an expired/rejected token fails auth before content + // processing. Respond immediately, as a real Artifactory 401 would. + w.WriteHeader(http.StatusUnauthorized) + _, _ = w.Write([]byte(`{"errors":[{"status":401,"message":"Bad credentials"}]}`)) + })) + defer server.Close() + + details := newTestTargetServerDetails(server.URL) + details.AccessToken = "stale-expired-token" + client, err := NewTargetClient(context.Background(), details, nil) + require.NoError(t, err) + + blockingReader := newBlockingAfterFirstByteReader() + metadata := testTargetMetadata() + metadata.Size = 10 * 1024 * 1024 // large enough that a body-first client would never finish writing + + done := make(chan error, 1) + go func() { + done <- client.Put(context.Background(), metadata, blockingReader, defaultTargetDeployOptions()) + }() + + select { + case err := <-done: + require.Error(t, err, "expired-token PUT must fail cleanly, not succeed") + assert.Contains(t, err.Error(), "401") + case <-time.After(3 * time.Second): + t.Fatal("TargetClient.Put hung instead of returning promptly on a 401 rejection") + } +} + +// TestTargetClient_Put_runsPreRequestInterceptors_tokenRefresh verifies the B-22 fix: Put's +// streaming PUT path (doManagerPutStream) now runs the target's registered pre-request +// interceptors (the same mechanism client-go's JfrogHttpClient.Send* uses for proactive +// access/refresh-token renewal) before issuing the raw HTTP request, instead of bypassing them +// entirely. A custom interceptor rewrites the Authorization header; the test asserts the target +// server actually observes the rewritten header, not the stale one the client was constructed +// with. +func TestTargetClient_Put_runsPreRequestInterceptors_tokenRefresh(t *testing.T) { + var observedAuth string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + observedAuth = r.Header.Get("Authorization") + w.WriteHeader(http.StatusCreated) + })) + defer server.Close() + + details := newTestTargetServerDetails(server.URL) + details.AccessToken = "stale-expired-token" + client, err := NewTargetClient(context.Background(), details, nil) + require.NoError(t, err) + + serviceDetails := client.streamServiceManager.GetConfig().GetServiceDetails() + serviceDetails.AppendPreRequestFunction(func(_ *auth.CommonConfigFields, httpClientDetails *httputils.HttpClientDetails) error { + if httpClientDetails.Headers == nil { + httpClientDetails.Headers = map[string]string{} + } + httpClientDetails.Headers["Authorization"] = "Bearer refreshed-token" + return nil + }) + + metadata := testTargetMetadata() + metadata.Size = int64(len("hello world")) + err = client.Put(context.Background(), metadata, bytes.NewReader([]byte("hello world")), defaultTargetDeployOptions()) + require.NoError(t, err) + + assert.Equal(t, "Bearer refreshed-token", observedAuth, "the pre-request interceptor must run before the streaming PUT is sent") +} + +// blockingAfterFirstByteReader yields one byte, then blocks on every subsequent Read until the +// test process exits, modeling a GET-stream reader that still has much more content pending. +type blockingAfterFirstByteReader struct { + yielded bool +} + +func newBlockingAfterFirstByteReader() *blockingAfterFirstByteReader { + return &blockingAfterFirstByteReader{} +} + +func (r *blockingAfterFirstByteReader) Read(p []byte) (int, error) { + if !r.yielded && len(p) > 0 { + r.yielded = true + p[0] = 'x' + return 1, nil + } + select {} // block forever; only a closed underlying connection/context should ever stop the caller +} + +// TestTargetClient_sendsTargetCredentials_putAndPatch complements +// TestTargetClient_sendsTargetCredentials, which only exercises Ping (routed through +// doManagerGet). Put and applyPropertiesViaPatch instead build their requests through +// applyHttpClientDetails directly, a separate code path that must apply the same auth. +func TestTargetClient_sendsTargetCredentials_putAndPatch(t *testing.T) { + receivedAuthByMethod := map[string]string{} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + receivedAuthByMethod[r.Method] = r.Header.Get("Authorization") + if r.Method == http.MethodPatch { + w.WriteHeader(http.StatusNoContent) + return + } + w.WriteHeader(http.StatusCreated) + })) + defer server.Close() + + details := newTestTargetServerDetails(server.URL) + details.AccessToken = "target-secret-token" + client, err := NewTargetClient(context.Background(), details, nil) + require.NoError(t, err) + + require.NoError(t, client.Put(context.Background(), testTargetMetadata(), strings.NewReader("hello world"), defaultTargetDeployOptions())) + + metadata := testTargetMetadata() + metadata.Properties = largeEligiblePropertiesMap(t) + _, err = client.ApplyProperties(context.Background(), metadata, defaultTargetDeployOptions()) + require.NoError(t, err) + + assert.Equal(t, "Bearer target-secret-token", receivedAuthByMethod[http.MethodPut]) + assert.Equal(t, "Bearer target-secret-token", receivedAuthByMethod[http.MethodPatch]) +} + +func TestTargetClient_TryChecksumDeploy_permanent4xxFailsFast(t *testing.T) { + requestCount := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + requestCount++ + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write([]byte(`{"errors":[{"message":"forbidden"}]}`)) + })) + defer server.Close() + + client, err := NewTargetClient(context.Background(), newTestTargetServerDetails(server.URL), nil) + require.NoError(t, err) + + outcome, err := client.TryChecksumDeploy(context.Background(), testTargetMetadata(), defaultTargetDeployOptions()) + assert.Equal(t, ChecksumDeployMiss, outcome) + require.Error(t, err) + assert.Contains(t, err.Error(), "permanent target HTTP 403") + assert.Equal(t, 1, requestCount, "4xx other than 429 must not be retried") +} + +func TestTargetClient_TryChecksumDeploy_retryableStatusUsesMetadataRetryPolicy(t *testing.T) { + const metadataRetries = 2 + for _, testCase := range []struct { + name string + statusCode int + }{ + {name: "too many requests", statusCode: http.StatusTooManyRequests}, + {name: "server error", statusCode: http.StatusServiceUnavailable}, + } { + t.Run(testCase.name, func(t *testing.T) { + requestCount := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + requestCount++ + w.WriteHeader(testCase.statusCode) + })) + defer server.Close() + + details := newTestTargetServerDetails(server.URL) + client, err := NewTargetClient(context.Background(), details, nil) + require.NoError(t, err) + client.metadataServiceManager, err = transferutils.CreateServiceManagerWithContext( + context.Background(), details, false, 0, metadataRetries, 0, time.Minute, + ) + require.NoError(t, err) + + outcome, err := client.TryChecksumDeploy( + context.Background(), testTargetMetadata(), defaultTargetDeployOptions(), + ) + assert.Equal(t, ChecksumDeployMiss, outcome) + require.Error(t, err) + assert.Equal(t, metadataRetries, client.metadataServiceManager.GetConfig().GetHttpRetries()) + assert.Equal(t, metadataRetries+1, requestCount, + "retryable responses must use the configured metadata retries plus the initial attempt") + }) + } +} + +// TestNewTargetClient_metadataServiceManagerUsesProductionRetryPolicy complements +// TestTargetClient_TryChecksumDeploy_retryableStatusUsesMetadataRetryPolicy, which swaps in a +// manager with a small retry count to keep the test fast. This asserts the manager NewTargetClient +// actually builds (without any override) carries the dedicated metadata retry budget. +func TestNewTargetClient_metadataServiceManagerUsesProductionRetryPolicy(t *testing.T) { + client, err := NewTargetClient(context.Background(), newTestTargetServerDetails("http://127.0.0.1:0"), nil) + require.NoError(t, err) + assert.Equal(t, metadataTransferRetries, client.metadataServiceManager.GetConfig().GetHttpRetries()) + assert.Equal(t, metadataRetryWaitMilliSecs, client.metadataServiceManager.GetConfig().GetHttpRetryWaitMilliSecs()) +} + +func TestTargetClient_eligiblePropertiesComputedOnceAcrossFileAndFolderOperations(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Header.Get("X-Checksum-Deploy") == "true": + w.WriteHeader(http.StatusNotFound) + case r.Method == http.MethodPatch: + w.WriteHeader(http.StatusNoContent) + default: + w.WriteHeader(http.StatusCreated) + } + })) + defer server.Close() + + client, err := NewTargetClient(context.Background(), newTestTargetServerDetails(server.URL), nil) + require.NoError(t, err) + var filterCalls atomic.Int32 + client.filterEligibleProperties = func( + properties map[string][]string, options TargetDeployOptions, + ) (*artifactoryutils.Properties, bool) { + filterCalls.Add(1) + return filterEligibleProperties(properties, options) + } + + fileMetadata := testTargetMetadata() + fileMetadata.Properties = largeEligiblePropertiesMap(t) + options := defaultTargetDeployOptions() + outcome, err := client.TryChecksumDeploy(context.Background(), fileMetadata, options) + require.NoError(t, err) + assert.Equal(t, ChecksumDeployMiss, outcome) + require.NoError(t, client.Put(context.Background(), fileMetadata, strings.NewReader("hello world"), options)) + _, err = client.ApplyProperties(context.Background(), fileMetadata, options) + require.NoError(t, err) + + folderMetadata := &SourceFileMetadata{ + Repo: testTargetRepo, + Path: "empty-dir", + Properties: largeEligiblePropertiesMap(t), + } + require.NoError(t, client.CreateFolder(context.Background(), folderMetadata, options)) + _, err = client.ApplyProperties(context.Background(), folderMetadata, options) + require.NoError(t, err) + + assert.Equal(t, int32(2), filterCalls.Load(), + "each file or folder transfer must compute eligible properties once across all target operations") +} + +func TestTargetClient_concurrentTransfersDoNotEvictEligibleProperties(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPatch { + w.WriteHeader(http.StatusNoContent) + return + } + w.WriteHeader(http.StatusCreated) + })) + defer server.Close() + + client, err := NewTargetClient(context.Background(), newTestTargetServerDetails(server.URL), nil) + require.NoError(t, err) + var filterCalls atomic.Int32 + client.filterEligibleProperties = func( + properties map[string][]string, options TargetDeployOptions, + ) (*artifactoryutils.Properties, bool) { + filterCalls.Add(1) + return filterEligibleProperties(properties, options) + } + + const transfers = 8 + metadata := make([]*SourceFileMetadata, transfers) + for i := range metadata { + metadata[i] = &SourceFileMetadata{ + Repo: testTargetRepo, + Path: fmt.Sprintf("folder-%d", i), + Properties: largeEligiblePropertiesMap(t), + } + } + options := defaultTargetDeployOptions() + runConcurrently := func(operation func(*SourceFileMetadata) error) { + var waitGroup sync.WaitGroup + errors := make(chan error, len(metadata)) + for _, item := range metadata { + waitGroup.Add(1) + go func() { + defer waitGroup.Done() + errors <- operation(item) + }() + } + waitGroup.Wait() + close(errors) + for operationErr := range errors { + require.NoError(t, operationErr) + } + } + + runConcurrently(func(item *SourceFileMetadata) error { + return client.CreateFolder(context.Background(), item, options) + }) + runConcurrently(func(item *SourceFileMetadata) error { + _, applyErr := client.ApplyProperties(context.Background(), item, options) + return applyErr + }) + + assert.Equal(t, int32(transfers), filterCalls.Load(), + "concurrent transfers must retain each transfer's cached eligible properties") +} + +func TestTargetClient_Put_errorReleasesEligibleProperties(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write([]byte(`{"errors":[{"message":"forbidden"}]}`)) + })) + defer server.Close() + + client, err := NewTargetClient(context.Background(), newTestTargetServerDetails(server.URL), nil) + require.NoError(t, err) + + metadata := testTargetMetadata() + options := defaultTargetDeployOptions() + err = client.Put(context.Background(), metadata, strings.NewReader("hello world"), options) + require.Error(t, err) + assert.False(t, eligibleCacheHas(client, metadata, options), + "failed Put must release eligibleCache without ApplyProperties") +} + +func TestTargetClient_Put_classifiesPermanentStatusAndKeepsBodyText(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write([]byte(`{"errors":[{"message":"forbidden by target"}]}`)) + })) + defer server.Close() + + client, err := NewTargetClient(context.Background(), newTestTargetServerDetails(server.URL), nil) + require.NoError(t, err) + + metadata := testTargetMetadata() + err = client.Put(context.Background(), metadata, strings.NewReader("hello world"), defaultTargetDeployOptions()) + require.Error(t, err) + assert.Contains(t, err.Error(), "permanent target HTTP 403", + "plain PUT must reach the permanent/retryable classification, not bail out on UploadFileFromReader's status-only error") + assert.Contains(t, err.Error(), "forbidden by target", + "the classified error must still carry the response body text") +} + +func TestTargetClient_Put_classifiesRetryableStatus(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusTooManyRequests) + _, _ = w.Write([]byte(`{"errors":[{"message":"slow down"}]}`)) + })) + defer server.Close() + + client, err := NewTargetClient(context.Background(), newTestTargetServerDetails(server.URL), nil) + require.NoError(t, err) + + metadata := testTargetMetadata() + err = client.Put(context.Background(), metadata, strings.NewReader("hello world"), defaultTargetDeployOptions()) + require.Error(t, err) + assert.NotContains(t, err.Error(), "permanent target HTTP", + "429 must remain retryable, not be classified as permanent") + assert.Contains(t, err.Error(), "slow down") +} + +func TestTargetClient_Put_ctxCancelAbortsInFlightRequest(t *testing.T) { + unblock := make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + <-unblock + w.WriteHeader(http.StatusCreated) + })) + defer server.Close() + defer close(unblock) + + client, err := NewTargetClient(context.Background(), newTestTargetServerDetails(server.URL), nil) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(50 * time.Millisecond) + cancel() + }() + + start := time.Now() + err = client.Put(ctx, testTargetMetadata(), strings.NewReader("hello world"), defaultTargetDeployOptions()) + elapsed := time.Since(start) + require.Error(t, err) + assert.Less(t, elapsed, 2*time.Second, + "Put must be bound to the per-call ctx and abort mid-flight on cancellation, not block until the (never-responding) server replies") +} + +// TestTargetClient_Put_ctxCancelAbortsMidBodyWrite complements +// TestTargetClient_Put_ctxCancelAbortsInFlightRequest, which cancels while still waiting for +// response headers (before any body byte reaches the server). This exercises cancellation once +// the target is already reading the streamed body. The reader is a real network response body +// (as production streams: the source GET's resp.Body feeds the target PUT), not an io.Pipe -- +// an io.Pipe's blocking Read isn't network I/O and isn't unblocked by the transport closing the +// connection on ctx cancellation, which would make that variant of this test hang forever. +func TestTargetClient_Put_ctxCancelAbortsMidBodyWrite(t *testing.T) { + sourceServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write(bytes.Repeat([]byte("a"), 4096)) + if flusher, ok := w.(http.Flusher); ok { + flusher.Flush() + } + <-r.Context().Done() + })) + defer sourceServer.Close() + + targetRequestReceived := make(chan struct{}) + targetServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + close(targetRequestReceived) + _, _ = io.Copy(io.Discard, r.Body) + w.WriteHeader(http.StatusCreated) + })) + defer targetServer.Close() + + client, err := NewTargetClient(context.Background(), newTestTargetServerDetails(targetServer.URL), nil) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + sourceReq, err := http.NewRequestWithContext(ctx, http.MethodGet, sourceServer.URL, nil) + require.NoError(t, err) + sourceResp, err := http.DefaultClient.Do(sourceReq) + require.NoError(t, err) + defer func() { _ = sourceResp.Body.Close() }() + + go func() { + <-targetRequestReceived + time.Sleep(50 * time.Millisecond) + cancel() + }() + + metadata := testTargetMetadata() + metadata.Size = 1024 * 1024 // larger than what the source ever supplies, so the body never completes on its own + + start := time.Now() + err = client.Put(ctx, metadata, sourceResp.Body, defaultTargetDeployOptions()) + elapsed := time.Since(start) + require.Error(t, err) + assert.Less(t, elapsed, 2*time.Second, + "Put must abort a stalled mid-body write on ctx cancellation, not block forever on the source reader") +} + +func TestTargetClient_ApplyProperties_ctxCancelAbortsInFlightPatch(t *testing.T) { + unblock := make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + <-unblock + w.WriteHeader(http.StatusNoContent) + })) + defer server.Close() + defer close(unblock) + + client, err := NewTargetClient(context.Background(), newTestTargetServerDetails(server.URL), nil) + require.NoError(t, err) + + metadata := testTargetMetadata() + metadata.Properties = largeEligiblePropertiesMap(t) + + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(50 * time.Millisecond) + cancel() + }() + + start := time.Now() + _, err = client.ApplyProperties(ctx, metadata, defaultTargetDeployOptions()) + elapsed := time.Since(start) + require.Error(t, err) + assert.Less(t, elapsed, 2*time.Second, + "ApplyProperties' PATCH must be bound to the per-call ctx and abort mid-flight on cancellation") +} + +func TestTargetClient_CreateFolder_errorReleasesEligibleProperties(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write([]byte(`{"errors":[{"message":"forbidden"}]}`)) + })) + defer server.Close() + + client, err := NewTargetClient(context.Background(), newTestTargetServerDetails(server.URL), nil) + require.NoError(t, err) + + metadata := &SourceFileMetadata{ + Repo: testTargetRepo, + Path: "empty-dir", + Properties: map[string][]string{"build.name": {"app"}}, + } + options := defaultTargetDeployOptions() + err = client.CreateFolder(context.Background(), metadata, options) + require.Error(t, err) + assert.False(t, eligibleCacheHas(client, metadata, options), + "failed CreateFolder must release eligibleCache without ApplyProperties") +} + +func eligibleCacheHas(tc *TargetClient, metadata *SourceFileMetadata, options TargetDeployOptions) bool { + tc.eligibleMu.Lock() + defer tc.eligibleMu.Unlock() + _, ok := tc.eligibleCache[eligiblePropertiesCacheKey{metadata: metadata, packageType: options.PackageType}] + return ok +} + +func Test_filterEligibleProperties_usesPackageType(t *testing.T) { + properties := map[string][]string{ + "build.name": {"app"}, + "npm.name": {"pkg"}, + "npm.version": {"1.0.0"}, + } + options := TargetDeployOptions{PackageType: "npm"} + eligibleProps, skippedLargeProps := filterEligibleProperties(properties, options) + assert.False(t, skippedLargeProps) + assert.Equal(t, 1, eligibleProps.KeysLen()) + assert.Equal(t, map[string][]string{"build.name": {"app"}}, eligibleProps.ToMap()) +} + +func testStatsMetadata() *SourceFileMetadata { + metadata := testTargetMetadata() + metadata.DownloadCount = 5 + metadata.LastDownloaded = 1788945284685 + metadata.LastDownloadedBy = "admin" + return metadata +} + +func Test_skipsDownloadStatistics(t *testing.T) { + tests := []struct { + name string + options TargetDeployOptions + want bool + }{ + {name: "generic", options: defaultTargetDeployOptions(), want: false}, + {name: "distribution", options: TargetDeployOptions{PackageType: "distribution"}, want: false}, + {name: "support", options: TargetDeployOptions{PackageType: "support"}, want: false}, + {name: "releasebundles", options: TargetDeployOptions{PackageType: "releasebundles"}, want: false}, + {name: "buildInfoRepoFlag", options: TargetDeployOptions{BuildInfoRepo: true}, want: true}, + {name: "buildinfoPackageType", options: TargetDeployOptions{PackageType: "BuildInfo"}, want: true}, + {name: "pipeinfoPackageType", options: TargetDeployOptions{PackageType: "PipeInfo"}, want: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, skipsDownloadStatistics(tt.options)) + }) + } +} + +// slowReader trickles data out with a delay between chunks, simulating a large upload over a +// slow network: the total time to write the whole body can vastly exceed streamResponseHeaderTimeout. +type slowReader struct { + data []byte + pos int + chunkSize int + delay time.Duration +} + +func (r *slowReader) Read(p []byte) (int, error) { + if r.pos >= len(r.data) { + return 0, io.EOF + } + time.Sleep(r.delay) + n := r.chunkSize + if remaining := len(r.data) - r.pos; n > remaining { + n = remaining + } + n = copy(p, r.data[r.pos:r.pos+n]) + r.pos += n + return n, nil +} + +// TestTargetClient_Put_slowLargeUpload_notBoundedByResponseHeaderTimeout answers a direct +// question about the B-17(b) fix: does bounding ResponseHeaderTimeout break large, slow uploads? +// Per net/http's own doc for Transport.ResponseHeaderTimeout, the timer only starts "after fully +// writing the request (including its body, if any)" - so a slow but steadily-progressing upload, +// whose total wall-clock time is many multiples of streamResponseHeaderTimeout, must still +// succeed as long as the target responds promptly once the body is fully received. Only a peer +// that stalls AFTER receiving the full body (or before starting to respond at all) should trip +// the timeout - that scenario is covered separately by +// TestTargetClient_Put_nonRespondingPeer_boundedByResponseHeaderTimeout. +func TestTargetClient_Put_slowLargeUpload_notBoundedByResponseHeaderTimeout(t *testing.T) { + origTimeout := streamResponseHeaderTimeout + streamResponseHeaderTimeout = 150 * time.Millisecond + defer func() { streamResponseHeaderTimeout = origTimeout }() + + deployPath := "/" + testTargetRelativePath() + var receivedLen int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !strings.HasPrefix(r.URL.Path, deployPath) { + http.Error(w, "unexpected request", http.StatusInternalServerError) + return + } + body, err := io.ReadAll(r.Body) + require.NoError(t, err) + receivedLen = len(body) + w.WriteHeader(http.StatusCreated) + })) + defer server.Close() + + client, err := NewTargetClient(context.Background(), newTestTargetServerDetails(server.URL), nil) + require.NoError(t, err) + + // 20 chunks x 40ms = ~800ms total upload time - more than 5x streamResponseHeaderTimeout + // (150ms), yet the upload never stalls, so it must succeed. + payload := bytes.Repeat([]byte("x"), 200) + reader := &slowReader{data: payload, chunkSize: 10, delay: 40 * time.Millisecond} + + start := time.Now() + metadata := testTargetMetadata() + metadata.Properties = nil // avoid a matrix-param suffix on the deploy path + metadata.Size = int64(len(payload)) + err = client.Put(context.Background(), metadata, reader, defaultTargetDeployOptions()) + elapsed := time.Since(start) + + require.NoError(t, err, "a slow-but-steady large upload must not be cut short by streamResponseHeaderTimeout") + assert.Equal(t, len(payload), receivedLen) + assert.Greater(t, elapsed, 5*streamResponseHeaderTimeout, "sanity check: the upload should genuinely have taken longer than the response-header timeout") +} + +// TestTargetClient_Put_nonRespondingPeer_boundedByResponseHeaderTimeout is the target-side +// counterpart of B-17(b) (see sourceclient_test.go's +// TestGetFileReader_nonRespondingPeer_boundedByResponseHeaderTimeout): without --proxy-key, +// NewTargetClient used to fall through to client-go's default transport for the streaming PUT +// path, which has no ResponseHeaderTimeout. A target peer that accepts the connection but never +// writes a response would hang the PUT indefinitely. +func TestTargetClient_Put_nonRespondingPeer_boundedByResponseHeaderTimeout(t *testing.T) { + origTimeout := streamResponseHeaderTimeout + streamResponseHeaderTimeout = 200 * time.Millisecond + defer func() { streamResponseHeaderTimeout = origTimeout }() + + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + defer func() { _ = ln.Close() }() + go func() { + for { + conn, acceptErr := ln.Accept() + if acceptErr != nil { + return + } + go func(c net.Conn) { + buf := make([]byte, 4096) + for { + if _, readErr := c.Read(buf); readErr != nil { + return + } + } + }(conn) + } + }() + + client, err := NewTargetClient(context.Background(), newTestTargetServerDetails("http://"+ln.Addr().String()), nil) + require.NoError(t, err) + + done := make(chan error, 1) + go func() { + done <- client.Put(context.Background(), testTargetMetadata(), strings.NewReader("hello world"), defaultTargetDeployOptions()) + }() + + select { + case err := <-done: + assert.Error(t, err, "a non-responding target peer must fail once the response-header timeout elapses, not hang or succeed") + case <-time.After(5 * time.Second): + t.Fatal("Put hung indefinitely against a non-responding target peer instead of being bounded by streamResponseHeaderTimeout") + } +} + +func TestTargetClient_ApplyStatistics_skipsInfoRepos(t *testing.T) { + requestCount := 0 + server := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) { + requestCount++ + })) + defer server.Close() + + client, err := NewTargetClient(context.Background(), newTestTargetServerDetails(server.URL), nil) + require.NoError(t, err) + + for _, options := range []TargetDeployOptions{ + {BuildInfoRepo: true}, + {PackageType: "buildinfo"}, + {PackageType: "pipeinfo"}, + } { + requestCount = 0 + err = client.ApplyStatistics(context.Background(), testStatsMetadata(), options) + require.NoError(t, err) + assert.Zero(t, requestCount, "options=%+v", options) + } +} diff --git a/artifactory/commands/transferfiles/transfer.go b/artifactory/commands/transferfiles/transfer.go index 73d4d9bae..05ff8d34c 100644 --- a/artifactory/commands/transferfiles/transfer.go +++ b/artifactory/commands/transferfiles/transfer.go @@ -5,27 +5,26 @@ import ( "encoding/json" "errors" "fmt" + "net/http" "os" "os/signal" "path/filepath" "strconv" - "strings" "syscall" "time" "github.com/jfrog/gofrog/safeconvert" - "github.com/jfrog/jfrog-client-go/artifactory/services" - "github.com/jfrog/gofrog/version" + "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/api" "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/state" "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/utils/precheckrunner" "github.com/jfrog/jfrog-cli-core/v2/artifactory/utils" "github.com/jfrog/jfrog-cli-core/v2/utils/config" "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" usageReporter "github.com/jfrog/jfrog-cli-core/v2/utils/usage" + "github.com/jfrog/jfrog-client-go/artifactory/services" serviceUtils "github.com/jfrog/jfrog-client-go/artifactory/services/utils" "github.com/jfrog/jfrog-client-go/artifactory/usage" - clientutils "github.com/jfrog/jfrog-client-go/utils" "github.com/jfrog/jfrog-client-go/utils/errorutils" "github.com/jfrog/jfrog-client-go/utils/io/fileutils" "github.com/jfrog/jfrog-client-go/utils/log" @@ -37,7 +36,8 @@ const ( fileWritersChannelSize = 500000 retries = 600 retriesWaitMilliSecs = 5000 - dataTransferPluginMinVersion = "1.7.0" + metadataTransferRetries = 5 + metadataRetryWaitMilliSecs = 1000 disableDistinctAqlMinVersion = "7.37" // Exact UTC-millisecond timestamp expected by --created-after / --downloaded-after. timestampFilterLayout = "2006-01-02T15:04:05.000Z" @@ -76,11 +76,16 @@ type TransferFilesCommand struct { status bool stop bool stopSignal chan os.Signal + threadDump func() error stateManager *state.TransferStateManager preChecks bool locallyGeneratedFilter *locallyGeneratedFilter // Optimization in Artifactory version 7.37 and above enables the exclusion of setting DISTINCT in SQL queries disabledDistinctiveAql bool + sourceClient *SourceClient + targetClient *TargetClient + fileTransfer *FileTransfer + targetProxyTransport http.RoundTripper } func NewTransferFilesCommand(sourceServer, targetServer *config.ServerDetails) (*TransferFilesCommand, error) { @@ -157,6 +162,7 @@ func parseTimestampFilter(field timestampFilterField, value string) (*timestampF return ×tampFilter{field: field, timestamp: value}, nil } +// SetProxyKey stores the --proxy-key value, which selects the proxy used for requests to the target. func (tdc *TransferFilesCommand) SetProxyKey(proxyKey string) { tdc.proxyKey = proxyKey } @@ -215,6 +221,16 @@ func (tdc *TransferFilesCommand) Run() (err error) { if tdc.stop { return tdc.signalStop() } + if tdc.proxyKey != "" { + proxyURL, parseErr := resolveProxyKeyURL(tdc.context, tdc.proxyKey, tdc.sourceServerDetails) + if parseErr != nil { + return parseErr + } + tdc.targetProxyTransport, err = newTargetProxyTransport(proxyURL, tdc.targetServerDetails) + if err != nil { + return err + } + } if tdc.timestampFilter, err = tdc.resolveTimestampFilter(); err != nil { return err } @@ -228,18 +244,22 @@ func (tdc *TransferFilesCommand) Run() (err error) { return err } - srcUpService, err := createSrcRtUserPluginServiceManager(tdc.context, tdc.sourceServerDetails) + tdc.sourceClient, err = NewSourceClient(tdc.context, tdc.sourceServerDetails) if err != nil { return err } - - if err = getAndValidateDataTransferPlugin(srcUpService); err != nil { + tdc.targetClient, err = NewTargetClient(tdc.context, tdc.targetServerDetails, tdc.targetProxyTransport) + if err != nil { return err } + tdc.fileTransfer = NewFileTransfer(tdc.sourceClient, tdc.targetClient, FileTransferOptions{}) - if err = tdc.verifySourceTargetConnectivity(srcUpService); err != nil { + log.Info("Verifying target Artifactory server connectivity...") + // Source compatibility is no longer plugin-gated; initDistinctAql() only toggles AQL distinct for >=7.37. + if err = tdc.targetClient.Ping(tdc.context); err != nil { return err } + log.Info("Connectivity check passed!") if err = tdc.initDistinctAql(); err != nil { return err @@ -265,12 +285,12 @@ func (tdc *TransferFilesCommand) Run() (err error) { return err } - sourceLocalRepos, sourceBuildInfoRepos, err := tdc.getAllLocalRepos(tdc.sourceServerDetails, tdc.sourceStorageInfoManager) + sourceLocalRepos, sourceBuildInfoRepos, err := tdc.getAllLocalRepos(tdc.sourceServerDetails, tdc.sourceStorageInfoManager, false) if err != nil { return err } allSourceLocalRepos := append(slices.Clone(sourceLocalRepos), sourceBuildInfoRepos...) - targetLocalRepos, targetBuildInfoRepos, err := tdc.getAllLocalRepos(tdc.targetServerDetails, tdc.targetStorageInfoManager) + targetLocalRepos, targetBuildInfoRepos, err := tdc.getAllLocalRepos(tdc.targetServerDetails, tdc.targetStorageInfoManager, true) if err != nil { return err } @@ -280,7 +300,7 @@ func (tdc *TransferFilesCommand) Run() (err error) { } // Handle interruptions - finishStopping, newPhase := tdc.handleStop(srcUpService) + finishStopping, newPhase := tdc.handleStop() defer finishStopping() if err = tdc.removeOldFilesIfNeeded(allSourceLocalRepos); err != nil { @@ -306,12 +326,12 @@ func (tdc *TransferFilesCommand) Run() (err error) { go tdc.reportTransferFilesUsage() // Transfer local repositories - if err := tdc.transferRepos(sourceLocalRepos, targetLocalRepos, false, newPhase, srcUpService); err != nil { + if err := tdc.transferRepos(sourceLocalRepos, targetLocalRepos, false, newPhase); err != nil { return tdc.cleanup(err, sourceLocalRepos) } // Transfer build-info repositories - if err := tdc.transferRepos(sourceBuildInfoRepos, targetBuildInfoRepos, true, newPhase, srcUpService); err != nil { + if err := tdc.transferRepos(sourceBuildInfoRepos, targetBuildInfoRepos, true, newPhase); err != nil { return tdc.cleanup(err, allSourceLocalRepos) } @@ -333,7 +353,7 @@ func (tdc *TransferFilesCommand) initStateManager(allSourceLocalRepos, sourceBui tdc.stateManager.OverallTransfer.TotalUnits = totalFiles tdc.stateManager.TotalRepositories.TotalUnits = int64(len(allSourceLocalRepos)) tdc.stateManager.OverallBiFiles.TotalUnits = totalBiFiles - tdc.stateManager.CurrentTotalTransferredBytes = 0 + tdc.stateManager.ResetCurrentTotalTransferredBytes() if !tdc.ignoreState { numberInitialErrors, e := getRetryErrorCount(allSourceLocalRepos) if e != nil { @@ -359,6 +379,8 @@ func (tdc *TransferFilesCommand) initStateManager(allSourceLocalRepos, sourceBui tdc.stateManager.TransferFailures = 0 tdc.stateManager.DelayedFiles = 0 } + // Counted per run (the skipped rows themselves stay in the errors files). + tdc.stateManager.SkippedSourceGone = 0 return nil } @@ -402,7 +424,7 @@ func (tdc *TransferFilesCommand) initStorageInfoManagers() error { } // Init target storage info manager - storageInfoManager, err = utils.NewStorageInfoManager(tdc.context, tdc.targetServerDetails) + storageInfoManager, err = utils.NewStorageInfoManagerWithHttpClient(tdc.context, tdc.targetServerDetails, tdc.targetServiceHTTPClient()) if err != nil { return err } @@ -412,7 +434,7 @@ func (tdc *TransferFilesCommand) initStorageInfoManagers() error { func (tdc *TransferFilesCommand) initDistinctAql() error { // Init source storage services manager - servicesManager, err := createTransferServiceManager(tdc.context, tdc.sourceServerDetails) + servicesManager, err := createTransferServiceManager(tdc.context, tdc.sourceServerDetails, nil) if err != nil { return err } @@ -435,7 +457,7 @@ func (tdc *TransferFilesCommand) initDistinctAql() error { // Creates the Pre-checks runner for the data transfer command func (tdc *TransferFilesCommand) NewTransferDataPreChecksRunner() (runner *precheckrunner.PreCheckRunner, err error) { // Get relevant repos - serviceManager, err := createTransferServiceManager(tdc.context, tdc.sourceServerDetails) + serviceManager, err := createTransferServiceManager(tdc.context, tdc.sourceServerDetails, nil) if err != nil { return } @@ -457,12 +479,12 @@ func (tdc *TransferFilesCommand) NewTransferDataPreChecksRunner() (runner *prech } func (tdc *TransferFilesCommand) transferRepos(sourceRepos []string, targetRepos []string, - buildInfoRepo bool, newPhase *transferPhase, srcUpService *srcUserPluginService) error { + buildInfoRepo bool, newPhase *transferPhase) error { for _, repoKey := range sourceRepos { if tdc.shouldStop() { return nil } - err := tdc.transferSingleRepo(repoKey, targetRepos, buildInfoRepo, newPhase, srcUpService) + err := tdc.transferSingleRepo(repoKey, targetRepos, buildInfoRepo, newPhase) if err != nil { return err } @@ -471,7 +493,7 @@ func (tdc *TransferFilesCommand) transferRepos(sourceRepos []string, targetRepos } func (tdc *TransferFilesCommand) transferSingleRepo(sourceRepoKey string, targetRepos []string, - buildInfoRepo bool, newPhase *transferPhase, srcUpService *srcUserPluginService) (err error) { + buildInfoRepo bool, newPhase *transferPhase) (err error) { if !slices.Contains(targetRepos, sourceRepoKey) { log.Error("repository '" + sourceRepoKey + "' does not exist in target. Skipping...") return @@ -512,17 +534,11 @@ func (tdc *TransferFilesCommand) transferSingleRepo(sourceRepoKey string, target if tdc.shouldStop() { return } - // Ensure the data structure which stores the upload tasks on Artifactory's side is wiped clean, - // in case some requests to delete handles tasks sent by JFrog CLI did not reach Artifactory. - err = stopTransferInArtifactory(tdc.sourceServerDetails, srcUpService) - if err != nil { - log.Error(err) - } *newPhase = createTransferPhase(currentPhaseId) if err = tdc.stateManager.SetRepoPhase(currentPhaseId); err != nil { return } - if err = tdc.startPhase(newPhase, sourceRepoKey, buildInfoRepo, *repoSummary, srcUpService, minChecksumDeploySize); err != nil { + if err = tdc.startPhase(newPhase, sourceRepoKey, buildInfoRepo, *repoSummary, minChecksumDeploySize); err != nil { return } } @@ -564,14 +580,18 @@ func (tdc *TransferFilesCommand) initTransferDir() error { func (tdc *TransferFilesCommand) removeOldFilesIfNeeded(repos []string) error { // If we ignore the old state, we need to remove all the old unused files so the process can start clean if tdc.ignoreState { - errFiles, err := getErrorsFiles(repos, true) - if err != nil { - return err - } - for _, file := range errFiles { - err = os.Remove(file) + // Both the retryable and the skipped errors of previous runs: stale skipped rows (e.g. items + // that vanished from the source) would otherwise reappear in the errors CSV of a clean run. + for _, isRetry := range []bool{true, false} { + errFiles, err := getErrorsFiles(repos, isRetry) if err != nil { - return errorutils.CheckError(err) + return err + } + for _, file := range errFiles { + err = os.Remove(file) + if err != nil { + return errorutils.CheckError(err) + } } } delayFiles, err := getDelayFiles(repos) @@ -588,8 +608,8 @@ func (tdc *TransferFilesCommand) removeOldFilesIfNeeded(repos []string) error { return nil } -func (tdc *TransferFilesCommand) startPhase(newPhase *transferPhase, repo string, buildInfoRepo bool, repoSummary serviceUtils.RepositorySummary, srcUpService *srcUserPluginService, minChecksumDeploySize int64) error { - tdc.initNewPhase(*newPhase, srcUpService, repoSummary, repo, buildInfoRepo, minChecksumDeploySize) +func (tdc *TransferFilesCommand) startPhase(newPhase *transferPhase, repo string, buildInfoRepo bool, repoSummary serviceUtils.RepositorySummary, minChecksumDeploySize int64) error { + tdc.initNewPhase(*newPhase, repoSummary, repo, buildInfoRepo, minChecksumDeploySize) skip, err := (*newPhase).shouldSkipPhase() if err != nil || skip { return err @@ -629,8 +649,7 @@ func (tdc *TransferFilesCommand) startPhase(newPhase *transferPhase, repo string // Handle interrupted signal. // shouldStop - Pointer to boolean variable, if the process gets interrupted shouldStop will be set to true // newPhase - The current running phase -// srcUpService - Source plugin service -func (tdc *TransferFilesCommand) handleStop(srcUpService *srcUserPluginService) (func(), *transferPhase) { +func (tdc *TransferFilesCommand) handleStop() (func(), *transferPhase) { var newPhase transferPhase finishStop := make(chan bool) signal.Notify(tdc.stopSignal, os.Interrupt, syscall.SIGTERM) @@ -642,7 +661,7 @@ func (tdc *TransferFilesCommand) handleStop(srcUpService *srcUserPluginService) return } // Before interrupting the process, do a thread dump - if err := doThreadDump(); err != nil { + if err := tdc.dumpThreads(); err != nil { log.Error(err) } tdc.cancelFunc() @@ -650,10 +669,6 @@ func (tdc *TransferFilesCommand) handleStop(srcUpService *srcUserPluginService) newPhase.StopGracefully() } log.Info("Gracefully stopping files transfer...") - err := stopTransferInArtifactory(tdc.sourceServerDetails, srcUpService) - if err != nil { - log.Error(err) - } }() // Return a cleanup function that closes the stopSignal channel and wait for close if needed @@ -667,16 +682,14 @@ func (tdc *TransferFilesCommand) handleStop(srcUpService *srcUserPluginService) }, &newPhase } -func (tdc *TransferFilesCommand) initNewPhase(newPhase transferPhase, srcUpService *srcUserPluginService, repoSummary serviceUtils.RepositorySummary, repoKey string, buildInfoRepo bool, minChecksumDeploySize int64) { +func (tdc *TransferFilesCommand) initNewPhase(newPhase transferPhase, repoSummary serviceUtils.RepositorySummary, repoKey string, buildInfoRepo bool, minChecksumDeploySize int64) { newPhase.setContext(tdc.context) newPhase.setRepoKey(repoKey) newPhase.setCheckExistenceInFilestore(tdc.checkExistenceInFilestore) newPhase.setSourceDetails(tdc.sourceServerDetails) newPhase.setTargetDetails(tdc.targetServerDetails) - newPhase.setSrcUserPluginService(srcUpService) newPhase.setRepoSummary(repoSummary) newPhase.setProgressBar(tdc.progressbar) - newPhase.setProxyKey(tdc.proxyKey) newPhase.setStateManager(tdc.stateManager) newPhase.setBuildInfo(buildInfoRepo) newPhase.setPackageType(repoSummary.PackageType) @@ -685,13 +698,24 @@ func (tdc *TransferFilesCommand) initNewPhase(newPhase transferPhase, srcUpServi newPhase.setMinCheckSumDeploySize(minChecksumDeploySize) newPhase.setIncludeFilesPatterns(tdc.includeFilesPatterns) newPhase.setTimestampFilter(tdc.timestampFilter) + if tdc.fileTransfer != nil { + tdc.fileTransfer.ConfigureOptions(FileTransferOptions{ + TargetDeployOptions: TargetDeployOptions{ + BuildInfoRepo: buildInfoRepo, + MinChecksumDeploySize: minChecksumDeploySize, + CheckExistenceInFilestore: tdc.checkExistenceInFilestore, + PackageType: repoSummary.PackageType, + }, + }) + } + newPhase.setFileTransfer(tdc.fileTransfer) } // Get all local and build-info repositories of the input server // serverDetails - Source or target server details // storageInfoManager - Source or target storage info manager -func (tdc *TransferFilesCommand) getAllLocalRepos(serverDetails *config.ServerDetails, storageInfoManager *utils.StorageInfoManager) ([]string, []string, error) { - serviceManager, err := createTransferServiceManager(tdc.context, serverDetails) +func (tdc *TransferFilesCommand) getAllLocalRepos(serverDetails *config.ServerDetails, storageInfoManager *utils.StorageInfoManager, isTarget bool) ([]string, []string, error) { + serviceManager, err := createTransferServiceManager(tdc.context, serverDetails, tdc.httpClientForServer(isTarget)) if err != nil { return []string{}, []string{}, err } @@ -733,13 +757,18 @@ func (tdc *TransferFilesCommand) initCurThreads(buildInfoRepo bool) error { log.Info("Build info transferring - using reduced number of threads") } } + if tdc.stateManager != nil { + if err = tdc.stateManager.SetWorkingThreads(curChunkUploaderThreads); err != nil { + return err + } + } log.Info("Running with maximum", strconv.Itoa(curChunkUploaderThreads), "working threads...") return nil } func (tdc *TransferFilesCommand) initLocallyGeneratedFilter() error { - servicesManager, err := createTransferServiceManager(tdc.context, tdc.targetServerDetails) + servicesManager, err := createTransferServiceManager(tdc.context, tdc.targetServerDetails, tdc.targetServiceHTTPClient()) if err != nil { return err } @@ -768,6 +797,10 @@ func (tdc *TransferFilesCommand) cleanup(originalErr error, sourceRepos []string if originalErr == nil { log.Info("Files transfer is complete!") } + // Items whose source vanished are skipped without failing the run; make the count visible. + if skippedGone := takeSourceItemGoneSkipCount(); skippedGone > 0 { + log.Warn(fmt.Sprintf("%d item(s) were skipped because they no longer exist in the source (deleted during the transfer). They are listed in the errors summary CSV with status %s and will not be retried automatically; verify they were really deleted.", skippedGone, api.SkippedSourceItemGone)) + } if tdc.stateManager.CurrentRepo.Name != "" { e := tdc.stateManager.SaveStateAndSnapshots() if e != nil { @@ -815,7 +848,7 @@ func (tdc *TransferFilesCommand) handleMaxUniqueSnapshots(repoSummary *serviceUt // If it's a Maven, Gradle, NuGet, Ivy, SBT or Docker repository, update its max unique snapshots setting to 0. // srcMaxUniqueSnapshots == -1 means it's a repository of another package type. if srcMaxUniqueSnapshots != -1 { - err = updateMaxUniqueSnapshots(tdc.context, tdc.targetServerDetails, repoSummary, 0) + err = updateMaxUniqueSnapshots(tdc.context, tdc.targetServerDetails, repoSummary, 0, tdc.targetServiceHTTPClient()) if err != nil { return } @@ -824,7 +857,7 @@ func (tdc *TransferFilesCommand) handleMaxUniqueSnapshots(repoSummary *serviceUt restoreFunc = func() (err error) { // Update the target repository's max unique snapshots setting to be the same as in the source, only if it's not 0. if srcMaxUniqueSnapshots > 0 { - err = updateMaxUniqueSnapshots(tdc.context, tdc.targetServerDetails, repoSummary, srcMaxUniqueSnapshots) + err = updateMaxUniqueSnapshots(tdc.context, tdc.targetServerDetails, repoSummary, srcMaxUniqueSnapshots, tdc.targetServiceHTTPClient()) } return } @@ -867,47 +900,6 @@ func (tdc *TransferFilesCommand) shouldStop() bool { return tdc.context.Err() != nil } -func (tdc *TransferFilesCommand) verifySourceTargetConnectivity(srcUpService *srcUserPluginService) error { - log.Info("Verifying source to target Artifactory servers connectivity...") - targetAuth := createTargetAuth(tdc.targetServerDetails, tdc.proxyKey) - err := srcUpService.verifyConnectivityRequest(targetAuth) - if err == nil { - log.Info("Connectivity check passed!") - } - return err -} - -func validateDataTransferPluginMinimumVersion(currentVersion string) error { - if strings.Contains(currentVersion, "SNAPSHOT") { - return nil - } - return clientutils.ValidateMinimumVersion(clientutils.DataTransfer, currentVersion, dataTransferPluginMinVersion) -} - -// Verify connection to the source Artifactory instance, and that the user plugin is installed, responsive, and stands in the minimal version requirement. -func getAndValidateDataTransferPlugin(srcUpService *srcUserPluginService) error { - verifyResponse, err := srcUpService.verifyCompatibilityRequest() - if err != nil { - errMsg := err.Error() - reason := "" - if strings.Contains(errMsg, "The execution name '") && strings.Contains(errMsg, "' could not be found") { - start := strings.Index(errMsg, "'") - missingApi := errMsg[start+1 : strings.Index(errMsg[start+1:], "'")+start+1] - reason = fmt.Sprintf(" This is because the '%s' API exposed by the plugin returns a '404 Not Found' response.", missingApi) - } - return errorutils.CheckErrorf("%s;\nIt looks like the 'data-transfer' user plugin isn't installed on the source instance."+ - "%s Please refer to the documentation available at "+coreutils.JFrogHelpUrl+"jfrog-hosting-models-documentation/transfer-artifactory-configuration-and-files-to-jfrog-cloud for installation instructions", - errMsg, reason) - } - - err = validateDataTransferPluginMinimumVersion(verifyResponse.Version) - if err != nil { - return err - } - log.Info("data-transfer plugin version: " + verifyResponse.Version) - return nil -} - // Loop on json files containing FilesErrors and collect them to one FilesErrors object. func parseErrorsFromLogFiles(logPaths []string) (allErrors FilesErrors, err error) { for _, logPath := range logPaths { @@ -939,6 +931,13 @@ func assertSupportedTransferDirStructure() error { return state.VerifyTransferRunStatusVersion() } +func (tdc *TransferFilesCommand) dumpThreads() error { + if tdc.threadDump != nil { + return tdc.threadDump() + } + return doThreadDump() +} + func doThreadDump() error { log.Info("Starting thread dumping...") threadDump, err := coreutils.NewProfiler().ThreadDump() diff --git a/artifactory/commands/transferfiles/transfer_files_test.go b/artifactory/commands/transferfiles/transfer_files_test.go new file mode 100644 index 000000000..93752892f --- /dev/null +++ b/artifactory/commands/transferfiles/transfer_files_test.go @@ -0,0 +1,315 @@ +package transferfiles + +import ( + "context" + "errors" + "sync" + "testing" + + "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/api" + "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/state" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type capturingFileTransferExecutor struct { + mu sync.Mutex + candidates []api.FileRepresentation +} + +func (c *capturingFileTransferExecutor) TransferFile(_ context.Context, candidate api.FileRepresentation) TransferResult { + c.mu.Lock() + defer c.mu.Unlock() + c.candidates = append(c.candidates, candidate) + return TransferResult{Candidate: candidate, Status: api.Success} +} + +func TestTransferFiles_invokesExecutorPerCandidateAndReportsFailure(t *testing.T) { + executor := &fakeFileTransferExecutor{} + pcWrapper := newProducerConsumerWrapper() + base := phaseBase{ + context: context.Background(), + fileTransfer: executor, + pcDetails: &pcWrapper, + } + delayHelper := delayUploadHelper{} + errorsChannelMng := createErrorsChannelMng() + + var collected []ExtendedFileUploadStatusResponse + var readWg sync.WaitGroup + readWg.Add(1) + go func() { + defer readWg.Done() + for e := range errorsChannelMng.channel { + collected = append(collected, e) + } + }() + + files := []api.FileRepresentation{ + {Repo: "repo", Path: "a", Name: "ok.jar", Size: 1}, + {Repo: "repo", Path: "b", Name: "fail.jar", Size: 2}, + } + shouldStop, err := transferFiles(files, base, delayHelper, &errorsChannelMng, &pcWrapper) + assert.NoError(t, err) + assert.False(t, shouldStop) + + assert.NoError(t, runProducerConsumers(&pcWrapper)) + errorsChannelMng.close() + readWg.Wait() + + assert.Equal(t, 2, executor.callCount) + assert.Len(t, collected, 1) + assert.Equal(t, api.Fail, collected[0].Status) + assert.Equal(t, "fail.jar", collected[0].Name) + assert.Equal(t, "transfer failed", collected[0].Reason) +} + +func TestTransferFiles_preservesNonEmptyDirFlag(t *testing.T) { + executor := &capturingFileTransferExecutor{} + pcWrapper := newProducerConsumerWrapper() + base := phaseBase{ + context: context.Background(), + fileTransfer: executor, + pcDetails: &pcWrapper, + } + delayHelper := delayUploadHelper{} + errorsChannelMng := createErrorsChannelMng() + + files := []api.FileRepresentation{ + {Repo: "repo", Path: "folder", NonEmptyDir: true}, + } + shouldStop, err := transferFiles(files, base, delayHelper, &errorsChannelMng, &pcWrapper) + assert.NoError(t, err) + assert.False(t, shouldStop) + assert.NoError(t, runProducerConsumers(&pcWrapper)) + + require.Len(t, executor.candidates, 1) + assert.True(t, executor.candidates[0].NonEmptyDir) + assert.Equal(t, "folder", executor.candidates[0].Path) +} + +func TestHandleTransferFileResult_successUpdatesStateAndTimeEstimation(t *testing.T) { + stateManager, cleanUp := state.InitStateTest(t) + defer cleanUp() + + const fileSize = int64(1024) + assert.NoError(t, stateManager.SetRepoState("test-repo", 0, 0, false, true)) + assert.NoError(t, stateManager.SetWorkingThreads(1)) + + phaseBase := &phaseBase{stateManager: stateManager} + result := TransferResult{ + Candidate: api.FileRepresentation{Repo: "test-repo", Path: "a", Name: "file.jar", Size: fileSize}, + Status: api.Success, + FileSize: fileSize, + DurationMillis: 100, + } + errorsChannelMng := createErrorsChannelMng() + + assert.NoError(t, handleTransferFileResult(phaseBase, result, &errorsChannelMng)) + assert.Equal(t, fileSize, stateManager.CurrentRepo.Phase1Info.TransferredSizeBytes) + assert.Equal(t, int64(1), stateManager.CurrentRepo.Phase1Info.TransferredUnits) + assert.Equal(t, uint64(fileSize), stateManager.CurrentTotalTransferredBytes) + assert.NotEmpty(t, stateManager.LastSpeeds) +} + +func TestHandleTransferFileResult_noByteSkipDoesNotUpdateTransferredBytesOrSpeed(t *testing.T) { + const candidateSize = int64(1024) + testCases := []struct { + name string + status api.ChunkFileStatusType + fileSize int64 + candidateName string + wantTransferredUnits int64 + }{ + // Named skips still count as a completed unit (matching the chunk-status polling path), + // so that --status can reach 100% even when metadata files are skipped. Gone source items are not + // transferred and are counted separately (SkippedSourceGone) instead of as transferred units. + {name: "source item gone", status: api.SkippedSourceItemGone, fileSize: 0, candidateName: "gone.jar", wantTransferredUnits: 0}, + {name: "metadata file", status: api.SkippedMetadataFile, fileSize: candidateSize, candidateName: "metadata.xml", wantTransferredUnits: 1}, + // Directories carry no name, so they're excluded from the unit count, just like UpdateChunkInState does elsewhere. + {name: "non-empty directory", status: api.SkippedNonEmptyDir, fileSize: candidateSize, candidateName: "", wantTransferredUnits: 0}, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + stateManager, cleanUp := state.InitStateTest(t) + defer cleanUp() + + // TotalUnits is set to the expected transferred units, mirroring a repo made up + // entirely of files like this one, so we can assert that --status can still reach + // 100% for repos with metadata files or gone source items (RTDEV: skipped files + // were not incrementing TransferredUnits, leaving --status stuck below 100%). + assert.NoError(t, stateManager.SetRepoState("test-repo", 0, testCase.wantTransferredUnits, false, true)) + assert.NoError(t, stateManager.SetWorkingThreads(1)) + + phaseBase := &phaseBase{stateManager: stateManager} + result := TransferResult{ + Candidate: api.FileRepresentation{Repo: "test-repo", Path: "a", Name: testCase.candidateName, Size: candidateSize}, + Status: testCase.status, + FileSize: testCase.fileSize, + DurationMillis: 100, + } + errorsChannelMng := createErrorsChannelMng() + + assert.NoError(t, handleTransferFileResult(phaseBase, result, &errorsChannelMng)) + assert.Zero(t, stateManager.CurrentRepo.Phase1Info.TransferredSizeBytes) + assert.Equal(t, testCase.wantTransferredUnits, stateManager.CurrentRepo.Phase1Info.TransferredUnits) + assert.Equal(t, stateManager.CurrentRepo.Phase1Info.TotalUnits, stateManager.CurrentRepo.Phase1Info.TransferredUnits) + assert.Zero(t, stateManager.CurrentTotalTransferredBytes) + assert.Empty(t, stateManager.LastSpeeds) + }) + } +} + +func TestHandleTransferFileResult_concurrentTimeEstimationUpdates(t *testing.T) { + stateManager, cleanUp := state.InitStateTest(t) + defer cleanUp() + + const ( + fileCount = 20 + fileSize = int64(1024) + ) + assert.NoError(t, stateManager.SetRepoState("test-repo", 0, 0, false, true)) + assert.NoError(t, stateManager.SetWorkingThreads(4)) + + phaseBase := &phaseBase{stateManager: stateManager} + errorsChannelMng := createErrorsChannelMng() + var wg sync.WaitGroup + for i := 0; i < fileCount; i++ { + wg.Add(1) + go func() { + defer wg.Done() + result := TransferResult{ + Candidate: api.FileRepresentation{Repo: "test-repo", Path: "a", Name: "file.jar", Size: fileSize}, + Status: api.Success, + FileSize: fileSize, + DurationMillis: 100, + } + assert.NoError(t, handleTransferFileResult(phaseBase, result, &errorsChannelMng)) + }() + } + wg.Wait() + + assert.Equal(t, uint64(fileCount*fileSize), stateManager.CurrentTotalTransferredBytes) + assert.NotEmpty(t, stateManager.LastSpeeds) +} + +func TestHandleTransferFileResult_failureReportsToErrorsChannel(t *testing.T) { + stateManager, cleanUp := state.InitStateTest(t) + defer cleanUp() + assert.NoError(t, stateManager.SetRepoState("test-repo", 0, 0, false, true)) + + phaseBase := &phaseBase{stateManager: stateManager} + result := TransferResult{ + Candidate: api.FileRepresentation{Repo: "test-repo", Path: "a", Name: "fail.jar", Size: 1}, + Status: api.Fail, + Err: errors.New("transfer failed"), + FileSize: 1, + } + errorsChannelMng := createErrorsChannelMng() + + var collected []ExtendedFileUploadStatusResponse + var readWg sync.WaitGroup + readWg.Add(1) + go func() { + defer readWg.Done() + for e := range errorsChannelMng.channel { + collected = append(collected, e) + } + }() + + assert.NoError(t, handleTransferFileResult(phaseBase, result, &errorsChannelMng)) + errorsChannelMng.close() + readWg.Wait() + + require.Len(t, collected, 1) + assert.Equal(t, api.Fail, collected[0].Status) + assert.Equal(t, "fail.jar", collected[0].Name) + assert.Equal(t, "transfer failed", collected[0].Reason) + assert.Zero(t, stateManager.CurrentRepo.Phase1Info.TransferredSizeBytes) +} + +// TestHandleTransferFileResult_sourceItemGone_reportedToSkippedErrors verifies that a gone-skip is +// forwarded to the errors channel (so it reaches the errors CSV as a skipped entry) and is NOT +// counted as a transferred unit (M2); it is counted separately as skipped-because-gone. +func TestHandleTransferFileResult_sourceItemGone_reportedToSkippedErrors(t *testing.T) { + stateManager, cleanUp := state.InitStateTest(t) + defer cleanUp() + + assert.NoError(t, stateManager.SetRepoState("test-repo", 0, 1, false, true)) + assert.NoError(t, stateManager.SetWorkingThreads(1)) + + phaseBase := &phaseBase{stateManager: stateManager} + result := TransferResult{ + Candidate: api.FileRepresentation{Repo: "test-repo", Path: "a", Name: "gone.jar", Size: 1024}, + Status: api.SkippedSourceItemGone, + SourceItemGone: true, + DurationMillis: 10, + } + errorsChannelMng := createErrorsChannelMng() + + assert.NoError(t, handleTransferFileResult(phaseBase, result, &errorsChannelMng)) + errorsChannelMng.close() + + var collected []ExtendedFileUploadStatusResponse + for e := range errorsChannelMng.channel { + collected = append(collected, e) + } + require.Len(t, collected, 1) + assert.Equal(t, api.SkippedSourceItemGone, collected[0].Status) + assert.Equal(t, "gone.jar", collected[0].Name) + assert.NotEmpty(t, collected[0].Reason, "the CSV row must say why the item was skipped") + assert.Zero(t, stateManager.CurrentRepo.Phase1Info.TransferredSizeBytes) + assert.Zero(t, stateManager.CurrentRepo.Phase1Info.TransferredUnits, "a gone item was not transferred") + assert.Zero(t, stateManager.OverallTransfer.TransferredUnits) + assert.Equal(t, uint64(1), stateManager.SkippedSourceGone) +} + +func TestHandleTransferFileResult_skippedLargePropsUpdatesProgressAndErrors(t *testing.T) { + stateManager, cleanUp := state.InitStateTest(t) + defer cleanUp() + + const fileSize = int64(2048) + assert.NoError(t, stateManager.SetRepoState("test-repo", 0, 0, false, true)) + assert.NoError(t, stateManager.SetWorkingThreads(1)) + + dirNode, err := stateManager.LookUpNode("pkg") + require.NoError(t, err) + require.NoError(t, dirNode.IncrementFilesCount(uint64(fileSize))) + require.NoError(t, dirNode.MarkDoneExploring()) + + phaseBase := &phaseBase{stateManager: stateManager} + result := TransferResult{ + Candidate: api.FileRepresentation{Repo: "test-repo", Path: "pkg", Name: "artifact.jar", Size: fileSize}, + Status: api.SkippedLargeProps, + FileSize: fileSize, + DurationMillis: 50, + } + errorsChannelMng := createErrorsChannelMng() + + var collected []ExtendedFileUploadStatusResponse + var readWg sync.WaitGroup + readWg.Add(1) + go func() { + defer readWg.Done() + for e := range errorsChannelMng.channel { + collected = append(collected, e) + } + }() + + assert.NoError(t, handleTransferFileResult(phaseBase, result, &errorsChannelMng)) + errorsChannelMng.close() + readWg.Wait() + + require.Len(t, collected, 1) + assert.Equal(t, api.SkippedLargeProps, collected[0].Status) + assert.Equal(t, "artifact.jar", collected[0].Name) + assert.Equal(t, fileSize, stateManager.CurrentRepo.Phase1Info.TransferredSizeBytes) + assert.Equal(t, int64(1), stateManager.CurrentRepo.Phase1Info.TransferredUnits) + assert.Equal(t, uint64(fileSize), stateManager.CurrentTotalTransferredBytes) + assert.NotEmpty(t, stateManager.LastSpeeds) + + transferredCount, _, err := dirNode.CalculateTransferredFilesAndSize() + require.NoError(t, err) + assert.EqualValues(t, 1, transferredCount) +} diff --git a/artifactory/commands/transferfiles/transfer_test.go b/artifactory/commands/transferfiles/transfer_test.go index 33e9c5879..4938c1f8d 100644 --- a/artifactory/commands/transferfiles/transfer_test.go +++ b/artifactory/commands/transferfiles/transfer_test.go @@ -3,37 +3,102 @@ package transferfiles import ( "context" "encoding/json" - "io" + "fmt" "net/http" - "net/http/httptest" "os" "path/filepath" - "sync" + "strings" + "sync/atomic" "testing" "time" "github.com/gocarina/gocsv" "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/api" - "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/state" "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/utils" coreUtils "github.com/jfrog/jfrog-cli-core/v2/artifactory/utils" commonTests "github.com/jfrog/jfrog-cli-core/v2/common/tests" - coreConfig "github.com/jfrog/jfrog-cli-core/v2/utils/config" "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" "github.com/jfrog/jfrog-cli-core/v2/utils/tests" - "github.com/jfrog/jfrog-client-go/artifactory" "github.com/jfrog/jfrog-client-go/artifactory/services" artifactoryUtils "github.com/jfrog/jfrog-client-go/artifactory/services/utils" - clientutils "github.com/jfrog/jfrog-client-go/utils" "github.com/jfrog/jfrog-client-go/utils/log" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) +func TestRunSetup_usesTargetPing(t *testing.T) { + cleanUpJfrogHome, err := tests.SetJfrogHome() + require.NoError(t, err) + defer cleanUpJfrogHome() + + var targetPingCalls atomic.Int32 + var pluginExecuteCalls atomic.Int32 + + sourceServer, sourceDetails, _ := commonTests.CreateRtRestsMockServer(t, func(w http.ResponseWriter, r *http.Request) { + if strings.Contains(r.RequestURI, "api/plugins/execute") { + pluginExecuteCalls.Add(1) + } + switch r.RequestURI { + case "/api/system/version": + w.WriteHeader(http.StatusOK) + _, err := w.Write([]byte(`{"version":"7.90.0"}`)) + assert.NoError(t, err) + case "/api/repositories?type=local": + w.WriteHeader(http.StatusOK) + _, err := w.Write([]byte(`[]`)) + assert.NoError(t, err) + case "/api/repositories?type=federated": + w.WriteHeader(http.StatusOK) + _, err := w.Write([]byte(`[]`)) + assert.NoError(t, err) + default: + w.WriteHeader(http.StatusOK) + } + }) + defer sourceServer.Close() + + targetServer, targetDetails, _ := commonTests.CreateRtRestsMockServer(t, func(w http.ResponseWriter, r *http.Request) { + if strings.Contains(r.RequestURI, "api/plugins/execute") { + pluginExecuteCalls.Add(1) + } + if r.RequestURI == "/api/system/ping" { + targetPingCalls.Add(1) + w.WriteHeader(http.StatusOK) + _, err := w.Write([]byte("OK")) + assert.NoError(t, err) + } + }) + defer targetServer.Close() + + cmd, err := NewTransferFilesCommand(sourceDetails, targetDetails) + require.NoError(t, err) + cmd.SetPreChecks(true) + + err = cmd.Run() + require.NoError(t, err) + assert.Greater(t, targetPingCalls.Load(), int32(0), "command setup should ping target /api/system/ping") + assert.Equal(t, int32(0), pluginExecuteCalls.Load(), "command setup must not call /api/plugins/execute") +} + +func TestRun_schemeLessProxyKeyWithoutSourceFailsClosed(t *testing.T) { + cmd, err := NewTransferFilesCommand(nil, nil) + require.NoError(t, err) + cmd.SetProxyKey("source-to-target") + + err = cmd.Run() + require.Error(t, err) + assert.Contains(t, err.Error(), `"source-to-target"`) + assert.Contains(t, err.Error(), "looked up on the source Artifactory") + assert.Contains(t, err.Error(), "source Artifactory is not configured") + assert.Contains(t, err.Error(), "http://proxy:3128") + assert.Contains(t, err.Error(), "HTTPS_PROXY") + assert.Contains(t, err.Error(), "NO_PROXY") +} + func TestHandleStopInitAndClose(t *testing.T) { transferFilesCommand, err := NewTransferFilesCommand(nil, nil) assert.NoError(t, err) - finishStopping, _ := transferFilesCommand.handleStop(nil) + finishStopping, _ := transferFilesCommand.handleStop() finishStopping() } @@ -46,6 +111,66 @@ func TestCancelFunc(t *testing.T) { assert.True(t, transferFilesCommand.shouldStop()) } +func TestCancelFuncCancelsInFlightTransferFile(t *testing.T) { + transferFilesCommand, err := NewTransferFilesCommand(nil, nil) + require.NoError(t, err) + + source := &cancelAwareTransferSource{started: make(chan struct{})} + fileTransfer := NewFileTransfer(source, &mockTransferTarget{}, FileTransferOptions{}) + resultChannel := make(chan TransferResult, 1) + go func() { + resultChannel <- fileTransfer.TransferFile(transferFilesCommand.context, testFileCandidate()) + }() + + <-source.started + transferFilesCommand.cancelFunc() + + select { + case result := <-resultChannel: + require.ErrorIs(t, result.Err, context.Canceled) + case <-time.After(2 * time.Second): + t.Fatal("in-flight TransferFile did not return promptly after stop") + } +} + +func TestHandleStopDumpsThreadsBeforeCancelingInFlightTransferFile(t *testing.T) { + transferFilesCommand, err := NewTransferFilesCommand(nil, nil) + require.NoError(t, err) + dumpStarted := make(chan struct{}) + finishDump := make(chan struct{}) + transferFilesCommand.threadDump = func() error { + close(dumpStarted) + <-finishDump + return nil + } + finishStopping, _ := transferFilesCommand.handleStop() + defer finishStopping() + + source := &cancelAwareTransferSource{started: make(chan struct{})} + fileTransfer := NewFileTransfer(source, &mockTransferTarget{}, FileTransferOptions{}) + resultChannel := make(chan TransferResult, 1) + go func() { + resultChannel <- fileTransfer.TransferFile(transferFilesCommand.context, testFileCandidate()) + }() + + <-source.started + transferFilesCommand.stopSignal <- os.Interrupt + <-dumpStarted + select { + case <-transferFilesCommand.context.Done(): + t.Fatal("transfer context was canceled before thread dump finished") + default: + } + + close(finishDump) + select { + case result := <-resultChannel: + require.ErrorIs(t, result.Err, context.Canceled) + case <-time.After(2 * time.Second): + t.Fatal("in-flight TransferFile did not return promptly after thread dump finished") + } +} + func TestSignalStop(t *testing.T) { cleanUpJfrogHome, err := tests.SetJfrogHome() assert.NoError(t, err) @@ -88,235 +213,6 @@ func TestSignalStopError(t *testing.T) { assert.EqualError(t, transferFilesCommand.signalStop(), "Graceful stop is already in progress. Please wait...") } -const ( - //#nosec G101 jfrog-ignore - Dummy token for tests. - firstUuidTokenForTest = "347cd3e9-86b6-4bec-9be9-e053a485f327" - //#nosec G101 jfrog-ignore - Dummy token for tests. - secondUuidTokenForTest = "af14706e-e0c1-4b7d-8791-6a18bd1fd339" - //#nosec G101 jfrog-ignore - Dummy token for tests. - nodeIdForTest = "nodea0gwihu76sk5g-artifactory-primary-0" -) - -func TestValidateDataTransferPluginMinimumVersion(t *testing.T) { - t.Run("valid version", func(t *testing.T) { testValidateDataTransferPluginMinimumVersion(t, "9.9.9", false) }) - t.Run("exact version", func(t *testing.T) { - testValidateDataTransferPluginMinimumVersion(t, dataTransferPluginMinVersion, false) - }) - t.Run("invalid version", func(t *testing.T) { testValidateDataTransferPluginMinimumVersion(t, "1.0.0", true) }) - t.Run("snapshot version", func(t *testing.T) { testValidateDataTransferPluginMinimumVersion(t, "1.0.x-SNAPSHOT", false) }) -} - -func testValidateDataTransferPluginMinimumVersion(t *testing.T, curVersion string, errorExpected bool) { - var pluginVersion string - testServer, serverDetails, _ := commonTests.CreateRtRestsMockServer(t, func(w http.ResponseWriter, r *http.Request) { - if r.RequestURI == "/"+utils.PluginsExecuteRestApi+"verifyCompatibility" { - content, err := json.Marshal(utils.VersionResponse{Version: pluginVersion}) - assert.NoError(t, err) - _, err = w.Write(content) - assert.NoError(t, err) - } - }) - defer testServer.Close() - srcPluginManager := initSrcUserPluginServiceManager(t, serverDetails) - - pluginVersion = curVersion - err := getAndValidateDataTransferPlugin(srcPluginManager) - if errorExpected { - assert.EqualError(t, err, clientutils.ValidateMinimumVersion(clientutils.DataTransfer, curVersion, dataTransferPluginMinVersion).Error()) - return - } - assert.NoError(t, err) -} - -func TestVerifySourceTargetConnectivity(t *testing.T) { - testServer, serverDetails, _ := commonTests.CreateRtRestsMockServer(t, func(w http.ResponseWriter, r *http.Request) { - if r.RequestURI == "/"+utils.PluginsExecuteRestApi+"verifySourceTargetConnectivity" { - w.WriteHeader(http.StatusOK) - } - }) - defer testServer.Close() - srcPluginManager := initSrcUserPluginServiceManager(t, serverDetails) - transferFilesCommand, err := NewTransferFilesCommand(serverDetails, serverDetails) - assert.NoError(t, err) - err = transferFilesCommand.verifySourceTargetConnectivity(srcPluginManager) - assert.NoError(t, err) -} - -func TestVerifySourceTargetConnectivityError(t *testing.T) { - testServer, serverDetails, _ := commonTests.CreateRtRestsMockServer(t, func(w http.ResponseWriter, r *http.Request) { - if r.RequestURI == "/"+utils.PluginsExecuteRestApi+"verifySourceTargetConnectivity" { - w.WriteHeader(http.StatusBadRequest) - _, err := w.Write([]byte("No connection to target")) - assert.NoError(t, err) - } - }) - defer testServer.Close() - srcPluginManager := initSrcUserPluginServiceManager(t, serverDetails) - transferFilesCommand, err := NewTransferFilesCommand(serverDetails, serverDetails) - assert.NoError(t, err) - err = transferFilesCommand.verifySourceTargetConnectivity(srcPluginManager) - assert.ErrorContains(t, err, "No connection to target") -} - -func initSrcUserPluginServiceManager(t *testing.T, serverDetails *coreConfig.ServerDetails) *srcUserPluginService { - srcPluginManager, err := createSrcRtUserPluginServiceManager(context.Background(), serverDetails) - assert.NoError(t, err) - return srcPluginManager -} - -func TestVerifyConfigImportPluginNotInstalled(t *testing.T) { - testServer, serverDetails, _ := commonTests.CreateRtRestsMockServer(t, func(w http.ResponseWriter, r *http.Request) { - if r.RequestURI == "/"+utils.PluginsExecuteRestApi+"dataTransferVersion" { - w.WriteHeader(http.StatusNotFound) - _, err := w.Write([]byte("Not found")) - assert.NoError(t, err) - } - }) - defer testServer.Close() - srcPluginManager := initSrcUserPluginServiceManager(t, serverDetails) - - _, err := srcPluginManager.version() - assert.ErrorContains(t, err, "Response from Artifactory: 404 Not Found.") -} - -func TestUploadChunkAndPollUploads(t *testing.T) { - stateManager, cleanUp := state.InitStateTest(t) - defer cleanUp() - - totalChunkStatusVisits := 0 - totalUploadChunkVisits := 0 - fileSample := api.FileRepresentation{ - Repo: repo1Key, - Path: "rel-path", - Name: "name-demo", - } - - testServer, serverDetails, _ := initPollUploadsTestMockServer(t, &totalChunkStatusVisits, &totalUploadChunkVisits, fileSample) - defer testServer.Close() - srcPluginManager := initSrcUserPluginServiceManager(t, serverDetails) - - assert.NoError(t, stateManager.SetRepoState(repo1Key, 0, 0, false, true)) - phaseBase := &phaseBase{context: context.Background(), stateManager: stateManager, srcUpService: srcPluginManager, repoKey: repo1Key} - uploadChunkAndPollTwice(t, phaseBase, fileSample) - - // Assert that exactly 2 requests to chunk status were made - // First request - get one DONE chunk and one IN PROGRESS - // Second Request - get DONE for the other chunk - assert.Equal(t, 2, totalChunkStatusVisits) -} - -// Sends chunk to upload, polls on chunk three times - once when it is still in progress, once after done received and once to notify back to the source. -func uploadChunkAndPollTwice(t *testing.T, phaseBase *phaseBase, fileSample api.FileRepresentation) { - curChunkUploaderThreads = coreUtils.DefaultThreads - curChunkBuilderThreads = coreUtils.DefaultThreads - uploadChunksChan := make(chan UploadedChunk, 3) - doneChan := make(chan bool, 1) - var runWaitGroup sync.WaitGroup - - pcWrapper := newProducerConsumerWrapper() - chunk := api.UploadChunk{} - chunk.AppendUploadCandidateIfNeeded(fileSample, false) - stopped := uploadChunkWhenPossible(&pcWrapper, phaseBase, chunk, uploadChunksChan, nil) - assert.False(t, stopped) - stopped = uploadChunkWhenPossible(&pcWrapper, phaseBase, chunk, uploadChunksChan, nil) - assert.False(t, stopped) - assert.Equal(t, 2, pcWrapper.totalProcessedUploadChunks) - - runWaitGroup.Add(1) - go func() { - defer runWaitGroup.Done() - pollUploads(&pcWrapper, phaseBase, phaseBase.srcUpService, uploadChunksChan, doneChan, nil) - }() - // Let the whole process run for a few chunk status checks, then mark it as done. - time.Sleep(5 * waitTimeBetweenChunkStatusSeconds * time.Second) - doneChan <- true - // Wait for the go routine to return. - runWaitGroup.Wait() -} - -func getUploadChunkMockResponse(t *testing.T, w http.ResponseWriter, totalUploadChunkVisits *int) { - w.WriteHeader(http.StatusAccepted) - var resp api.UploadChunkResponse - if *totalUploadChunkVisits == 1 { - resp = api.UploadChunkResponse{UuidTokenResponse: api.UuidTokenResponse{UuidToken: firstUuidTokenForTest}, NodeIdResponse: api.NodeIdResponse{NodeId: nodeIdForTest}} - } else { - resp = api.UploadChunkResponse{UuidTokenResponse: api.UuidTokenResponse{UuidToken: secondUuidTokenForTest}, NodeIdResponse: api.NodeIdResponse{NodeId: nodeIdForTest}} - } - writeMockResponse(t, w, resp) -} - -func validateChunkStatusBody(t *testing.T, r *http.Request) { - // Read body - content, err := io.ReadAll(r.Body) - assert.NoError(t, err) - var actual api.UploadChunksStatusBody - assert.NoError(t, json.Unmarshal(content, &actual)) - - // Make sure all parameters as expected - if len(actual.ChunksToDelete) == 0 { - assert.Len(t, actual.AwaitingStatusChunks, 2) - assert.ElementsMatch(t, []api.ChunkId{firstUuidTokenForTest, secondUuidTokenForTest}, actual.AwaitingStatusChunks) - } else { - assert.Len(t, actual.ChunksToDelete, 1) - assert.Len(t, actual.AwaitingStatusChunks, 1) - assert.Equal(t, api.ChunkId(firstUuidTokenForTest), actual.ChunksToDelete[0]) - assert.Equal(t, api.ChunkId(secondUuidTokenForTest), actual.AwaitingStatusChunks[0]) - } - -} - -func getChunkStatusMockFirstResponse(t *testing.T, w http.ResponseWriter) { - resp := getSampleChunkStatus() - resp.ChunksStatus[0].Status = api.Done - writeMockResponse(t, w, resp) -} - -func getChunkStatusMockSecondResponse(t *testing.T, w http.ResponseWriter, file api.FileRepresentation) { - resp := api.UploadChunksStatusResponse{ - ChunksStatus: []api.ChunkStatus{ - { - UuidTokenResponse: api.UuidTokenResponse{UuidToken: secondUuidTokenForTest}, - Status: api.Done, - Files: []api.FileUploadStatusResponse{{FileRepresentation: file, Status: api.Success, StatusCode: http.StatusOK}}, - }, - }, - DeletedChunks: []string{ - firstUuidTokenForTest, - }, - NodeIdResponse: api.NodeIdResponse{ - NodeId: nodeIdForTest, - }, - } - - writeMockResponse(t, w, resp) -} - -func writeMockResponse(t *testing.T, w http.ResponseWriter, resp interface{}) { - content, err := json.Marshal(resp) - assert.NoError(t, err) - _, err = w.Write(content) - assert.NoError(t, err) -} - -func initPollUploadsTestMockServer(t *testing.T, totalChunkStatusVisits *int, totalUploadChunkVisits *int, file api.FileRepresentation) (*httptest.Server, *coreConfig.ServerDetails, artifactory.ArtifactoryServicesManager) { - return commonTests.CreateRtRestsMockServer(t, func(w http.ResponseWriter, r *http.Request) { - switch r.RequestURI { - case "/" + utils.PluginsExecuteRestApi + "uploadChunk": - *totalUploadChunkVisits++ - getUploadChunkMockResponse(t, w, totalUploadChunkVisits) - case "/" + utils.PluginsExecuteRestApi + syncChunks: - *totalChunkStatusVisits++ - validateChunkStatusBody(t, r) - // If already visited chunk status, return status done this time. - if *totalChunkStatusVisits == 1 { - getChunkStatusMockFirstResponse(t, w) - } else { - getChunkStatusMockSecondResponse(t, w, file) - } - } - }) -} - func TestGetAllLocalRepositories(t *testing.T) { // Prepare mock server testServer, serverDetails, _ := commonTests.CreateRtRestsMockServer(t, func(w http.ResponseWriter, r *http.Request) { @@ -362,7 +258,7 @@ func TestGetAllLocalRepositories(t *testing.T) { assert.NoError(t, err) storageInfoManager, err := coreUtils.NewStorageInfoManager(context.Background(), serverDetails) assert.NoError(t, err) - localRepos, localBuildInfoRepo, err := transferFilesCommand.getAllLocalRepos(serverDetails, storageInfoManager) + localRepos, localBuildInfoRepo, err := transferFilesCommand.getAllLocalRepos(serverDetails, storageInfoManager, false) assert.NoError(t, err) assert.ElementsMatch(t, []string{"repo-1", "repo-2", "federated-repo-1", "federated-repo-2"}, localRepos) assert.ElementsMatch(t, []string{"artifactory-build-info", "proj-build-info"}, localBuildInfoRepo) @@ -397,60 +293,6 @@ func TestInitStorageInfoManagers(t *testing.T) { assert.True(t, targetServerCalculated) } -func getSampleChunkStatus() api.UploadChunksStatusResponse { - return api.UploadChunksStatusResponse{ - NodeIdResponse: api.NodeIdResponse{NodeId: nodeIdForTest}, - ChunksStatus: []api.ChunkStatus{ - { - UuidTokenResponse: api.UuidTokenResponse{UuidToken: firstUuidTokenForTest}, - Status: api.InProgress, - Files: []api.FileUploadStatusResponse{ - { - FileRepresentation: api.FileRepresentation{ - Repo: "my-repo-local-2", - Path: "rel-path-2", - Name: "name-demo-2", - }, - }, - }, - }, - { - UuidTokenResponse: api.UuidTokenResponse{UuidToken: secondUuidTokenForTest}, - Status: api.InProgress, - Files: []api.FileUploadStatusResponse{ - { - FileRepresentation: api.FileRepresentation{ - Repo: "my-repo-local-1", - Path: "rel-path-1", - Name: "name-demo-1", - }, - }, - }, - }, - }, - } -} - -func TestCheckChunkStatusSync(t *testing.T) { - chunkStatus := getSampleChunkStatus() - manager := ChunksLifeCycleManager{ - nodeToChunksMap: map[api.NodeId]map[api.ChunkId]UploadedChunkData{}, - } - manager.nodeToChunksMap[nodeIdForTest] = map[api.ChunkId]UploadedChunkData{} - manager.nodeToChunksMap[nodeIdForTest][firstUuidTokenForTest] = UploadedChunkData{} - manager.nodeToChunksMap[nodeIdForTest][secondUuidTokenForTest] = UploadedChunkData{} - pcWrapper := newProducerConsumerWrapper() - errChanMng := createErrorsChannelMng() - checkChunkStatusSync(&pcWrapper, &chunkStatus, &manager, &errChanMng) - assert.Len(t, manager.nodeToChunksMap[nodeIdForTest], 2) - chunkStatus.ChunksStatus = chunkStatus.ChunksStatus[:len(chunkStatus.ChunksStatus)-1] - checkChunkStatusSync(&pcWrapper, &chunkStatus, &manager, &errChanMng) - assert.Len(t, manager.nodeToChunksMap[nodeIdForTest], 1) - chunkStatus.ChunksStatus = chunkStatus.ChunksStatus[:len(chunkStatus.ChunksStatus)-1] - checkChunkStatusSync(&pcWrapper, &chunkStatus, &manager, &errChanMng) - assert.Len(t, manager.nodeToChunksMap[nodeIdForTest], 0) -} - func TestCreateErrorsSummaryFile(t *testing.T) { cleanUpJfrogHome, err := tests.SetJfrogHome() assert.NoError(t, err) @@ -483,6 +325,63 @@ func TestCreateErrorsSummaryFile(t *testing.T) { assert.ElementsMatch(t, *expectedFileErrors, *actualFileErrors) } +// TestRemoveOldFilesIfNeeded_ignoreState_clearsSkippedErrors: --ignore-state starts clean, so the +// skipped-errors rows of earlier runs (e.g. "source item gone") must go too; otherwise a clean rerun +// reports stale rows in its errors CSV (M1). +func TestRemoveOldFilesIfNeeded_ignoreState_clearsSkippedErrors(t *testing.T) { + for _, ignoreState := range []bool{true, false} { + t.Run(fmt.Sprintf("ignoreState=%v", ignoreState), func(t *testing.T) { + cleanUpJfrogHome, err := tests.SetJfrogHome() + require.NoError(t, err) + defer cleanUpJfrogHome() + writeErrorRows(t, testRepoKey, 1, goneRow(testRepoKey, "a.bin"), api.FileUploadStatusResponse{ + FileRepresentation: api.FileRepresentation{Repo: testRepoKey, Path: "path", Name: "failed.bin"}, Status: api.Fail, Reason: "boom"}) + + cmd, err := NewTransferFilesCommand(nil, nil) + require.NoError(t, err) + cmd.ignoreState = ignoreState + require.NoError(t, cmd.removeOldFilesIfNeeded([]string{testRepoKey})) + + skipped, err := getErrorsFiles([]string{testRepoKey}, false) + require.NoError(t, err) + retryable, err := getErrorsFiles([]string{testRepoKey}, true) + require.NoError(t, err) + if ignoreState { + assert.Empty(t, skipped, "stale skipped rows must be removed") + assert.Empty(t, retryable) + } else { + assert.Len(t, skipped, 1) + assert.Len(t, retryable, 1) + } + }) + } +} + +// TestCleanup_warnsAboutSourceGoneSkips covers the end-of-run Warn: the count of items skipped +// because they vanished from the source is reported once and then reset. +func TestCleanup_warnsAboutSourceGoneSkips(t *testing.T) { + cleanUpJfrogHome, err := tests.SetJfrogHome() + require.NoError(t, err) + defer cleanUpJfrogHome() + takeSourceItemGoneSkipCount() + buffer, stderrBuffer, previousLog := tests.RedirectLogOutputToBuffer() + defer log.SetLogger(previousLog) + + cmd, err := NewTransferFilesCommand(nil, nil) + require.NoError(t, err) + logSourceItemGone(api.FileRepresentation{Repo: "r", Path: "p", Name: "a"}, "test") + logSourceItemGone(api.FileRepresentation{Repo: "r", Path: "p", Name: "b"}, "test") + require.NoError(t, cmd.cleanup(nil, []string{})) + out := buffer.String() + stderrBuffer.String() + assert.Contains(t, out, "2 item(s) were skipped because they no longer exist in the source") + assert.Contains(t, out, string(api.SkippedSourceItemGone)) + + buffer.Reset() + stderrBuffer.Reset() + require.NoError(t, cmd.cleanup(nil, []string{})) + assert.NotContains(t, buffer.String()+stderrBuffer.String(), "were skipped because they no longer exist", "the count is reset after being reported") +} + func TestResolveTimestampFilter(t *testing.T) { const validTs = "2025-01-01T00:00:00.000Z" @@ -600,6 +499,7 @@ func TestResolveTimestampFilter(t *testing.T) { func TestInitNewPhasePropagatesTimestampFilter(t *testing.T) { cmd, err := NewTransferFilesCommand(nil, nil) assert.NoError(t, err) + cmd.fileTransfer = NewFileTransfer(nil, nil, FileTransferOptions{}) cmd.SetCreatedAfter("2025-01-01T00:00:00.000Z") filter, err := cmd.resolveTimestampFilter() assert.NoError(t, err) @@ -607,6 +507,6 @@ func TestInitNewPhasePropagatesTimestampFilter(t *testing.T) { cmd.timestampFilter = filter phase := &fullTransferPhase{} - cmd.initNewPhase(phase, nil, artifactoryUtils.RepositorySummary{}, "repo1", false, 0) + cmd.initNewPhase(phase, artifactoryUtils.RepositorySummary{}, "repo1", false, 0) assert.Equal(t, filter, phase.timestampFilter) } diff --git a/artifactory/commands/transferfiles/utils.go b/artifactory/commands/transferfiles/utils.go index d6106eb4f..c5a3334ca 100644 --- a/artifactory/commands/transferfiles/utils.go +++ b/artifactory/commands/transferfiles/utils.go @@ -6,15 +6,15 @@ import ( "errors" "fmt" "io" + "net/http" "os" - "path" "path/filepath" "strconv" "strings" + "sync" "time" buildInfoUtils "github.com/jfrog/build-info-go/utils" - "github.com/jfrog/gofrog/datastructures" "github.com/jfrog/gofrog/parallel" "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/api" "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/state" @@ -29,17 +29,12 @@ import ( "github.com/jfrog/jfrog-client-go/utils/errorutils" "github.com/jfrog/jfrog-client-go/utils/io/fileutils" "github.com/jfrog/jfrog-client-go/utils/log" - "golang.org/x/exp/maps" ) const ( - waitTimeBetweenChunkStatusSeconds = 3 waitTimeBetweenThreadsUpdateSeconds = 20 DefaultAqlPaginationLimit = 10000 - SyncErrorReason = "un-synchronized chunk status due to network issue" - SyncErrorStatusCode = 404 - StopFileName = "stop" ) @@ -47,87 +42,6 @@ var AqlPaginationLimit = DefaultAqlPaginationLimit var curChunkBuilderThreads int var curChunkUploaderThreads int -type UploadedChunk struct { - api.UploadChunkResponse - UploadedChunkData -} - -type UploadedChunkData struct { - ChunkFiles []api.FileRepresentation - TimeSent time.Time -} - -type ChunksLifeCycleManager struct { - // deletedChunksSet stores chunk uuids that have received a 'DONE' response from the source Artifactory instance - // It is used to notify the source Artifactory instance that these chunks can be deleted from the source's status map. - deletedChunksSet *datastructures.Set[api.ChunkId] - // nodeToChunksMap stores a map of the node IDs of the source Artifactory instance, - // In each node, we store a map of the chunks that are currently in progress and their matching files. - // In case network fails, and the uploaded chunks data is lost, - // These chunks files will be written to the errors file using this map. - nodeToChunksMap map[api.NodeId]map[api.ChunkId]UploadedChunkData -} - -// Convert to map of nodeID to list of chunk IDs to allow printing it -func (clcm *ChunksLifeCycleManager) GetNodeIdToChunkIdsMap() map[api.NodeId][]api.ChunkId { - nodeIdToChunks := make(map[api.NodeId][]api.ChunkId, len(clcm.nodeToChunksMap)) - for nodeId, chunks := range clcm.nodeToChunksMap { - nodeIdToChunks[nodeId] = maps.Keys(chunks) - } - return nodeIdToChunks -} - -func (clcm *ChunksLifeCycleManager) GetInProgressTokensSlice() []api.ChunkId { - var inProgressTokens []api.ChunkId - for _, node := range clcm.nodeToChunksMap { - for id := range node { - inProgressTokens = append(inProgressTokens, id) - } - } - - return inProgressTokens -} - -func (clcm *ChunksLifeCycleManager) GetInProgressTokensSliceByNodeId(nodeId api.NodeId) []api.ChunkId { - var inProgressTokens []api.ChunkId - for chunkId := range clcm.nodeToChunksMap[nodeId] { - inProgressTokens = append(inProgressTokens, chunkId) - } - - return inProgressTokens -} - -// Save in the TransferRunStatus the chunks that have been in transit for more than 30 minutes. -// This allows them to be displayed using the '--status' option. -// stateManager - Transfer state manager -func (clcm *ChunksLifeCycleManager) StoreStaleChunks(stateManager *state.TransferStateManager) error { - var staleChunks []state.StaleChunks - for nodeId, chunkIdToData := range clcm.nodeToChunksMap { - staleNodeChunks := state.StaleChunks{NodeID: string(nodeId)} - for chunkId, uploadedChunkData := range chunkIdToData { - if time.Since(uploadedChunkData.TimeSent).Hours() < 0.5 { - continue - } - staleNodeChunk := state.StaleChunk{ - ChunkID: string(chunkId), - Sent: uploadedChunkData.TimeSent.Unix(), - } - for _, file := range uploadedChunkData.ChunkFiles { - var sizeStr string - if file.Size > 0 { - sizeStr = " (" + serviceUtils.ConvertIntToStorageSizeString(file.Size) + ")" - } - staleNodeChunk.Files = append(staleNodeChunk.Files, path.Join(file.Repo, file.Path, file.Name)+sizeStr) - } - staleNodeChunks.Chunks = append(staleNodeChunks.Chunks, staleNodeChunk) - } - if len(staleNodeChunks.Chunks) > 0 { - staleChunks = append(staleChunks, staleNodeChunks) - } - } - return stateManager.SetStaleChunks(staleChunks) -} - // Set the JFrog CLI temp dir to be ~/.jfrog/transfer/tmp/ func initTempDir() (unsetTempDir func(), err error) { // If JFROG_CLI_TEMP_DIR environment variable provided, use it @@ -161,16 +75,12 @@ func (m *InterruptionErr) Error() string { return "Files transfer was interrupted by user" } -func createTransferServiceManager(ctx context.Context, serverDetails *config.ServerDetails) (artifactory.ArtifactoryServicesManager, error) { - return utils.CreateServiceManagerWithContext(ctx, serverDetails, false, 0, retries, retriesWaitMilliSecs, time.Minute) +func createTransferServiceManager(ctx context.Context, serverDetails *config.ServerDetails, httpClient *http.Client) (artifactory.ArtifactoryServicesManager, error) { + return utils.CreateServiceManagerWithContextAndHttpClient(ctx, serverDetails, false, 0, retries, retriesWaitMilliSecs, time.Minute, httpClient) } -func createSrcRtUserPluginServiceManager(ctx context.Context, sourceRtDetails *config.ServerDetails) (*srcUserPluginService, error) { - serviceManager, err := createTransferServiceManager(ctx, sourceRtDetails) - if err != nil { - return nil, err - } - return NewSrcUserPluginService(serviceManager.GetConfig().GetServiceDetails(), serviceManager.Client()), nil +func createMetadataTransferServiceManager(ctx context.Context, serverDetails *config.ServerDetails, httpClient *http.Client) (artifactory.ArtifactoryServicesManager, error) { + return utils.CreateServiceManagerWithContextAndHttpClient(ctx, serverDetails, false, 0, metadataTransferRetries, metadataRetryWaitMilliSecs, time.Minute, httpClient) } func appendDistinctIfNeeded(disabledDistinctiveAql bool) string { @@ -181,7 +91,7 @@ func appendDistinctIfNeeded(disabledDistinctiveAql bool) string { } func runAql(ctx context.Context, sourceRtDetails *config.ServerDetails, query string) (result *serviceUtils.AqlSearchResult, err error) { - serviceManager, err := createTransferServiceManager(ctx, sourceRtDetails) + serviceManager, err := createTransferServiceManager(ctx, sourceRtDetails, nil) if err != nil { return nil, err } @@ -205,80 +115,6 @@ func runAql(ctx context.Context, sourceRtDetails *config.ServerDetails, query st return result, errorutils.CheckError(err) } -func createTargetAuth(targetRtDetails *config.ServerDetails, proxyKey string) api.TargetAuth { - targetAuth := api.TargetAuth{ - TargetArtifactoryUrl: targetRtDetails.ArtifactoryUrl, - TargetToken: targetRtDetails.AccessToken, - TargetProxyKey: proxyKey, - } - if targetAuth.TargetToken == "" { - targetAuth.TargetUsername = targetRtDetails.User - targetAuth.TargetPassword = targetRtDetails.Password - } - return targetAuth -} - -func handleFilesOfCompletedChunk(chunkFiles []api.FileUploadStatusResponse, errorsChannelMng *ErrorsChannelMng) (stopped bool) { - for _, file := range chunkFiles { - if file.Status == api.Fail || file.Status == api.SkippedLargeProps { - stopped = addErrorToChannel(errorsChannelMng, file) - if stopped { - return - } - } - } - return -} - -// Uploads chunk when there is room in queue. -// This is a blocking method. -func uploadChunkWhenPossible(pcWrapper *producerConsumerWrapper, phaseBase *phaseBase, chunk api.UploadChunk, uploadTokensChan chan UploadedChunk, errorsChannelMng *ErrorsChannelMng) (stopped bool) { - for { - if ShouldStop(phaseBase, nil, errorsChannelMng) { - return true - } - // If increment done, this go routine can proceed to upload the chunk. Otherwise, sleep and try again. - isIncr := pcWrapper.incProcessedChunksWhenPossible() - if !isIncr { - time.Sleep(waitTimeBetweenChunkStatusSeconds * time.Second) - continue - } - err := uploadChunkAndAddToken(phaseBase.srcUpService, chunk, uploadTokensChan) - if err != nil { - // Chunk not uploaded due to error. Reduce processed chunks count and send all chunk content to error channel, so that the files could be uploaded on next run. - pcWrapper.decProcessedChunks() - // If the transfer is interrupted by the user, we shouldn't write it in the CSV file - if errors.Is(err, context.Canceled) { - return true - } - return sendAllChunkToErrorChannel(chunk, errorsChannelMng, err, phaseBase.stateManager) - } - return ShouldStop(phaseBase, nil, errorsChannelMng) - } -} - -func sendAllChunkToErrorChannel(chunk api.UploadChunk, errorsChannelMng *ErrorsChannelMng, errReason error, stateManager *state.TransferStateManager) (stopped bool) { - var failures []api.FileUploadStatusResponse - for _, file := range chunk.UploadCandidates { - fileFailureResponse := api.FileUploadStatusResponse{ - FileRepresentation: file, - Reason: errReason.Error(), - } - // In case an error occurred while handling errors files - stop transferring. - stopped = addErrorToChannel(errorsChannelMng, fileFailureResponse) - if stopped { - return - } - failures = append(failures, fileFailureResponse) - } - err := setChunkCompletedInRepoSnapshot(stateManager, failures) - if err != nil { - // We are logging the error instead of returning it since the original error is already handled. - log.Error(err) - } - return -} - // If repo snapshot is tracked, mark all files of a chunk as completed in their directory's node and check if node completed (done handling the directory and child directories). func setChunkCompletedInRepoSnapshot(stateManager *state.TransferStateManager, chunkFiles []api.FileUploadStatusResponse) (err error) { if !stateManager.IsRepoTransferSnapshotEnabled() { @@ -306,45 +142,123 @@ func setChunkCompletedInRepoSnapshot(stateManager *state.TransferStateManager, c return } -// Sends an upload chunk to the source Artifactory instance, to be handled asynchronously by the data-transfer plugin. -// An uuid token is returned in order to poll on it for status. -// This function sends the token to the uploadTokensChan for the pollUploads function to read and poll on. -func uploadChunkAndAddToken(sup *srcUserPluginService, chunk api.UploadChunk, uploadTokensChan chan UploadedChunk) error { - uploadResponse, err := sup.uploadChunk(chunk) - if err != nil { - return err +func GetChunkBuilderThreads() int { + return curChunkBuilderThreads +} + +func GetChunkUploaderThreads() int { + return curChunkUploaderThreads +} + +func transferFileHandler(phaseBase *phaseBase, file api.FileRepresentation, errorsChannelMng *ErrorsChannelMng) parallel.TaskFunc { + return func(threadId int) error { + logMsgPrefix := clientUtils.GetLogMsgPrefix(threadId, false) + log.Debug(logMsgPrefix + "Handling file transfer") + if ShouldStop(phaseBase, nil, errorsChannelMng) { + return errorutils.CheckErrorf("%sstopped", logMsgPrefix) + } + result := phaseBase.fileTransfer.TransferFile(phaseBase.context, file) + return handleTransferFileResult(phaseBase, result, errorsChannelMng) + } +} + +// transferFileResultMutex serializes transfer-state and repository-snapshot updates. +// It may be held while snapshot/state locks are acquired, but must be released before +// TimeEstimationManager.AddChunkStatus acquires the time-estimation lock. +var transferFileResultMutex sync.Mutex + +func handleTransferFileResult(phaseBase *phaseBase, result TransferResult, errorsChannelMng *ErrorsChannelMng) error { + fileStatus := result.ToFileUploadStatus() + if statusMovesNoBytes(result.Status) { + fileStatus.SizeBytes = 0 + } + // Gone-skips are reported like other skips so they appear in the errors CSV for audit; they + // are not retryable errors and do not count as failures. + if result.Status == api.Fail || result.Status == api.SkippedLargeProps || result.Status == api.SkippedSourceItemGone { + if addErrorToChannel(errorsChannelMng, fileStatus) { + return errorutils.CheckErrorf("stopped") + } + } + if result.Status == api.Fail { + if phaseBase.stateManager != nil { + withTransferFileResultLock(func() { + if err := setChunkCompletedInRepoSnapshot(phaseBase.stateManager, []api.FileUploadStatusResponse{fileStatus}); err != nil { + log.Error(err) + } + }) + } + return nil + } + if statusMovesNoBytes(result.Status) { + if phaseBase.stateManager != nil { + chunk := api.ChunkStatus{Files: []api.FileUploadStatusResponse{fileStatus}} + withTransferFileResultLock(func() { + if err := state.UpdateChunkInState(phaseBase.stateManager, &chunk); err != nil { + log.Error(err) + } + if err := setChunkCompletedInRepoSnapshot(phaseBase.stateManager, chunk.Files); err != nil { + log.Error(err) + } + }) + } + return nil } + if phaseBase.stateManager != nil { + chunk := api.ChunkStatus{Files: []api.FileUploadStatusResponse{fileStatus}} + withTransferFileResultLock(func() { + if err := state.UpdateChunkInState(phaseBase.stateManager, &chunk); err != nil { + log.Error(err) + } + if err := setChunkCompletedInRepoSnapshot(phaseBase.stateManager, chunk.Files); err != nil { + log.Error(err) + } + }) - // Add chunk data for polling. - log.Debug("Chunk sent to node " + uploadResponse.NodeId + ". Adding chunk token '" + uploadResponse.UuidToken + "' to poll on for status.") - uploadTokensChan <- newUploadedChunkStruct(uploadResponse, chunk) + timeEstMng := &phaseBase.stateManager.TimeEstimationManager + if err := timeEstMng.AddChunkStatus(chunk, result.DurationMillis); err != nil { + log.Error(err) + } + } return nil } -func newUploadedChunkStruct(uploadChunkResponse api.UploadChunkResponse, chunk api.UploadChunk) UploadedChunk { - return UploadedChunk{ - UploadChunkResponse: uploadChunkResponse, - UploadedChunkData: UploadedChunkData{ - ChunkFiles: chunk.UploadCandidates, - TimeSent: time.Now(), - }, - } +func withTransferFileResultLock(action func()) { + transferFileResultMutex.Lock() + defer transferFileResultMutex.Unlock() + action() } -func GetChunkBuilderThreads() int { - return curChunkBuilderThreads +func statusMovesNoBytes(status api.ChunkFileStatusType) bool { + return status == api.SkippedMetadataFile || + status == api.SkippedNonEmptyDir || + status == api.SkippedSourceItemGone } -func GetChunkUploaderThreads() int { - return curChunkUploaderThreads +func transferFiles(files []api.FileRepresentation, base phaseBase, delayHelper delayUploadHelper, errorsChannelMng *ErrorsChannelMng, pcWrapper *producerConsumerWrapper) (shouldStop bool, err error) { + for _, item := range files { + file := item + var delayed bool + delayed, shouldStop = delayHelper.delayUploadIfNecessary(base, file) + if shouldStop { + return + } + if delayed { + continue + } + _, err = pcWrapper.chunkUploaderProducerConsumer.AddTaskWithError(transferFileHandler(&base, file, errorsChannelMng), pcWrapper.errorsQueue.AddError) + if err != nil { + return + } + } + return } // Periodically reads settings file and updates the number of threads. // Number of threads in the settings files is expected to change by running a separate command. // The new number of threads should be almost immediately (checked every waitTimeBetweenThreadsUpdateSeconds) reflected on -// the CLI side (by updating the producer consumer if used and the local variable) and as a result reflected on the Artifactory User Plugin side. +// the CLI side (by updating the producer consumer if used and the local variable). // This method also looks for '~/.jfrog/transfer/stop' file and interrupts the transfer if exists. -func periodicallyUpdateThreadsAndStopStatus(pcWrapper *producerConsumerWrapper, doneChan chan bool, buildInfoRepo bool, stopSignal chan os.Signal) { +func periodicallyUpdateThreadsAndStopStatus(pcWrapper *producerConsumerWrapper, doneChan chan bool, buildInfoRepo bool, stateManager *state.TransferStateManager, stopSignal chan os.Signal) { log.Debug("Initializing polling on the settings and stop files...") for { time.Sleep(waitTimeBetweenThreadsUpdateSeconds * time.Second) @@ -355,13 +269,13 @@ func periodicallyUpdateThreadsAndStopStatus(pcWrapper *producerConsumerWrapper, log.Debug("Stopping the polling on the settings and stop files for the current phase.") return } - if err := updateThreads(pcWrapper, buildInfoRepo); err != nil { + if err := updateThreadsWithState(pcWrapper, buildInfoRepo, stateManager); err != nil { log.Error(err) } } } -func updateThreads(pcWrapper *producerConsumerWrapper, buildInfoRepo bool) error { +func updateThreadsWithState(pcWrapper *producerConsumerWrapper, buildInfoRepo bool, stateManager *state.TransferStateManager) error { settings, err := utils.LoadTransferSettings() if err != nil || settings == nil { return err @@ -377,6 +291,11 @@ func updateThreads(pcWrapper *producerConsumerWrapper, buildInfoRepo bool) error log.Info(fmt.Sprintf("Number of threads has been updated to %s (was %s).", strconv.Itoa(calculatedChunkUploaderThreads), strconv.Itoa(curChunkUploaderThreads))) curChunkBuilderThreads = calculatedChunkBuilderThreads curChunkUploaderThreads = calculatedChunkUploaderThreads + if stateManager != nil { + if err = stateManager.SetWorkingThreads(calculatedChunkUploaderThreads); err != nil { + return err + } + } } else { log.Debug(fmt.Sprintf("No change to the number of threads has been detected. Max chunks builder threads: %d. Max chunks uploader threads: %d.", calculatedChunkBuilderThreads, calculatedChunkUploaderThreads)) @@ -384,6 +303,15 @@ func updateThreads(pcWrapper *producerConsumerWrapper, buildInfoRepo bool) error return nil } +func shouldStopPolling(doneChan chan bool) bool { + select { + case done := <-doneChan: + return done + default: + } + return false +} + // Interrupt the transfer by populating the stopSignal channel with the Interrupt signal if the '~/.jfrog/transfer/stop' file exists. func interruptIfRequested(stopSignal chan os.Signal) error { transferDir, err := coreutils.GetJfrogTransferDir() @@ -409,60 +337,6 @@ func updateProducerConsumerMaxParallel(producerConsumer parallel.Runner, calcula } } -func uploadChunkWhenPossibleHandler(pcWrapper *producerConsumerWrapper, phaseBase *phaseBase, chunk api.UploadChunk, - uploadTokensChan chan UploadedChunk, errorsChannelMng *ErrorsChannelMng) parallel.TaskFunc { - return func(threadId int) error { - logMsgPrefix := clientUtils.GetLogMsgPrefix(threadId, false) - log.Debug(logMsgPrefix + "Handling chunk upload") - shouldStop := uploadChunkWhenPossible(pcWrapper, phaseBase, chunk, uploadTokensChan, errorsChannelMng) - if shouldStop { - // The specific error that triggered the stop is already in the errors channel - return errorutils.CheckErrorf("%sstopped", logMsgPrefix) - } - return nil - } -} - -// Collects files in chunks of size uploadChunkSize and sends them to be uploaded whenever possible (the amount of chunks uploaded is limited by the number of threads). -// An uuid token is returned after the chunk is sent and is being polled on for status. -func uploadByChunks(files []api.FileRepresentation, uploadTokensChan chan UploadedChunk, base phaseBase, delayHelper delayUploadHelper, errorsChannelMng *ErrorsChannelMng, pcWrapper *producerConsumerWrapper) (shouldStop bool, err error) { - curUploadChunk := api.UploadChunk{ - TargetAuth: createTargetAuth(base.targetRtDetails, base.proxyKey), - CheckExistenceInFilestore: base.checkExistenceInFilestore, - SkipFileFiltering: base.locallyGeneratedFilter.IsEnabled(), - MinCheckSumDeploySize: base.minCheckSumDeploySize, - } - - for _, item := range files { - file := api.FileRepresentation{Repo: item.Repo, Path: item.Path, Name: item.Name, Size: item.Size} - var delayed bool - delayed, shouldStop = delayHelper.delayUploadIfNecessary(base, file) - if shouldStop { - return - } - if delayed { - continue - } - curUploadChunk.AppendUploadCandidateIfNeeded(file, base.buildInfoRepo) - if curUploadChunk.IsChunkFull() { - _, err = pcWrapper.chunkUploaderProducerConsumer.AddTaskWithError(uploadChunkWhenPossibleHandler(pcWrapper, &base, curUploadChunk, uploadTokensChan, errorsChannelMng), pcWrapper.errorsQueue.AddError) - if err != nil { - return - } - // Empty the uploaded chunk. - curUploadChunk.UploadCandidates = []api.FileRepresentation{} - } - } - // Chunk didn't reach full size. Upload the remaining files. - if len(curUploadChunk.UploadCandidates) > 0 { - _, err = pcWrapper.chunkUploaderProducerConsumer.AddTaskWithError(uploadChunkWhenPossibleHandler(pcWrapper, &base, curUploadChunk, uploadTokensChan, errorsChannelMng), pcWrapper.errorsQueue.AddError) - if err != nil { - return - } - } - return -} - // Add a new error to the common error channel. // In case an error occurs when creating the upload errors files, we would like to stop the transfer right away and stop adding elements to the channel. func addErrorToChannel(errorsChannelMng *ErrorsChannelMng, file api.FileUploadStatusResponse) (stopped bool) { @@ -493,42 +367,12 @@ func ShouldStop(phase *phaseBase, delayHelper *delayUploadHelper, errorsChannelM return false } -func getRunningNodes(ctx context.Context, sourceRtDetails *config.ServerDetails) ([]string, error) { - serviceManager, err := createTransferServiceManager(ctx, sourceRtDetails) - if err != nil { - return nil, err - } - return serviceManager.GetRunningNodes() -} - -func stopTransferInArtifactoryNodes(srcUpService *srcUserPluginService, runningNodes []string) { - remainingNodesToStop := make(map[string]string) - for _, s := range runningNodes { - remainingNodesToStop[s] = s - } - log.Debug("Running Artifactory nodes to stop transfer on:", remainingNodesToStop) - // Send a stop command up to 5 times the number of Artifactory nodes, to make sure we reach out to all nodes - for i := 0; i < len(runningNodes)*5; i++ { - if len(remainingNodesToStop) == 0 { - log.Debug("Transfer on all Artifactory nodes stopped successfully") - return - } - nodeId, err := srcUpService.stop() - if err != nil { - log.Error(err) - } else { - log.Debug("Node " + nodeId + " stopped") - delete(remainingNodesToStop, nodeId) - } - } -} - // getMaxUniqueSnapshots gets the local repository's setting of max unique snapshots (Maven, Gradle, NuGet, Ivy and SBT) // or max unique tags (Docker). // For repositories of other package types or if an error is thrown, this function returns -1. func getMaxUniqueSnapshots(ctx context.Context, rtDetails *config.ServerDetails, repoSummary *serviceUtils.RepositorySummary) (maxUniqueSnapshots int, err error) { maxUniqueSnapshots = -1 - serviceManager, err := createTransferServiceManager(ctx, rtDetails) + serviceManager, err := createTransferServiceManager(ctx, rtDetails, nil) if err != nil { return } @@ -582,8 +426,8 @@ func getMaxUniqueSnapshots(ctx context.Context, rtDetails *config.ServerDetails, // updateMaxUniqueSnapshots updates the local repository's setting of max unique snapshots (Maven, Gradle, NuGet, Ivy and SBT) // or max unique tags (Docker). // For repositories of other package types, this function does nothing. -func updateMaxUniqueSnapshots(ctx context.Context, rtDetails *config.ServerDetails, repoSummary *serviceUtils.RepositorySummary, newMaxUniqueSnapshots int) error { - serviceManager, err := createTransferServiceManager(ctx, rtDetails) +func updateMaxUniqueSnapshots(ctx context.Context, rtDetails *config.ServerDetails, repoSummary *serviceUtils.RepositorySummary, newMaxUniqueSnapshots int, httpClient *http.Client) error { + serviceManager, err := createTransferServiceManager(ctx, rtDetails, httpClient) if err != nil { return err } @@ -682,17 +526,6 @@ func updateMaxDockerUniqueSnapshots(serviceManager artifactory.ArtifactoryServic return serviceManager.UpdateLocalRepository().Docker(repoParams) } -func stopTransferInArtifactory(serverDetails *config.ServerDetails, srcUpService *srcUserPluginService) error { - // To avoid situations where context has already been canceled, we use a new context here instead of the old context of the transfer phase. - runningNodes, err := getRunningNodes(context.Background(), serverDetails) - if err != nil { - return err - } else { - stopTransferInArtifactoryNodes(srcUpService, runningNodes) - } - return nil -} - func getJfrogTransferRepoDelaysDir(repoKey string) (string, error) { return state.GetJfrogTransferRepoSubDir(repoKey, coreutils.JfrogTransferDelaysDirName) } diff --git a/artifactory/commands/transferfiles/utils_test.go b/artifactory/commands/transferfiles/utils_test.go index ac754bcbf..de9130590 100644 --- a/artifactory/commands/transferfiles/utils_test.go +++ b/artifactory/commands/transferfiles/utils_test.go @@ -11,8 +11,8 @@ import ( "path/filepath" "strconv" "strings" + "sync/atomic" "testing" - "time" "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/api" "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/state" @@ -25,38 +25,11 @@ import ( "github.com/jfrog/jfrog-client-go/utils/log" clientutilstests "github.com/jfrog/jfrog-client-go/utils/tests" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) type transferFilesHandler func(w http.ResponseWriter, r *http.Request) -const runningNodesResponse = ` -{ - "isHa": true, - "nodes": [ - { - "id": "node-1", - "state": "RUNNING" - }, - { - "id": "node-2", - "state": "RUNNING" - }, - { - "id": "node-3", - "state": "RUNNING" - } - ] - } -` - -const ( - staleChunksNodeIdOne = "node-id-1" - staleChunksNodeIdTwo = "node-id-2" - staleChunksChunkId = "chunk-id" - staleChunksPath = "path-in-repo" - staleChunksName = "file-name" -) - func TestInitTempDir(t *testing.T) { // Create JFrog home cleanUpJfrogHome, err := tests.SetJfrogHome() @@ -75,43 +48,6 @@ func TestInitTempDir(t *testing.T) { assert.NotContains(t, fileutils.GetTempDirBase(), filepath.Join("transfer", "tmp")) } -func TestGetRunningNodes(t *testing.T) { - testServer, serverDetails, _ := createMockServer(t, func(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(http.StatusOK) - _, err := w.Write([]byte(runningNodesResponse)) - assert.NoError(t, err) - }) - defer testServer.Close() - - runningNodes, err := getRunningNodes(context.Background(), serverDetails) - assert.NoError(t, err) - assert.ElementsMatch(t, runningNodes, []string{"node-1", "node-2", "node-3"}) -} - -func TestStopTransferOnArtifactoryNodes(t *testing.T) { - stoppedNodeOne, stoppedNodeTwo := false, false - requestNumber := 0 - testServer, _, srcUpService := createMockServer(t, func(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(http.StatusOK) - var nodeId string - if requestNumber == 0 { - nodeId = "node-1" - stoppedNodeOne = true - } else { - nodeId = "node-2" - stoppedNodeTwo = true - } - _, err := fmt.Fprintf(w, `{"node_id": "%s"}`, nodeId) - assert.NoError(t, err) - requestNumber++ - }) - defer testServer.Close() - - stopTransferInArtifactoryNodes(srcUpService, []string{"node-1", "node-2"}) - assert.True(t, stoppedNodeOne) - assert.True(t, stoppedNodeTwo) -} - const repoConfigurationResponse = ` { "key" : "%[1]s-local", @@ -174,7 +110,7 @@ func TestGetMaxUniqueSnapshots(t *testing.T) { {docker, 3}, } - testServer, serverDetails, _ := createMockServer(t, func(w http.ResponseWriter, r *http.Request) { + testServer, serverDetails := createMockServer(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) packageType := strings.TrimSuffix(strings.TrimPrefix(r.RequestURI, "/api/repositories/"), "-local") var response string @@ -205,7 +141,7 @@ func TestGetMaxUniqueSnapshots(t *testing.T) { func TestUpdateMaxUniqueSnapshots(t *testing.T) { packageTypes := []string{conan, maven, gradle, nuget, ivy, sbt, docker} - testServer, serverDetails, _ := createMockServer(t, func(w http.ResponseWriter, r *http.Request) { + testServer, serverDetails := createMockServer(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) body, err := io.ReadAll(r.Body) assert.NoError(t, err) @@ -240,16 +176,46 @@ func TestUpdateMaxUniqueSnapshots(t *testing.T) { t.Run(packageType, func(t *testing.T) { lowerPackageType := strings.ToLower(packageType) repoSummary := &utils.RepositorySummary{RepoKey: lowerPackageType + "-local", PackageType: packageType, RepoType: "LOCAL"} - err := updateMaxUniqueSnapshots(context.Background(), serverDetails, repoSummary, 5) + err := updateMaxUniqueSnapshots(context.Background(), serverDetails, repoSummary, 5, nil) assert.NoError(t, err) repoSummary = &utils.RepositorySummary{RepoKey: lowerPackageType + "-federated", PackageType: packageType, RepoType: "FEDERATED"} - err = updateMaxUniqueSnapshots(context.Background(), serverDetails, repoSummary, 5) + err = updateMaxUniqueSnapshots(context.Background(), serverDetails, repoSummary, 5, nil) assert.NoError(t, err) }) } } +// TestUpdateMaxUniqueSnapshots_usesProvidedHttpClient locks in that the httpClient parameter +// (the target-only --proxy-key transport, when set) is actually used by the underlying service +// manager, rather than silently falling back to client-go's default transport. +func TestUpdateMaxUniqueSnapshots_usesProvidedHttpClient(t *testing.T) { + testServer, serverDetails := createMockServer(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + _, err := fmt.Fprint(w, "Repository updated successfully.") + assert.NoError(t, err) + }) + defer testServer.Close() + + var roundTripCount int32 + httpClient := &http.Client{Transport: countingRoundTripper{inner: http.DefaultTransport, count: &roundTripCount}} + + repoSummary := &utils.RepositorySummary{RepoKey: "maven-local", PackageType: maven, RepoType: "LOCAL"} + err := updateMaxUniqueSnapshots(context.Background(), serverDetails, repoSummary, 5, httpClient) + assert.NoError(t, err) + assert.Greater(t, atomic.LoadInt32(&roundTripCount), int32(0), "updateMaxUniqueSnapshots must route through the provided httpClient") +} + +type countingRoundTripper struct { + inner http.RoundTripper + count *int32 +} + +func (c countingRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + atomic.AddInt32(c.count, 1) + return c.inner.RoundTrip(req) +} + func TestInterruptIfRequested(t *testing.T) { cleanUpJfrogHome, err := tests.SetJfrogHome() assert.NoError(t, err) @@ -279,139 +245,10 @@ func TestInterruptIfRequested(t *testing.T) { assert.Equal(t, os.Interrupt, actualSignal) } -func TestGetNodeIdToChunkIdsMap(t *testing.T) { - // Test empty ChunksLifeCycleManager - chunksLifeCycleManager := ChunksLifeCycleManager{} - assert.Empty(t, chunksLifeCycleManager.GetNodeIdToChunkIdsMap()) - - // Create ChunksLifeCycleManager with 3 nodes - chunksLifeCycleManager = ChunksLifeCycleManager{ - nodeToChunksMap: make(map[api.NodeId]map[api.ChunkId]UploadedChunkData), - } - chunksLifeCycleManager.nodeToChunksMap["nodeId-1"] = map[api.ChunkId]UploadedChunkData{"0": {}, "1": {}} - chunksLifeCycleManager.nodeToChunksMap["nodeId-2"] = map[api.ChunkId]UploadedChunkData{"2": {}} - chunksLifeCycleManager.nodeToChunksMap["nodeId-3"] = map[api.ChunkId]UploadedChunkData{} - - // Generate the map and check response - nodeIdToChunkIdsMap := chunksLifeCycleManager.GetNodeIdToChunkIdsMap() - assert.ElementsMatch(t, nodeIdToChunkIdsMap["nodeId-1"], []api.ChunkId{"0", "1"}) - assert.ElementsMatch(t, nodeIdToChunkIdsMap["nodeId-2"], []api.ChunkId{"2"}) - assert.ElementsMatch(t, nodeIdToChunkIdsMap["nodeId-3"], []api.ChunkId{}) -} - -func TestStoreStaleChunksEmpty(t *testing.T) { - // Init state manager - stateManager, cleanUp := state.InitStateTest(t) - defer cleanUp() - - // Store empty stale chunks - chunksLifeCycleManager := ChunksLifeCycleManager{ - nodeToChunksMap: make(map[api.NodeId]map[api.ChunkId]UploadedChunkData), - } - assert.NoError(t, chunksLifeCycleManager.StoreStaleChunks(stateManager)) - - // Make sure no chunks - staleChunks, err := stateManager.GetStaleChunks() - assert.NoError(t, err) - assert.Empty(t, staleChunks) -} - -func TestStoreStaleChunksNoStale(t *testing.T) { - // Init state manager - stateManager, cleanUp := state.InitStateTest(t) - defer cleanUp() - - // Store chunk that is not stale - chunksLifeCycleManager := ChunksLifeCycleManager{ - nodeToChunksMap: map[api.NodeId]map[api.ChunkId]UploadedChunkData{ - staleChunksNodeIdOne: { - staleChunksChunkId: { - TimeSent: time.Now().Add(-time.Minute), - ChunkFiles: []api.FileRepresentation{{Repo: repo1Key, Path: staleChunksPath, Name: staleChunksName}}, - }, - }, - }, - } - assert.NoError(t, chunksLifeCycleManager.StoreStaleChunks(stateManager)) - - // Make sure no chunks - staleChunks, err := stateManager.GetStaleChunks() - assert.NoError(t, err) - assert.Empty(t, staleChunks) -} - -func TestStoreStaleChunksStale(t *testing.T) { - // Init state manager - stateManager, cleanUp := state.InitStateTest(t) - defer cleanUp() - - // Store stale chunk - sent := time.Now().Add(-time.Hour) - chunksLifeCycleManager := ChunksLifeCycleManager{ - nodeToChunksMap: map[api.NodeId]map[api.ChunkId]UploadedChunkData{ - staleChunksNodeIdOne: { - staleChunksChunkId: { - TimeSent: sent, - ChunkFiles: []api.FileRepresentation{{Repo: repo1Key, Path: staleChunksPath, Name: staleChunksName, Size: 100}}, - }, - }, - }, - } - assert.NoError(t, chunksLifeCycleManager.StoreStaleChunks(stateManager)) - - // Make sure the stale chunk was stored in the state - staleChunks, err := stateManager.GetStaleChunks() - assert.NoError(t, err) - assert.Len(t, staleChunks, 1) - assert.Equal(t, staleChunksNodeIdOne, staleChunks[0].NodeID) - assert.Len(t, staleChunks[0].Chunks, 1) - assert.Equal(t, staleChunksChunkId, staleChunks[0].Chunks[0].ChunkID) - assert.Equal(t, sent.Unix(), staleChunks[0].Chunks[0].Sent) - assert.Len(t, staleChunks[0].Chunks[0].Files, 1) - assert.Equal(t, fmt.Sprintf("%s/%s/%s (0.1KB)", repo1Key, staleChunksPath, staleChunksName), staleChunks[0].Chunks[0].Files[0]) -} - -func TestStoreStaleChunksTwoNodes(t *testing.T) { - // Init state manager - stateManager, cleanUp := state.InitStateTest(t) - defer cleanUp() - - // Store 1 stale chunk and 1 non-stale chunk - chunksLifeCycleManager := ChunksLifeCycleManager{ - nodeToChunksMap: map[api.NodeId]map[api.ChunkId]UploadedChunkData{ - staleChunksNodeIdOne: { - staleChunksChunkId: { - TimeSent: time.Now().Add(-time.Hour), // Older than 0.5 hours - ChunkFiles: []api.FileRepresentation{{Repo: repo1Key, Path: staleChunksPath, Name: staleChunksName, Size: 1024}}, - }, - }, - staleChunksNodeIdTwo: { - staleChunksChunkId: { - TimeSent: time.Now(), // Less than 0.5 hours - ChunkFiles: []api.FileRepresentation{{Repo: repo2Key, Path: staleChunksPath, Name: staleChunksName, Size: 0}}, - }, - }, - }, - } - assert.NoError(t, chunksLifeCycleManager.StoreStaleChunks(stateManager)) - - // Make sure only the stale chunk was stored in the state - staleChunks, err := stateManager.GetStaleChunks() - assert.NoError(t, err) - assert.Len(t, staleChunks, 1) - assert.Equal(t, staleChunksNodeIdOne, staleChunks[0].NodeID) -} - -// Create mock server to test transfer config commands -// t - The testing object -// testHandler - The HTTP handler of the test -func createMockServer(t *testing.T, testHandler transferFilesHandler) (*httptest.Server, *config.ServerDetails, *srcUserPluginService) { +func createMockServer(testHandler transferFilesHandler) (*httptest.Server, *config.ServerDetails) { testServer := httptest.NewServer(http.HandlerFunc(testHandler)) serverDetails := &config.ServerDetails{ArtifactoryUrl: testServer.URL + "/"} - - serviceManager, err := createSrcRtUserPluginServiceManager(context.Background(), serverDetails) - assert.NoError(t, err) - return testServer, serverDetails, serviceManager + return testServer, serverDetails } func TestGetUniqueErrorOrDelayFilePath(t *testing.T) { @@ -467,13 +304,45 @@ func TestUpdateThreads(t *testing.T) { transferSettings := &artifactoryutils.TransferSettings{ThreadsNumber: testCase.threadsNumber} assert.NoError(t, artifactoryutils.SaveTransferSettings(transferSettings)) - assert.NoError(t, updateThreads(nil, testCase.buildInfo)) + assert.NoError(t, updateThreadsWithState(nil, testCase.buildInfo, nil)) assert.Equal(t, testCase.expectedChunkBuilderThreads, curChunkBuilderThreads) assert.Equal(t, testCase.expectedChunkUploaderThreads, curChunkUploaderThreads) }) } } +func TestUpdateThreads_updatesReportedWorkingThreads(t *testing.T) { + stateManager, cleanUp := state.InitStateTest(t) + defer cleanUp() + + const updatedThreads = 7 + curChunkBuilderThreads = 1 + curChunkUploaderThreads = 1 + require.NoError(t, artifactoryutils.SaveTransferSettings(&artifactoryutils.TransferSettings{ThreadsNumber: updatedThreads})) + + require.NoError(t, updateThreadsWithState(nil, false, stateManager)) + workingThreads, err := stateManager.GetWorkingThreads() + require.NoError(t, err) + assert.Equal(t, updatedThreads, workingThreads) +} + +func TestTransferServiceManagers_useOperationSpecificRetryBudgets(t *testing.T) { + details := newTestTargetServerDetails("http://127.0.0.1:1") + + sharedManager, err := createTransferServiceManager(context.Background(), details, nil) + require.NoError(t, err) + assert.Equal(t, retries, sharedManager.GetConfig().GetHttpRetries()) + + metadataManager, err := createMetadataTransferServiceManager(context.Background(), details, nil) + require.NoError(t, err) + assert.Equal(t, metadataTransferRetries, metadataManager.GetConfig().GetHttpRetries()) + assert.LessOrEqual(t, metadataManager.GetConfig().GetHttpRetries(), 5) + + streamManager, err := createStreamingTransferServiceManager(context.Background(), details, nil) + require.NoError(t, err) + assert.Zero(t, streamManager.GetConfig().GetHttpRetries()) +} + // Test cases for convertPatternToPathPrefix var convertPatternToPathPrefixTestCases = []struct { input string diff --git a/artifactory/commands/transferinstall/datatransferinstall.go b/artifactory/commands/transferinstall/datatransferinstall.go deleted file mode 100644 index 54518e2ba..000000000 --- a/artifactory/commands/transferinstall/datatransferinstall.go +++ /dev/null @@ -1,364 +0,0 @@ -package transferinstall - -import ( - "fmt" - "net/http" - "net/url" - "os" - "path" - "path/filepath" - "strings" - - biutils "github.com/jfrog/build-info-go/utils" - "github.com/jfrog/gofrog/version" - "github.com/jfrog/jfrog-cli-core/v2/artifactory/utils" - "github.com/jfrog/jfrog-cli-core/v2/utils/config" - "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" - "github.com/jfrog/jfrog-client-go/utils/errorutils" - "github.com/jfrog/jfrog-client-go/utils/io/fileutils" - "github.com/jfrog/jfrog-client-go/utils/log" -) - -const ( - pluginName = "data-transfer" - groovyFileName = "dataTransfer.groovy" - jarFileName = "data-transfer.jar" - dataTransferUrl = "https://releases.jfrog.io/artifactory/jfrog-releases/data-transfer" - libDir = "lib" - artifactory = "artifactory" - pluginReloadRestApi = "api/plugins/reload" - jfrogHomeEnvVar = "JFROG_HOME" - latest = "[RELEASE]" -) - -var ( - defaultSearchPath = filepath.Join("opt", "jfrog") - // Plugin directory locations - originalDirPath = FileItem{"etc", "plugins"} - v7DirPath = FileItem{"var", "etc", "artifactory", "plugins"} - // Error types - errNotValidDestination = fmt.Errorf("can't find the directory in which to install the data-transfer plugin. Please ensure you're running this command on the machine on which Artifactory is installed. You can also use the --home-dir option to specify the directory") - downloadConnectionErr = func(baseUrl, fileName, err string) error { - return fmt.Errorf("could not download the plugin file - '%s' from '%s' due to the following error: '%s'. If this machine has no network access to the download URL, you can download these files from another machine and place them in a directory on this machine. You can then run this command again with the --dir command option, with the directory containing the files as the value", fileName, baseUrl, err) - } - // Plugin files - transferPluginFiles = PluginFiles{ - FileItem{groovyFileName}, - FileItem{libDir, jarFileName}, - } -) - -type InstallDataTransferPluginCommand struct { - // The server that the plugin will be installed on - targetServer *config.ServerDetails - // install manager manages the list of all the plugin files and the optional target destinations for plugin directory - transferManger *PluginInstallManager - // The plugin version to download - installVersion *version.Version - // The local directory the plugin files will be copied from - localPluginFilesDir string - // The JFrog home directory path provided from cliutils.InstallPluginHomeDir flag - localJFrogHomePath string -} - -func (idtp *InstallDataTransferPluginCommand) CommandName() string { - return "rt_transfer_install" -} - -func (idtp *InstallDataTransferPluginCommand) ServerDetails() (*config.ServerDetails, error) { - return idtp.targetServer, nil -} - -// Set the local file system directory path that the plugin files will be copied from -func (idtp *InstallDataTransferPluginCommand) SetLocalPluginFiles(localDir string) *InstallDataTransferPluginCommand { - idtp.localPluginFilesDir = localDir - return idtp -} - -// Set the plugin version we want to download -func (idtp *InstallDataTransferPluginCommand) SetInstallVersion(installVersion *version.Version) *InstallDataTransferPluginCommand { - idtp.installVersion = installVersion - return idtp -} - -// Set the Jfrog home directory path -func (idtp *InstallDataTransferPluginCommand) SetJFrogHomePath(path string) *InstallDataTransferPluginCommand { - idtp.localJFrogHomePath = path - return idtp -} - -// Create InstallDataTransferCommand -func NewInstallDataTransferCommand(server *config.ServerDetails) *InstallDataTransferPluginCommand { - manager := NewArtifactoryPluginInstallManager(transferPluginFiles) - cmd := &InstallDataTransferPluginCommand{ - targetServer: server, - transferManger: manager, - // Latest - installVersion: nil, - localPluginFilesDir: "", - localJFrogHomePath: "", - } - return cmd -} - -// PluginInstallManager holds all the plugin files and optional plugin directory destinations. -// We construct the destination path by searching for the existing plugin directory destination from the options -// and joining the local path of the plugin file with the plugin directory -type PluginInstallManager struct { - // All the plugin files, represented as FileItem with path starting from inside the plugin directory - // i.e. for file plugins/lib/file we represent them as FileItem{"lib","file"} - files PluginFiles - // All the optional destinations of the plugin directory starting from the JFrog home directory represented as FileItem - destinations []FileItem -} - -// Creates a new Artifactory plugin installation manager -func NewArtifactoryPluginInstallManager(bundle PluginFiles) *PluginInstallManager { - manager := &PluginInstallManager{ - files: bundle, - destinations: []FileItem{}, - } - // Add all the optional destinations for the plugin dir - manager.addDestination(originalDirPath) - manager.addDestination(v7DirPath) - return manager -} - -// Add optional plugin directory location as destination -func (ftm *PluginInstallManager) addDestination(directory FileItem) { - ftm.destinations = append(ftm.destinations, directory) -} - -// Search all the local target directories that the plugin directory can exist in base on a given root JFrog Artifactory home directory -// We are searching by Joining rootDir/optionalDestination to find the right structure of the JFrog home directory. -// The first option that matched and the directory exists is returned as target. -func (ftm *PluginInstallManager) findDestination(rootDir string) (exists bool, destination FileItem, err error) { - if exists, err = fileutils.IsDirExists(rootDir, false); err != nil || !exists { - return - } - exists = false - // Search for the product Artifactory folder - folderItems, err := fileutils.ListFiles(rootDir, true) - if err != nil { - return - } - for _, folder := range folderItems { - if strings.Contains(folder, artifactory) { - // Search for the destination inside the folder - for _, optionalPluginDirDst := range ftm.destinations { - if exists, err = fileutils.IsDirExists(optionalPluginDirDst.toPath(folder), false); err != nil { - return - } - if exists { - destination = append([]string{folder}, optionalPluginDirDst...) - return - } - } - } - } - return -} - -// Represents a path to file/directory -// We represent file 'dir/dir2/file' as FileItem{"dir", "dir2", "file"} -// We represent directory 'dir/dir2/' as FileItem{"dir", "dir2"} -type FileItem []string - -// List of files represented as FileItem -type PluginFiles []FileItem - -// Get the name (last) componenet of the item -// i.e. for FileItem{"root", "", "dir", "file.ext"} -> "file.ext" -func (f *FileItem) Name() string { - size := len(*f) - if size == 0 { - return "" - } - return (*f)[size-1] -} - -// Get the directory FileItem representation of the item, removing the last componenet and ignoring empty entries. -// i.e. for FileItem{"root", "", "dir", "file.ext"} -> FileItem{"root", "dir"} -func (f *FileItem) Dirs() *FileItem { - dirs := FileItem{} - for i := 0; i < len(*f)-1; i++ { - dir := (*f)[i] - if dir != "" { - dirs = append(dirs, dir) - } - } - return &dirs -} - -// Split and get the name and directory componenets of the item -func (f *FileItem) SplitNameAndDirs() (string, *FileItem) { - return f.Name(), f.Dirs() -} - -// Convert the item to URL representation, adding prefix tokens as provided -func (f *FileItem) toURL(prefixUrl string) (string, error) { - myUrl, err := url.Parse(prefixUrl) - if err != nil { - return "", errorutils.CheckError(err) - } - myUrl.Path = path.Join(myUrl.Path, path.Join(*f...)) - return myUrl.String(), nil -} - -// Convert the item to file path representation, ignoring empty entries, adding prefix tokens as provided -func (f *FileItem) toPath(previousTokens ...string) string { - return filepath.Join(filepath.Join(previousTokens...), filepath.Join(*f.Dirs()...), f.Name()) -} - -// Get the plugin directory destination to install the plugin in it. The search is starting from JFrog home directory -func (idtp *InstallDataTransferPluginCommand) getPluginDirDestination() (target FileItem, err error) { - var exists bool - var envVal string - - // Flag override - if idtp.localJFrogHomePath != "" { - jfrogHomeDir := strings.TrimSpace(idtp.localJFrogHomePath) - log.Debug(fmt.Sprintf("Searching for the 'plugins' directory in the JFrog home directory '%s'.", jfrogHomeDir)) - if exists, target, err = idtp.transferManger.findDestination(jfrogHomeDir); err != nil || exists { - return - } - if !exists { - err = errNotValidDestination - return - } - } - // Environment variable override - if envVal, exists = os.LookupEnv(jfrogHomeEnvVar); exists { - jfrogHomeDir := strings.TrimSpace(envVal) - log.Debug(fmt.Sprintf("Searching for the 'plugins' directory in the JFrog home directory '%s' retrieved from the '%s' environment variable.", jfrogHomeDir, jfrogHomeEnvVar)) - if exists, target, err = idtp.transferManger.findDestination(jfrogHomeDir); err != nil || exists { - return - } - } - // Default value - if !coreutils.IsWindows() { - log.Debug(fmt.Sprintf("Searching for the 'plugins' directory in the default path '%s'.", defaultSearchPath)) - if exists, target, err = idtp.transferManger.findDestination(defaultSearchPath); err != nil || exists { - return - } - } - - err = errNotValidDestination - return -} - -// Transfers the file bundle from src to dst -type TransferAction func(src string, dst string, bundle PluginFiles) error - -// Returns the src path and a transfer action -func (idtp *InstallDataTransferPluginCommand) getTransferSourceAndAction() (src string, transferAction TransferAction, err error) { - // Check if local directory was provided - if idtp.localPluginFilesDir != "" { - src = idtp.localPluginFilesDir - transferAction = CopyFiles - log.Debug("Local plugin files provided, copying from file system.") - return - } - // Download files from web - var baseSrc *url.URL - if baseSrc, err = url.Parse(dataTransferUrl); err != nil { - return - } - if idtp.installVersion == nil { - // Latest - baseSrc.Path = path.Join(baseSrc.Path, latest) - log.Debug("Fetching latest version to the target.") - } else { - baseSrc.Path = path.Join(baseSrc.Path, idtp.installVersion.GetVersion()) - log.Debug(fmt.Sprintf("Fetching plugin version '%s' to the target.", idtp.installVersion.GetVersion())) - } - src = baseSrc.String() - transferAction = DownloadFiles - return -} - -// Download the plugin files from the given url to the target directory (create path if needed or override existing files) -func DownloadFiles(src string, pluginDir string, bundle PluginFiles) (err error) { - for _, file := range bundle { - fileName, fileDirs := file.SplitNameAndDirs() - srcURL, e := file.toURL(src) - if e != nil { - return e - } - dstDirPath := fileDirs.toPath(pluginDir) - dirURL, e := fileDirs.toURL(src) - if e != nil { - return e - } - log.Debug(fmt.Sprintf("Downloading '%s' from '%s' to '%s'", fileName, dirURL, dstDirPath)) - if err = fileutils.CreateDirIfNotExist(dstDirPath); err != nil { - return - } - if err = biutils.DownloadFile(filepath.Join(dstDirPath, fileName), srcURL); err != nil { - err = downloadConnectionErr(src, fileName, err.Error()) - return - } - } - return -} - -// Copy the plugin files from the given source to the target directory (create path if needed or override existing files) -func CopyFiles(src string, pluginDir string, bundle PluginFiles) (err error) { - for _, file := range bundle { - fileName, fileDirs := file.SplitNameAndDirs() - srcPath := filepath.Join(src, fileName) - dstDirPath := fileDirs.toPath(pluginDir) - log.Debug(fmt.Sprintf("Copying '%s' from '%s' to '%s'", fileName, src, dstDirPath)) - if err = fileutils.CreateDirIfNotExist(dstDirPath); err != nil { - return - } - if err = biutils.CopyFile(dstDirPath, srcPath); err != nil { - return - } - } - return -} - -// Send reload request to the Artifactory in order to reload the plugin files. -func (idtp *InstallDataTransferPluginCommand) sendReloadRequest() error { - serviceManager, err := utils.CreateServiceManager(idtp.targetServer, -1, 0, false) - if err != nil { - return err - } - serviceDetails := serviceManager.GetConfig().GetServiceDetails() - httpDetails := serviceDetails.CreateHttpClientDetails() - - resp, body, err := serviceManager.Client().SendPost(serviceDetails.GetUrl()+pluginReloadRestApi, []byte{}, &httpDetails) - if err != nil { - return err - } - if err = errorutils.CheckResponseStatusWithBody(resp, body, http.StatusOK); err != nil { - return err - } - return nil -} - -func (idtp *InstallDataTransferPluginCommand) Run() (err error) { - log.Info(coreutils.PrintBoldTitle(fmt.Sprintf("Installing '%s' plugin...", pluginName))) - - // Get source, destination and transfer action - dst, err := idtp.getPluginDirDestination() - if err != nil { - return - } - src, transferAction, err := idtp.getTransferSourceAndAction() - if err != nil { - return - } - // Execute transferring action - if err = transferAction(src, dst.toPath(), idtp.transferManger.files); err != nil { - return - } - // Reload plugins - if err = idtp.sendReloadRequest(); err != nil { - return - } - - log.Info(coreutils.PrintBoldTitle(fmt.Sprintf("The %s plugin installed successfully.", pluginName))) - return -} diff --git a/artifactory/commands/transferinstall/datatransferinstall_test.go b/artifactory/commands/transferinstall/datatransferinstall_test.go deleted file mode 100644 index 7abed0693..000000000 --- a/artifactory/commands/transferinstall/datatransferinstall_test.go +++ /dev/null @@ -1,256 +0,0 @@ -package transferinstall - -import ( - "fmt" - "net/http" - "net/url" - "os" - "path" - "path/filepath" - "reflect" - "runtime" - "strings" - "testing" - - "github.com/jfrog/gofrog/version" - commonTests "github.com/jfrog/jfrog-cli-core/v2/common/tests" - "github.com/jfrog/jfrog-cli-core/v2/utils/tests" - "github.com/jfrog/jfrog-client-go/utils/io/fileutils" - testsutils "github.com/jfrog/jfrog-client-go/utils/tests" - "github.com/stretchr/testify/assert" -) - -func TestPluginFileItemNameAndDirs(t *testing.T) { - cases := []struct { - expectedDirs FileItem - expectedName string - file FileItem - }{ - {FileItem{"dir", "dir2"}, "name.txt", FileItem{"dir", "dir2", "name.txt"}}, - {FileItem{}, "name.txt", FileItem{"name.txt"}}, - {FileItem{}, "", FileItem{}}, - {FileItem{}, "", FileItem{""}}, - {FileItem{}, "", FileItem{"", "", ""}}, - {FileItem{"dir"}, "", FileItem{"", "dir", ""}}, - {FileItem{"dir"}, "name", FileItem{"dir", "", "name"}}, - } - - for _, testCase := range cases { - testFileItemNameDirs(t, testCase.expectedDirs, testCase.expectedName, testCase.file) - } -} - -func testFileItemNameDirs(t *testing.T, expectedDirs FileItem, expectedName string, file FileItem) { - assert.Equal(t, expectedName, file.Name()) - assert.Equal(t, &expectedDirs, file.Dirs()) - fileName, fileDirs := file.SplitNameAndDirs() - assert.Equal(t, expectedName, fileName) - assert.Equal(t, &expectedDirs, fileDirs) - assert.Len(t, *fileDirs, len(expectedDirs)) - for i, DirName := range *fileDirs { - assert.Equal(t, expectedDirs[i], DirName) - } -} - -func TestPluginFileItemToUrlAndToPath(t *testing.T) { - cases := []struct { - expectedUrl string - expectedPath string - file FileItem - }{ - {"dir/dir2/name.txt", filepath.Join("dir", "dir2", "name.txt"), FileItem{"dir", "dir2", "name.txt"}}, - {"name.txt", "name.txt", FileItem{"name.txt"}}, - {"", "", FileItem{}}, - {"", "", FileItem{""}}, - {"", "", FileItem{"", ""}}, - {"dir", filepath.Join("dir", ""), FileItem{"", "dir", ""}}, - {"dir/name", filepath.Join("dir", "name"), FileItem{"dir", "", "name"}}, - } - - for _, testCase := range cases { - testFileItemToUrlAndToPath(t, testCase.expectedUrl, testCase.expectedPath, testCase.file) - } -} - -func testFileItemToUrlAndToPath(t *testing.T, expectedUrl string, expectedPath string, file FileItem) { - fUrl, err := file.toURL("") - assert.NoError(t, err) - assert.Equal(t, expectedUrl, fUrl) - assert.Equal(t, expectedPath, file.toPath("")) - prefix := "prefix" - assert.Equal(t, filepath.Join(prefix, expectedPath), file.toPath(prefix)) - expectedPrefixUrl := prefix - if expectedUrl != "" { - expectedPrefixUrl += "/" - } - fUrl, err = file.toURL(prefix) - assert.NoError(t, err) - assert.Equal(t, expectedPrefixUrl+expectedUrl, fUrl) -} - -func populateDirWith(rootDir string, dirs ...FileItem) (err error) { - for _, dir := range dirs { - if err = fileutils.CreateDirIfNotExist(dir.toPath(rootDir)); err != nil { - return - } - } - return -} - -func TestSearchDestinationPath(t *testing.T) { - testDir := FileItem{artifactory, "test_plugin_install_dir", "test"} - confuse := FileItem{artifactory, "test_plugin_install_dir", "test2"} - manager := &PluginInstallManager{} - temp, clean := tests.CreateTempDirWithCallbackAndAssert(t) - defer clean() - assert.NoError(t, populateDirWith(temp, confuse)) - // No destinations - exists, target, err := manager.findDestination(temp) - assert.NoError(t, err) - assert.False(t, exists, fmt.Sprintf("The match is %s", target)) - // Destination not exists - manager.addDestination(testDir[1:]) - exists, target, err = manager.findDestination(temp) - assert.NoError(t, err) - assert.False(t, exists, fmt.Sprintf("The match is %s", target)) - // Destination exists - assert.NoError(t, populateDirWith(temp, testDir)) - exists, dst, err := manager.findDestination(temp) - assert.NoError(t, err) - assert.True(t, exists) - assert.Equal(t, testDir.toPath(temp), dst.toPath()) -} - -func TestGetPluginDirDestination(t *testing.T) { - // Init mock and test env - testEnvDir := "testEnv" - testCustomDir := "testCustom" - targetDir := "plugins_test_target" - testHomePath, clean := tests.CreateTempDirWithCallbackAndAssert(t) - defer clean() - if oldVal, exists := os.LookupEnv(jfrogHomeEnvVar); exists { - testsutils.UnSetEnvAndAssert(t, jfrogHomeEnvVar) - defer testsutils.SetEnvAndAssert(t, jfrogHomeEnvVar, oldVal) - } - assert.NoError(t, populateDirWith(testHomePath, FileItem{testEnvDir, artifactory + "-confuse", targetDir}, FileItem{testCustomDir, "confuse-" + artifactory, targetDir})) - manager := NewArtifactoryPluginInstallManager(nil) - manager.addDestination(FileItem{targetDir}) - cmd := &InstallDataTransferPluginCommand{transferManger: manager} - defaultExists, err := fileutils.IsDirExists(defaultSearchPath, false) - assert.NoError(t, err) - - // Make sure contains artifactory structures as destinations - assert.Contains(t, manager.destinations, originalDirPath) - assert.Contains(t, manager.destinations, v7DirPath) - - // Default - dst, err := cmd.getPluginDirDestination() - if defaultExists { - assert.NoError(t, err) - assert.True(t, dst.toPath() == originalDirPath.toPath(defaultSearchPath) || (dst.toPath() == v7DirPath.toPath(defaultSearchPath))) - } else { - assert.Errorf(t, err, errNotValidDestination.Error()) - } - - // Env var override - testsutils.SetEnvAndAssert(t, jfrogHomeEnvVar, filepath.Join(testHomePath, testEnvDir)) - dst, err = cmd.getPluginDirDestination() - assert.NoError(t, err) - assert.Equal(t, filepath.Join(testHomePath, testEnvDir, artifactory+"-confuse", targetDir), dst.toPath()) - - // Flag override - cmd.SetJFrogHomePath(filepath.Join(testHomePath, testCustomDir)) - dst, err = cmd.getPluginDirDestination() - assert.NoError(t, err) - assert.Equal(t, filepath.Join(testHomePath, testCustomDir, "confuse-"+artifactory, targetDir), dst.toPath()) - cmd.SetJFrogHomePath("not_existing_dir") - _, err = cmd.getPluginDirDestination() - assert.Errorf(t, err, errNotValidDestination.Error()) -} - -func TestGetTransferSourceAndAction(t *testing.T) { - // baseUrl := "baseurl" - v1 := "1.0.0" - cmd := &InstallDataTransferPluginCommand{} - - // Latest - src, action, err := cmd.getTransferSourceAndAction() - assert.NoError(t, err) - vUrl, err := url.Parse(dataTransferUrl) - assert.NoError(t, err) - vUrl.Path = path.Join(vUrl.Path, latest) - assert.Equal(t, vUrl.String(), src) - assert.Contains(t, runtime.FuncForPC(reflect.ValueOf(action).Pointer()).Name(), "transferinstall.DownloadFiles") - - // Specific version - cmd.SetInstallVersion(version.NewVersion(v1)) - src, action, err = cmd.getTransferSourceAndAction() - assert.NoError(t, err) - vUrl, err = url.Parse(dataTransferUrl) - assert.NoError(t, err) - vUrl.Path = path.Join(vUrl.Path, v1) - assert.Equal(t, vUrl.String(), src) - assert.True(t, strings.Contains(runtime.FuncForPC(reflect.ValueOf(action).Pointer()).Name(), "transferinstall.DownloadFiles")) - - // Local file system - cmd.SetLocalPluginFiles(dataTransferUrl) - src, action, err = cmd.getTransferSourceAndAction() - assert.NoError(t, err) - assert.Equal(t, dataTransferUrl, src) - assert.True(t, strings.Contains(runtime.FuncForPC(reflect.ValueOf(action).Pointer()).Name(), "transferinstall.CopyFiles")) -} - -func TestInstallCopy(t *testing.T) { - fileBundle := PluginFiles{ - FileItem{"file"}, - FileItem{"dir", "file1"}, - FileItem{"dir1", "dir2", "file2"}, - } - srcPath, cleanUpSrc := tests.CreateTempDirWithCallbackAndAssert(t) - defer cleanUpSrc() - tempDst, cleanTempDst := tests.CreateTempDirWithCallbackAndAssert(t) - defer cleanTempDst() - dstPath, cleanUpDst := tests.CreateTempDirWithCallbackAndAssert(t) - defer cleanUpDst() - - // Empty - assert.NoError(t, CopyFiles(srcPath, tempDst, PluginFiles{})) - // No src files in dir - assert.Error(t, CopyFiles(srcPath, tempDst, fileBundle)) - // Generate empty files in dir (and another to confuse) - for _, file := range fileBundle { - assert.NoError(t, os.WriteFile(filepath.Join(srcPath, file.Name()), nil, 0644)) - assert.NoError(t, os.WriteFile(filepath.Join(srcPath, "not_"+file.Name()), nil, 0644)) - } - // First time in plugin dir (i.e. need to create folders) - assert.NoError(t, CopyFiles(srcPath, dstPath, fileBundle)) - for _, file := range fileBundle { - assert.FileExists(t, file.toPath(dstPath)) - } - // Dir already has plugin (i.e. need to override items, no error) - assert.NoError(t, CopyFiles(srcPath, dstPath, fileBundle)) -} - -func TestDownloadTransferInstall(t *testing.T) { - testDstPath, clean := tests.CreateTempDirWithCallbackAndAssert(t) - defer clean() - // Download latest and make sure exists at the end - assert.NoError(t, DownloadFiles(dataTransferUrl+"/"+latest, testDstPath, transferPluginFiles)) - for _, file := range transferPluginFiles { - assert.FileExists(t, file.toPath(testDstPath)) - } -} - -func TestReloadPlugins(t *testing.T) { - testServer, details, _ := commonTests.CreateRtRestsMockServer(t, func(w http.ResponseWriter, r *http.Request) { - if r.RequestURI == "/"+pluginReloadRestApi { - w.WriteHeader(http.StatusOK) - assert.Equal(t, http.MethodPost, r.Method) - } else { - w.WriteHeader(http.StatusBadRequest) - } - }) - defer testServer.Close() - installCmd := NewInstallDataTransferCommand(details) - assert.NoError(t, installCmd.sendReloadRequest()) -} diff --git a/artifactory/utils/storageinfo.go b/artifactory/utils/storageinfo.go index 87321f8b2..7816147c9 100644 --- a/artifactory/utils/storageinfo.go +++ b/artifactory/utils/storageinfo.go @@ -3,6 +3,7 @@ package utils import ( "context" "errors" + "net/http" "strconv" "strings" "time" @@ -30,7 +31,11 @@ type StorageInfoManager struct { } func NewStorageInfoManager(ctx context.Context, serverDetails *config.ServerDetails) (*StorageInfoManager, error) { - serviceManager, err := CreateServiceManagerWithContext(ctx, serverDetails, false, 0, serviceManagerRetriesPerRequest, serviceManagerRetriesWaitPerRequestMilliSecs, time.Minute) + return NewStorageInfoManagerWithHttpClient(ctx, serverDetails, nil) +} + +func NewStorageInfoManagerWithHttpClient(ctx context.Context, serverDetails *config.ServerDetails, httpClient *http.Client) (*StorageInfoManager, error) { + serviceManager, err := CreateServiceManagerWithContextAndHttpClient(ctx, serverDetails, false, 0, serviceManagerRetriesPerRequest, serviceManagerRetriesWaitPerRequestMilliSecs, time.Minute, httpClient) if err != nil { return nil, err } diff --git a/artifactory/utils/storageinfo_test.go b/artifactory/utils/storageinfo_test.go index cc36dbcf0..8d0e4a1ea 100644 --- a/artifactory/utils/storageinfo_test.go +++ b/artifactory/utils/storageinfo_test.go @@ -3,16 +3,44 @@ package utils import ( "context" "encoding/json" + "io" "net/http" "net/http/httptest" + "strings" "testing" "time" "github.com/jfrog/jfrog-cli-core/v2/utils/config" clientUtils "github.com/jfrog/jfrog-client-go/artifactory/services/utils" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (fn roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { + return fn(req) +} + +func TestNewStorageInfoManagerWithHttpClient(t *testing.T) { + var requestedURL string + httpClient := &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { + requestedURL = req.URL.String() + return &http.Response{ + StatusCode: http.StatusAccepted, + Status: "202 Accepted", + Header: make(http.Header), + Body: io.NopCloser(strings.NewReader("")), + Request: req, + }, nil + })} + manager, err := NewStorageInfoManagerWithHttpClient(context.Background(), + &config.ServerDetails{ArtifactoryUrl: "https://target.example/artifactory/"}, httpClient) + require.NoError(t, err) + require.NoError(t, manager.CalculateStorageInfo()) + assert.Equal(t, "https://target.example/artifactory/api/storageinfo/calculate", requestedURL) +} + func TestCalculateStorageInfo(t *testing.T) { calculated := false // Prepare mock server diff --git a/artifactory/utils/utils.go b/artifactory/utils/utils.go index 88a970c6f..78b10ca94 100644 --- a/artifactory/utils/utils.go +++ b/artifactory/utils/utils.go @@ -106,6 +106,15 @@ func CreateServiceManagerWithThreads(serverDetails *config.ServerDetails, isDryR } func CreateServiceManagerWithContext(context context.Context, serverDetails *config.ServerDetails, isDryRun bool, threads, httpRetries, httpRetryWaitMilliSecs int, timeout time.Duration) (artifactory.ArtifactoryServicesManager, error) { + return CreateServiceManagerWithContextAndHttpClient(context, serverDetails, isDryRun, threads, httpRetries, httpRetryWaitMilliSecs, timeout, nil) +} + +// CreateServiceManagerWithContextAndHttpClient matches CreateServiceManagerWithContext, and optionally +// attaches a custom HTTP client (for example a target-only proxy). A nil httpClient preserves default client-go transport behavior. +// When httpClient is non-nil, client-go's Build() treats it as pass-through: timeout, InsecureTls, the +// certificates path, and client-cert settings configured on this builder have no effect on it. Callers +// that need those applied to a non-nil httpClient must configure them on the http.Client/Transport itself. +func CreateServiceManagerWithContextAndHttpClient(context context.Context, serverDetails *config.ServerDetails, isDryRun bool, threads, httpRetries, httpRetryWaitMilliSecs int, timeout time.Duration, httpClient *http.Client) (artifactory.ArtifactoryServicesManager, error) { certsPath, err := coreutils.GetJfrogCertsDir() if err != nil { return nil, err @@ -120,6 +129,9 @@ func CreateServiceManagerWithContext(context context.Context, serverDetails *con SetInsecureTls(serverDetails.InsecureTls). SetDryRun(isDryRun). SetContext(context) + if httpClient != nil { + configBuilder.SetHttpClient(httpClient) + } if httpRetries >= 0 { configBuilder.SetHttpRetries(httpRetries) configBuilder.SetHttpRetryWaitMilliSecs(httpRetryWaitMilliSecs) diff --git a/utils/reposnapshot/snapshotmanager.go b/utils/reposnapshot/snapshotmanager.go index 99ccd4174..2d8be817b 100644 --- a/utils/reposnapshot/snapshotmanager.go +++ b/utils/reposnapshot/snapshotmanager.go @@ -60,7 +60,7 @@ func newRepoSnapshotManager(root *Node, repoKey, snapshotFilePath string) RepoSn root: root, repoKey: repoKey, snapshotFilePath: snapshotFilePath, - lruCache: lru.New(cacheSize, lru.WithoutSync()), + lruCache: lru.New(cacheSize), } } diff --git a/utils/reposnapshot/snapshotmanager_test.go b/utils/reposnapshot/snapshotmanager_test.go index 454a00925..40aa375a8 100644 --- a/utils/reposnapshot/snapshotmanager_test.go +++ b/utils/reposnapshot/snapshotmanager_test.go @@ -2,15 +2,17 @@ package reposnapshot import ( "encoding/json" - "github.com/jfrog/gofrog/safeconvert" "os" "path" "path/filepath" + "sync" "testing" + "github.com/jfrog/gofrog/safeconvert" clientutils "github.com/jfrog/jfrog-client-go/utils" "github.com/jfrog/jfrog-client-go/utils/io/fileutils" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) const dummyRepoKey = "dummy-repo-local" @@ -252,3 +254,32 @@ func getNodeAndAssert(t *testing.T, manager RepoSnapshotManager, relativePath st assertReturnedNode(t, manager, node, path.Dir(relativePath), expectedLen) return node } + +// GET/PUT transfer-files marks snapshot completion from every uploader goroutine. +// The LRU must be safe for concurrent Get+Add of overlapping directory paths. +func TestGetDirectorySnapshotNodeWithLruConcurrent(t *testing.T) { + manager := initSnapshotManagerTest(t) + paths := []string{"1/b/", "./", "2/", "0/a/", "1/a/"} + const goroutines = 32 + const iterations = 200 + + errCh := make(chan error, goroutines) + var wg sync.WaitGroup + wg.Add(goroutines) + for i := 0; i < goroutines; i++ { + go func() { + defer wg.Done() + for n := 0; n < iterations; n++ { + if _, err := manager.GetDirectorySnapshotNodeWithLru(paths[n%len(paths)]); err != nil { + errCh <- err + return + } + } + }() + } + wg.Wait() + close(errCh) + for err := range errCh { + require.NoError(t, err) + } +}