esti: test lakectl fs upload of files over 5GiB

esti/golden/lakectl_fs_upload_large.golden

@@ -0,0 +1,7 @@
+Path: ${FILE_PATH}
+Modified Time: <DATE> <TIME> <TZ>
+Size: 5368709121 bytes
+Human Size: 5.4 GB
+Physical Address: <OBJECT_KEY>
+Checksum: <CHECKSUM>
+Content-Type: application/octet-stream

esti/lakectl_test.go

@@ -162,6 +162,39 @@ func TestLakectlPreSignUpload(t *testing.T) {
 	})
 }
 
+func TestLakectlFsUploadLarge(t *testing.T) {
+	RequireBlockstoreType(t, block.BlockstoreTypeS3)
+
+	repoName := GenerateUniqueRepositoryName()
+	storage := GenerateUniqueStorageNamespace(repoName)
+	vars := map[string]string{
+		"REPO":    repoName,
+		"STORAGE": storage,
+		"BRANCH":  mainBranch,
+	}
+	RunCmdAndVerifySuccessWithFile(t, Lakectl()+" repo create lakefs://"+repoName+" "+storage, false, "lakectl_repo_create", vars)
+
+	// S3 rejects single-part uploads above 5GiB, so one byte more forces
+	// the multipart path (#9284). A sparse file keeps disk usage low.
+	const largeFileSize = 5<<30 + 1
+	f, err := os.CreateTemp("", "lakectl-large-upload-")
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		require.NoError(t, os.Remove(f.Name()))
+	})
+	require.NoError(t, f.Truncate(largeFileSize))
+	require.NoError(t, f.Close())
+
+	t.Run("pre-sign", func(t *testing.T) {
+		vars["FILE_PATH"] = "large_file_presign"
+		RunCmdAndVerifySuccessWithFile(t, Lakectl()+" fs upload -s "+f.Name()+" lakefs://"+repoName+"/"+mainBranch+"/"+vars["FILE_PATH"]+" --pre-sign", false, "lakectl_fs_upload_large", vars)
+	})
+	t.Run("no pre-sign", func(t *testing.T) {
+		vars["FILE_PATH"] = "large_file"
+		RunCmdAndVerifySuccessWithFile(t, Lakectl()+" fs upload -s "+f.Name()+" lakefs://"+repoName+"/"+mainBranch+"/"+vars["FILE_PATH"], false, "lakectl_fs_upload_large", vars)
+	})
+}
+
 func TestLakectlCommit(t *testing.T) {
 	repoName := GenerateUniqueRepositoryName()
 	storage := GenerateUniqueStorageNamespace(repoName)

esti/lakectl_util.go

@@ -28,13 +28,17 @@ var (
 )
 
 var (
-	reTimestamp       = regexp.MustCompile(`timestamp: \d+`)
-	reTime            = regexp.MustCompile(`\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} [-+]\d{4} \w{1,4}`)
-	reCommitID        = regexp.MustCompile(`[\d|a-f]{64}`)
-	reShortCommitID   = regexp.MustCompile(`[\d|a-f]{16}`)
-	reChecksum        = regexp.MustCompile(`([\d|a-f]{32})|(0x[0-9A-F]{15})`)
-	reEndpoint        = regexp.MustCompile(`https?://\w+(:\d+)?/api/v\d+/`)
-	rePhysicalAddress = regexp.MustCompile(`/data/[0-9a-v]{20}/(?:[0-9a-v]{20}(?:,.+)?)?`)
+	reTimestamp     = regexp.MustCompile(`timestamp: \d+`)
+	reTime          = regexp.MustCompile(`\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} [-+]\d{4} \w{1,4}`)
+	reCommitID      = regexp.MustCompile(`[\d|a-f]{64}`)
+	reShortCommitID = regexp.MustCompile(`[\d|a-f]{16}`)
+	// multipart uploads report an etag with a part count suffix, e.g. <md5>-1025
+	reChecksum = regexp.MustCompile(`([\d|a-f]{32}(-\d+)?)|(0x[0-9A-F]{15})`)
+	reEndpoint = regexp.MustCompile(`https?://\w+(:\d+)?/api/v\d+/`)
+	// physical addresses may be printed relative to the storage namespace (no
+	// leading slash), and multipart uploads append a part-count suffix, e.g.
+	// data/<prefix>/<key>,<suffix>
+	rePhysicalAddress = regexp.MustCompile(`/?data/[0-9a-v]{20}/(?:[0-9a-v]{20}(?:,.+)?)?`)
 	reVariable        = regexp.MustCompile(`\$\{([^${}]+)}`)
 	rePreSignURL      = regexp.MustCompile(`https?://\S+\?\S+`)
 	reSecretAccessKey = regexp.MustCompile(`secret_access_key: \S{16,128}`)
@@ -260,7 +264,12 @@ func normalizeProgramTimestamp(output string) string {
 func normalizeRandomObjectKey(output string, objectPrefix string) string {
 	objectPrefix = strings.TrimPrefix(objectPrefix, "/")
 	for _, match := range rePhysicalAddress.FindAllString(output, -1) {
-		output = strings.Replace(output, objectPrefix+match, objectPrefix+"/<OBJECT_KEY>", 1)
+		if strings.Contains(output, objectPrefix+match) {
+			output = strings.Replace(output, objectPrefix+match, objectPrefix+"/<OBJECT_KEY>", 1)
+		} else {
+			// relative physical address, not prefixed by the storage namespace
+			output = strings.Replace(output, match, "<OBJECT_KEY>", 1)
+		}
 	}
 	return output
 }