diff --git a/pkg/fileservice/object_storage_arguments_test.go b/pkg/fileservice/object_storage_arguments_test.go index f526e5eb419b1..6fc956064a474 100644 --- a/pkg/fileservice/object_storage_arguments_test.go +++ b/pkg/fileservice/object_storage_arguments_test.go @@ -29,7 +29,6 @@ import ( "testing" "time" - "github.com/matrixorigin/matrixone/pkg/logutil" "github.com/stretchr/testify/assert" "go.uber.org/zap" "go.uber.org/zap/zapcore" @@ -68,14 +67,31 @@ func TestObjectStorageArguments(t *testing.T) { }) } -func objectStorageArgumentsForTest(defaultName string, t *testing.T) (ret []ObjectStorageArguments) { +const runExternalObjectStorageTestsEnv = "MO_RUN_EXTERNAL_OBJECT_STORAGE_TESTS" - // disk - ret = append(ret, ObjectStorageArguments{ +func localObjectStorageArgumentsForTest(defaultName string, t *testing.T) []ObjectStorageArguments { + return []ObjectStorageArguments{{ Name: defaultName, Endpoint: "disk", Bucket: t.TempDir(), - }) + }} +} + +func shouldRunExternalObjectStorageTests(short bool, enabled string) bool { + return !short && enabled == "1" +} + +func requireExternalObjectStorageTests(t *testing.T) { + t.Helper() + if shouldRunExternalObjectStorageTests(testing.Short(), os.Getenv(runExternalObjectStorageTestsEnv)) { + return + } + t.Skipf("external object storage tests require -short=false and %s=1", runExternalObjectStorageTestsEnv) +} + +func externalObjectStorageArgumentsForTest(defaultName string, t *testing.T) (ret []ObjectStorageArguments) { + t.Helper() + requireExternalObjectStorageTests(t) // s3.json content, err := os.ReadFile("s3.json") @@ -90,19 +106,22 @@ func objectStorageArgumentsForTest(defaultName string, t *testing.T) (ret []Obje RoleARN string `json:"role-arn"` } - if err := json.Unmarshal(content, &config); err == nil { - ret = append(ret, ObjectStorageArguments{ - Name: "s3.json " + defaultName, - Endpoint: config.Endpoint, - Region: config.Region, - KeyID: config.APIKey, - KeySecret: config.APISecret, - Bucket: config.Bucket, - RoleARN: config.RoleARN, - KeyPrefix: fmt.Sprintf("%v", rand.Int64()), - }) + if err := json.Unmarshal(content, &config); err != nil { + t.Fatalf("parse s3.json: %v", err) } + ret = append(ret, ObjectStorageArguments{ + Name: "s3.json " + defaultName, + Endpoint: config.Endpoint, + Region: config.Region, + KeyID: config.APIKey, + KeySecret: config.APISecret, + Bucket: config.Bucket, + RoleARN: config.RoleARN, + KeyPrefix: fmt.Sprintf("%v", rand.Int64()), + }) } + } else if !os.IsNotExist(err) { + t.Fatalf("read s3.json: %v", err) } // s3_fs_test_new.xml @@ -112,15 +131,18 @@ func objectStorageArgumentsForTest(defaultName string, t *testing.T) (ret []Obje XMLName xml.Name `xml:"Spec"` Cases []S3CredentialTestCase `xml:"Case"` } - if err := xml.Unmarshal(content, &spec); err == nil { - for _, kase := range spec.Cases { - if kase.Skip { - continue - } - kase.KeyPrefix = fmt.Sprintf("%v", rand.Int64()) - ret = append(ret, kase.ObjectStorageArguments) + if err := xml.Unmarshal(content, &spec); err != nil { + t.Fatalf("parse s3_fs_test_new.xml: %v", err) + } + for _, kase := range spec.Cases { + if kase.Skip { + continue } + kase.KeyPrefix = fmt.Sprintf("%v", rand.Int64()) + ret = append(ret, kase.ObjectStorageArguments) } + } else if !os.IsNotExist(err) { + t.Fatalf("read s3_fs_test_new.xml: %v", err) } // envs @@ -141,22 +163,52 @@ func objectStorageArgumentsForTest(defaultName string, t *testing.T) (ret []Obje reader := csv.NewReader(strings.NewReader(value)) argStrs, err := reader.Read() if err != nil { - logutil.Warn("bad S3FS test spec", zap.Any("spec", value)) - continue + t.Fatalf("parse %s: %v", name, err) } var args ObjectStorageArguments if err := args.SetFromString(argStrs); err != nil { - logutil.Warn("bad S3FS test spec", zap.Any("spec", value)) - continue + t.Fatalf("parse %s: %v", name, err) } args.KeyPrefix = fmt.Sprintf("%v", rand.Int64()) ret = append(ret, args) } + if len(ret) == 0 { + t.Fatalf("%s=1 but no external object storage test configuration was found", runExternalObjectStorageTestsEnv) + } return ret } +func TestShouldRunExternalObjectStorageTests(t *testing.T) { + testCases := []struct { + name string + short bool + enabled string + want bool + }{ + {name: "explicitly enabled", enabled: "1", want: true}, + {name: "disabled by default"}, + {name: "short mode wins", short: true, enabled: "1"}, + {name: "invalid value", enabled: "true"}, + } + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + assert.Equal(t, testCase.want, shouldRunExternalObjectStorageTests(testCase.short, testCase.enabled)) + }) + } +} + +func TestLocalObjectStorageArgumentsIgnoreExternalConfiguration(t *testing.T) { + t.Setenv("TEST_S3FS_ALIYUN", "name=aliyun,endpoint=https://127.0.0.1:1,bucket=test,key-id=id,key-secret=secret") + args := localObjectStorageArgumentsForTest("test", t) + if !assert.Len(t, args, 1) { + return + } + assert.Equal(t, "test", args[0].Name) + assert.Equal(t, "disk", args[0].Endpoint) +} + func TestQCloudRegion(t *testing.T) { args := ObjectStorageArguments{ Endpoint: "http://cos.foobar.myqcloud.com", diff --git a/pkg/fileservice/object_storage_test.go b/pkg/fileservice/object_storage_test.go index f0df570162558..01253e1f401f7 100644 --- a/pkg/fileservice/object_storage_test.go +++ b/pkg/fileservice/object_storage_test.go @@ -224,7 +224,16 @@ func testObjectStorageWithParentContext[T ObjectStorage]( } func TestObjectStorages(t *testing.T) { - for _, args := range objectStorageArgumentsForTest("test", t) { + testObjectStorages(t, localObjectStorageArgumentsForTest("test", t)) +} + +func TestObjectStoragesExternal(t *testing.T) { + testObjectStorages(t, externalObjectStorageArgumentsForTest("test", t)) +} + +func testObjectStorages(t *testing.T, testArguments []ObjectStorageArguments) { + t.Helper() + for _, args := range testArguments { t.Run(args.Name, func(t *testing.T) { specCtx, cancel := context.WithTimeout(t.Context(), 3*time.Minute) diff --git a/pkg/fileservice/qcloud_sdk_test.go b/pkg/fileservice/qcloud_sdk_test.go index f650d5576fd81..fd37a6e171f9b 100644 --- a/pkg/fileservice/qcloud_sdk_test.go +++ b/pkg/fileservice/qcloud_sdk_test.go @@ -35,9 +35,10 @@ import ( ) func TestQCloudSDK(t *testing.T) { + testArguments := externalObjectStorageArgumentsForTest("test", t) t.Run("object storage", func(t *testing.T) { - for _, args := range objectStorageArgumentsForTest("test", t) { + for _, args := range testArguments { if !strings.Contains(args.Endpoint, "myqcloud") { continue } @@ -62,7 +63,7 @@ func TestQCloudSDK(t *testing.T) { }) t.Run("file service", func(t *testing.T) { - for _, args := range objectStorageArgumentsForTest("test", t) { + for _, args := range testArguments { if !strings.Contains(args.Endpoint, "myqcloud") { continue } diff --git a/pkg/fileservice/s3_fs_test.go b/pkg/fileservice/s3_fs_test.go index 4d03371ee43a3..7bd54302ed17c 100644 --- a/pkg/fileservice/s3_fs_test.go +++ b/pkg/fileservice/s3_fs_test.go @@ -2636,13 +2636,14 @@ type S3CredentialTestCase struct { ObjectStorageArguments } -var s3CredentialTestCases = func() []S3CredentialTestCase { +func s3CredentialTestCasesForTest(t *testing.T) []S3CredentialTestCase { + t.Helper() content, err := os.ReadFile("s3_fs_test_new.xml") if os.IsNotExist(err) { return nil } if err != nil { - panic(err) + t.Fatal(err) } var spec struct { XMLName xml.Name `xml:"Spec"` @@ -2650,12 +2651,14 @@ var s3CredentialTestCases = func() []S3CredentialTestCase { } err = xml.Unmarshal(content, &spec) if err != nil { - panic(err) + t.Fatal(err) } return spec.Cases -}() +} func TestNewS3FSFromSpec(t *testing.T) { + requireExternalObjectStorageTests(t) + s3CredentialTestCases := s3CredentialTestCasesForTest(t) if len(s3CredentialTestCases) == 0 { t.Skip("no case") } @@ -2930,8 +2933,16 @@ func BenchmarkS3FSAllocateCacheDataHighCardinality(b *testing.B) { } func TestS3FSFromSpecs(t *testing.T) { + testS3FSFromSpecs(t, localObjectStorageArgumentsForTest("test", t)) +} + +func TestS3FSFromExternalSpecs(t *testing.T) { + testS3FSFromSpecs(t, externalObjectStorageArgumentsForTest("test", t)) +} - for _, args := range objectStorageArgumentsForTest("test", t) { +func testS3FSFromSpecs(t *testing.T, testArguments []ObjectStorageArguments) { + t.Helper() + for _, args := range testArguments { t.Run(args.Name, func(t *testing.T) {