diff --git a/alioss/client/client.go b/alioss/client/client.go index 712a42a..644db96 100644 --- a/alioss/client/client.go +++ b/alioss/client/client.go @@ -3,7 +3,6 @@ package client import ( "crypto/md5" "encoding/base64" - "errors" "fmt" "io" "log" @@ -80,21 +79,21 @@ func (client *AliBlobstore) getMD5(filePath string) (string, error) { } func (client *AliBlobstore) List(prefix string) ([]string, error) { - return nil, errors.New("not implemented") + return client.storageClient.List(prefix) } func (client *AliBlobstore) Copy(srcBlob string, dstBlob string) error { - return errors.New("not implemented") + return client.storageClient.Copy(srcBlob, dstBlob) } func (client *AliBlobstore) Properties(dest string) error { - return errors.New("not implemented") + return client.storageClient.Properties(dest) } func (client *AliBlobstore) EnsureStorageExists() error { - return errors.New("not implemented") + return client.storageClient.EnsureBucketExists() } func (client *AliBlobstore) DeleteRecursive(prefix string) error { - return errors.New("not implemented") + return client.storageClient.DeleteRecursive(prefix) } diff --git a/alioss/client/clientfakes/fake_storage_client.go b/alioss/client/clientfakes/fake_storage_client.go index b963907..2630565 100644 --- a/alioss/client/clientfakes/fake_storage_client.go +++ b/alioss/client/clientfakes/fake_storage_client.go @@ -8,6 +8,18 @@ import ( ) type FakeStorageClient struct { + CopyStub func(string, string) error + copyMutex sync.RWMutex + copyArgsForCall []struct { + arg1 string + arg2 string + } + copyReturns struct { + result1 error + } + copyReturnsOnCall map[int]struct { + result1 error + } DeleteStub func(string) error deleteMutex sync.RWMutex deleteArgsForCall []struct { @@ -19,6 +31,17 @@ type FakeStorageClient struct { deleteReturnsOnCall map[int]struct { result1 error } + DeleteRecursiveStub func(string) error + deleteRecursiveMutex sync.RWMutex + deleteRecursiveArgsForCall []struct { + arg1 string + } + deleteRecursiveReturns struct { + result1 error + } + deleteRecursiveReturnsOnCall map[int]struct { + result1 error + } DownloadStub func(string, string) error downloadMutex sync.RWMutex downloadArgsForCall []struct { @@ -31,6 +54,16 @@ type FakeStorageClient struct { downloadReturnsOnCall map[int]struct { result1 error } + EnsureBucketExistsStub func() error + ensureBucketExistsMutex sync.RWMutex + ensureBucketExistsArgsForCall []struct { + } + ensureBucketExistsReturns struct { + result1 error + } + ensureBucketExistsReturnsOnCall map[int]struct { + result1 error + } ExistsStub func(string) (bool, error) existsMutex sync.RWMutex existsArgsForCall []struct { @@ -44,6 +77,30 @@ type FakeStorageClient struct { result1 bool result2 error } + ListStub func(string) ([]string, error) + listMutex sync.RWMutex + listArgsForCall []struct { + arg1 string + } + listReturns struct { + result1 []string + result2 error + } + listReturnsOnCall map[int]struct { + result1 []string + result2 error + } + PropertiesStub func(string) error + propertiesMutex sync.RWMutex + propertiesArgsForCall []struct { + arg1 string + } + propertiesReturns struct { + result1 error + } + propertiesReturnsOnCall map[int]struct { + result1 error + } SignedUrlGetStub func(string, int64) (string, error) signedUrlGetMutex sync.RWMutex signedUrlGetArgsForCall []struct { @@ -89,6 +146,68 @@ type FakeStorageClient struct { invocationsMutex sync.RWMutex } +func (fake *FakeStorageClient) Copy(arg1 string, arg2 string) error { + fake.copyMutex.Lock() + ret, specificReturn := fake.copyReturnsOnCall[len(fake.copyArgsForCall)] + fake.copyArgsForCall = append(fake.copyArgsForCall, struct { + arg1 string + arg2 string + }{arg1, arg2}) + stub := fake.CopyStub + fakeReturns := fake.copyReturns + fake.recordInvocation("Copy", []interface{}{arg1, arg2}) + fake.copyMutex.Unlock() + if stub != nil { + return stub(arg1, arg2) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeStorageClient) CopyCallCount() int { + fake.copyMutex.RLock() + defer fake.copyMutex.RUnlock() + return len(fake.copyArgsForCall) +} + +func (fake *FakeStorageClient) CopyCalls(stub func(string, string) error) { + fake.copyMutex.Lock() + defer fake.copyMutex.Unlock() + fake.CopyStub = stub +} + +func (fake *FakeStorageClient) CopyArgsForCall(i int) (string, string) { + fake.copyMutex.RLock() + defer fake.copyMutex.RUnlock() + argsForCall := fake.copyArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeStorageClient) CopyReturns(result1 error) { + fake.copyMutex.Lock() + defer fake.copyMutex.Unlock() + fake.CopyStub = nil + fake.copyReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeStorageClient) CopyReturnsOnCall(i int, result1 error) { + fake.copyMutex.Lock() + defer fake.copyMutex.Unlock() + fake.CopyStub = nil + if fake.copyReturnsOnCall == nil { + fake.copyReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.copyReturnsOnCall[i] = struct { + result1 error + }{result1} +} + func (fake *FakeStorageClient) Delete(arg1 string) error { fake.deleteMutex.Lock() ret, specificReturn := fake.deleteReturnsOnCall[len(fake.deleteArgsForCall)] @@ -150,6 +269,67 @@ func (fake *FakeStorageClient) DeleteReturnsOnCall(i int, result1 error) { }{result1} } +func (fake *FakeStorageClient) DeleteRecursive(arg1 string) error { + fake.deleteRecursiveMutex.Lock() + ret, specificReturn := fake.deleteRecursiveReturnsOnCall[len(fake.deleteRecursiveArgsForCall)] + fake.deleteRecursiveArgsForCall = append(fake.deleteRecursiveArgsForCall, struct { + arg1 string + }{arg1}) + stub := fake.DeleteRecursiveStub + fakeReturns := fake.deleteRecursiveReturns + fake.recordInvocation("DeleteRecursive", []interface{}{arg1}) + fake.deleteRecursiveMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeStorageClient) DeleteRecursiveCallCount() int { + fake.deleteRecursiveMutex.RLock() + defer fake.deleteRecursiveMutex.RUnlock() + return len(fake.deleteRecursiveArgsForCall) +} + +func (fake *FakeStorageClient) DeleteRecursiveCalls(stub func(string) error) { + fake.deleteRecursiveMutex.Lock() + defer fake.deleteRecursiveMutex.Unlock() + fake.DeleteRecursiveStub = stub +} + +func (fake *FakeStorageClient) DeleteRecursiveArgsForCall(i int) string { + fake.deleteRecursiveMutex.RLock() + defer fake.deleteRecursiveMutex.RUnlock() + argsForCall := fake.deleteRecursiveArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeStorageClient) DeleteRecursiveReturns(result1 error) { + fake.deleteRecursiveMutex.Lock() + defer fake.deleteRecursiveMutex.Unlock() + fake.DeleteRecursiveStub = nil + fake.deleteRecursiveReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeStorageClient) DeleteRecursiveReturnsOnCall(i int, result1 error) { + fake.deleteRecursiveMutex.Lock() + defer fake.deleteRecursiveMutex.Unlock() + fake.DeleteRecursiveStub = nil + if fake.deleteRecursiveReturnsOnCall == nil { + fake.deleteRecursiveReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.deleteRecursiveReturnsOnCall[i] = struct { + result1 error + }{result1} +} + func (fake *FakeStorageClient) Download(arg1 string, arg2 string) error { fake.downloadMutex.Lock() ret, specificReturn := fake.downloadReturnsOnCall[len(fake.downloadArgsForCall)] @@ -212,6 +392,59 @@ func (fake *FakeStorageClient) DownloadReturnsOnCall(i int, result1 error) { }{result1} } +func (fake *FakeStorageClient) EnsureBucketExists() error { + fake.ensureBucketExistsMutex.Lock() + ret, specificReturn := fake.ensureBucketExistsReturnsOnCall[len(fake.ensureBucketExistsArgsForCall)] + fake.ensureBucketExistsArgsForCall = append(fake.ensureBucketExistsArgsForCall, struct { + }{}) + stub := fake.EnsureBucketExistsStub + fakeReturns := fake.ensureBucketExistsReturns + fake.recordInvocation("EnsureBucketExists", []interface{}{}) + fake.ensureBucketExistsMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeStorageClient) EnsureBucketExistsCallCount() int { + fake.ensureBucketExistsMutex.RLock() + defer fake.ensureBucketExistsMutex.RUnlock() + return len(fake.ensureBucketExistsArgsForCall) +} + +func (fake *FakeStorageClient) EnsureBucketExistsCalls(stub func() error) { + fake.ensureBucketExistsMutex.Lock() + defer fake.ensureBucketExistsMutex.Unlock() + fake.EnsureBucketExistsStub = stub +} + +func (fake *FakeStorageClient) EnsureBucketExistsReturns(result1 error) { + fake.ensureBucketExistsMutex.Lock() + defer fake.ensureBucketExistsMutex.Unlock() + fake.EnsureBucketExistsStub = nil + fake.ensureBucketExistsReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeStorageClient) EnsureBucketExistsReturnsOnCall(i int, result1 error) { + fake.ensureBucketExistsMutex.Lock() + defer fake.ensureBucketExistsMutex.Unlock() + fake.EnsureBucketExistsStub = nil + if fake.ensureBucketExistsReturnsOnCall == nil { + fake.ensureBucketExistsReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.ensureBucketExistsReturnsOnCall[i] = struct { + result1 error + }{result1} +} + func (fake *FakeStorageClient) Exists(arg1 string) (bool, error) { fake.existsMutex.Lock() ret, specificReturn := fake.existsReturnsOnCall[len(fake.existsArgsForCall)] @@ -276,6 +509,131 @@ func (fake *FakeStorageClient) ExistsReturnsOnCall(i int, result1 bool, result2 }{result1, result2} } +func (fake *FakeStorageClient) List(arg1 string) ([]string, error) { + fake.listMutex.Lock() + ret, specificReturn := fake.listReturnsOnCall[len(fake.listArgsForCall)] + fake.listArgsForCall = append(fake.listArgsForCall, struct { + arg1 string + }{arg1}) + stub := fake.ListStub + fakeReturns := fake.listReturns + fake.recordInvocation("List", []interface{}{arg1}) + fake.listMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeStorageClient) ListCallCount() int { + fake.listMutex.RLock() + defer fake.listMutex.RUnlock() + return len(fake.listArgsForCall) +} + +func (fake *FakeStorageClient) ListCalls(stub func(string) ([]string, error)) { + fake.listMutex.Lock() + defer fake.listMutex.Unlock() + fake.ListStub = stub +} + +func (fake *FakeStorageClient) ListArgsForCall(i int) string { + fake.listMutex.RLock() + defer fake.listMutex.RUnlock() + argsForCall := fake.listArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeStorageClient) ListReturns(result1 []string, result2 error) { + fake.listMutex.Lock() + defer fake.listMutex.Unlock() + fake.ListStub = nil + fake.listReturns = struct { + result1 []string + result2 error + }{result1, result2} +} + +func (fake *FakeStorageClient) ListReturnsOnCall(i int, result1 []string, result2 error) { + fake.listMutex.Lock() + defer fake.listMutex.Unlock() + fake.ListStub = nil + if fake.listReturnsOnCall == nil { + fake.listReturnsOnCall = make(map[int]struct { + result1 []string + result2 error + }) + } + fake.listReturnsOnCall[i] = struct { + result1 []string + result2 error + }{result1, result2} +} + +func (fake *FakeStorageClient) Properties(arg1 string) error { + fake.propertiesMutex.Lock() + ret, specificReturn := fake.propertiesReturnsOnCall[len(fake.propertiesArgsForCall)] + fake.propertiesArgsForCall = append(fake.propertiesArgsForCall, struct { + arg1 string + }{arg1}) + stub := fake.PropertiesStub + fakeReturns := fake.propertiesReturns + fake.recordInvocation("Properties", []interface{}{arg1}) + fake.propertiesMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeStorageClient) PropertiesCallCount() int { + fake.propertiesMutex.RLock() + defer fake.propertiesMutex.RUnlock() + return len(fake.propertiesArgsForCall) +} + +func (fake *FakeStorageClient) PropertiesCalls(stub func(string) error) { + fake.propertiesMutex.Lock() + defer fake.propertiesMutex.Unlock() + fake.PropertiesStub = stub +} + +func (fake *FakeStorageClient) PropertiesArgsForCall(i int) string { + fake.propertiesMutex.RLock() + defer fake.propertiesMutex.RUnlock() + argsForCall := fake.propertiesArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeStorageClient) PropertiesReturns(result1 error) { + fake.propertiesMutex.Lock() + defer fake.propertiesMutex.Unlock() + fake.PropertiesStub = nil + fake.propertiesReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeStorageClient) PropertiesReturnsOnCall(i int, result1 error) { + fake.propertiesMutex.Lock() + defer fake.propertiesMutex.Unlock() + fake.PropertiesStub = nil + if fake.propertiesReturnsOnCall == nil { + fake.propertiesReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.propertiesReturnsOnCall[i] = struct { + result1 error + }{result1} +} + func (fake *FakeStorageClient) SignedUrlGet(arg1 string, arg2 int64) (string, error) { fake.signedUrlGetMutex.Lock() ret, specificReturn := fake.signedUrlGetReturnsOnCall[len(fake.signedUrlGetArgsForCall)] @@ -472,18 +830,6 @@ func (fake *FakeStorageClient) UploadReturnsOnCall(i int, result1 error) { func (fake *FakeStorageClient) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() - fake.deleteMutex.RLock() - defer fake.deleteMutex.RUnlock() - fake.downloadMutex.RLock() - defer fake.downloadMutex.RUnlock() - fake.existsMutex.RLock() - defer fake.existsMutex.RUnlock() - fake.signedUrlGetMutex.RLock() - defer fake.signedUrlGetMutex.RUnlock() - fake.signedUrlPutMutex.RLock() - defer fake.signedUrlPutMutex.RUnlock() - fake.uploadMutex.RLock() - defer fake.uploadMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/alioss/client/storage_client.go b/alioss/client/storage_client.go index 2813512..0ada623 100644 --- a/alioss/client/storage_client.go +++ b/alioss/client/storage_client.go @@ -1,7 +1,13 @@ package client import ( + "encoding/json" + "errors" + "fmt" "log" + "strconv" + "strings" + "time" "github.com/aliyun/aliyun-oss-go-sdk/oss" "github.com/cloudfoundry/storage-cli/alioss/config" @@ -20,10 +26,19 @@ type StorageClient interface { destinationFilePath string, ) error + Copy( + srcBlob string, + destBlob string, + ) error + Delete( object string, ) error + DeleteRecursive( + objects string, + ) error + Exists( object string, ) (bool, error) @@ -37,14 +52,26 @@ type StorageClient interface { object string, expiredInSec int64, ) (string, error) -} + List( + prefix string, + ) ([]string, error) + + Properties( + object string, + ) error + + EnsureBucketExists() error +} type DefaultStorageClient struct { storageConfig config.AliStorageConfig } func NewStorageClient(storageConfig config.AliStorageConfig) (StorageClient, error) { - return DefaultStorageClient{storageConfig: storageConfig}, nil + + return DefaultStorageClient{ + storageConfig: storageConfig, + }, nil } func (dsc DefaultStorageClient) Upload( @@ -86,6 +113,31 @@ func (dsc DefaultStorageClient) Download( return bucket.GetObjectToFile(sourceObject, destinationFilePath) } +func (dsc DefaultStorageClient) Copy( + sourceObject string, + destinationObject string, +) error { + log.Printf("Copying object from %s to %s", sourceObject, destinationObject) + srcOut := fmt.Sprintf("%s/%s", dsc.storageConfig.BucketName, sourceObject) + destOut := fmt.Sprintf("%s/%s", dsc.storageConfig.BucketName, destinationObject) + + client, err := oss.New(dsc.storageConfig.Endpoint, dsc.storageConfig.AccessKeyID, dsc.storageConfig.AccessKeySecret) + if err != nil { + return err + } + + bucket, err := client.Bucket(dsc.storageConfig.BucketName) + if err != nil { + return err + } + + if _, err := bucket.CopyObject(sourceObject, destinationObject); err != nil { + return fmt.Errorf("failed to copy object from %s to %s: %w", srcOut, destOut, err) + } + + return nil +} + func (dsc DefaultStorageClient) Delete( object string, ) error { @@ -104,6 +156,74 @@ func (dsc DefaultStorageClient) Delete( return bucket.DeleteObject(object) } +func (dsc DefaultStorageClient) DeleteRecursive( + prefix string, +) error { + if prefix != "" { + log.Printf("Deleting all objects in bucket %s with prefix '%s'\n", + dsc.storageConfig.BucketName, prefix) + } else { + log.Printf("Deleting all objects in bucket %s\n", dsc.storageConfig.BucketName) + } + + client, err := oss.New( + dsc.storageConfig.Endpoint, + dsc.storageConfig.AccessKeyID, + dsc.storageConfig.AccessKeySecret, + ) + if err != nil { + return err + } + + bucket, err := client.Bucket(dsc.storageConfig.BucketName) + if err != nil { + return err + } + + var marker string + + for { + opts := []oss.Option{ + oss.MaxKeys(1000), + } + if prefix != "" { + opts = append(opts, oss.Prefix(prefix)) + } + if marker != "" { + opts = append(opts, oss.Marker(marker)) + } + + resp, err := bucket.ListObjects(opts...) + if err != nil { + return fmt.Errorf("error listing objects for delete: %w", err) + } + + if len(resp.Objects) == 0 && !resp.IsTruncated { + return nil + } + + keys := make([]string, 0, len(resp.Objects)) + for _, obj := range resp.Objects { + keys = append(keys, obj.Key) + } + + if len(keys) > 0 { + quiet := true + _, err := bucket.DeleteObjects(keys, oss.DeleteObjectsQuiet(quiet)) + if err != nil { + return fmt.Errorf("failed to batch delete %d objects (prefix=%q): %w", len(keys), prefix, err) + } + } + + if !resp.IsTruncated { + break + } + marker = resp.NextMarker + } + + return nil +} + func (dsc DefaultStorageClient) Exists(object string) (bool, error) { log.Printf("Checking if blob: %s/%s\n", dsc.storageConfig.BucketName, object) @@ -170,3 +290,152 @@ func (dsc DefaultStorageClient) SignedUrlGet( return bucket.SignURL(object, oss.HTTPGet, expiredInSec) } + +func (dsc DefaultStorageClient) List( + prefix string, +) ([]string, error) { + if prefix != "" { + log.Printf("Listing objects in bucket %s with prefix '%s'\n", + dsc.storageConfig.BucketName, prefix) + } else { + log.Printf("Listing objects in bucket %s\n", dsc.storageConfig.BucketName) + } + + var ( + objects []string + marker string + ) + + for { + var opts []oss.Option + if prefix != "" { + opts = append(opts, oss.Prefix(prefix)) + } + if marker != "" { + opts = append(opts, oss.Marker(marker)) + } + + client, err := oss.New(dsc.storageConfig.Endpoint, dsc.storageConfig.AccessKeyID, dsc.storageConfig.AccessKeySecret) + if err != nil { + return nil, err + } + + bucket, err := client.Bucket(dsc.storageConfig.BucketName) + if err != nil { + return nil, err + } + + resp, err := bucket.ListObjects(opts...) + if err != nil { + return nil, fmt.Errorf("error retrieving page of objects: %w", err) + } + + for _, obj := range resp.Objects { + objects = append(objects, obj.Key) + } + + if !resp.IsTruncated { + break + } + marker = resp.NextMarker + } + + return objects, nil +} + +type BlobProperties struct { + ETag string `json:"etag,omitempty"` + LastModified time.Time `json:"last_modified,omitempty"` + ContentLength int64 `json:"content_length,omitempty"` +} + +func (dsc DefaultStorageClient) Properties( + object string, +) error { + log.Printf("Getting properties for object %s/%s\n", + dsc.storageConfig.BucketName, object) + + client, err := oss.New(dsc.storageConfig.Endpoint, dsc.storageConfig.AccessKeyID, dsc.storageConfig.AccessKeySecret) + if err != nil { + return err + } + + bucket, err := client.Bucket(dsc.storageConfig.BucketName) + if err != nil { + return err + } + + meta, err := bucket.GetObjectDetailedMeta(object) + if err != nil { + var ossErr oss.ServiceError + if errors.As(err, &ossErr) && ossErr.StatusCode == 404 { + fmt.Println(`{}`) + return nil + } + + return fmt.Errorf("failed to get properties for object %s: %w", object, err) + } + + eTag := meta.Get("ETag") + lastModifiedStr := meta.Get("Last-Modified") + contentLengthStr := meta.Get("Content-Length") + + var ( + lastModified time.Time + contentLength int64 + ) + + if lastModifiedStr != "" { + t, parseErr := time.Parse(time.RFC1123, lastModifiedStr) + if parseErr == nil { + lastModified = t + } + } + + if contentLengthStr != "" { + cl, convErr := strconv.ParseInt(contentLengthStr, 10, 64) + if convErr == nil { + contentLength = cl + } + } + + props := BlobProperties{ + ETag: strings.Trim(eTag, `"`), + LastModified: lastModified, + ContentLength: contentLength, + } + + output, err := json.MarshalIndent(props, "", " ") + if err != nil { + return fmt.Errorf("failed to marshal object properties: %w", err) + } + + fmt.Println(string(output)) + return nil +} + +func (dsc DefaultStorageClient) EnsureBucketExists() error { + log.Printf("Ensuring bucket '%s' exists\n", dsc.storageConfig.BucketName) + + client, err := oss.New(dsc.storageConfig.Endpoint, dsc.storageConfig.AccessKeyID, dsc.storageConfig.AccessKeySecret) + if err != nil { + return err + } + + exists, err := client.IsBucketExist(dsc.storageConfig.BucketName) + if err != nil { + return fmt.Errorf("failed to check if bucket exists: %w", err) + } + + if exists { + log.Printf("Bucket '%s' already exists\n", dsc.storageConfig.BucketName) + return nil + } + + if err := client.CreateBucket(dsc.storageConfig.BucketName); err != nil { + return fmt.Errorf("failed to create bucket '%s': %w", dsc.storageConfig.BucketName, err) + } + + log.Printf("Bucket '%s' created successfully\n", dsc.storageConfig.BucketName) + return nil +} diff --git a/alioss/integration/general_ali_test.go b/alioss/integration/general_ali_test.go index cb64833..8a15d9f 100644 --- a/alioss/integration/general_ali_test.go +++ b/alioss/integration/general_ali_test.go @@ -2,9 +2,14 @@ package integration_test import ( "bytes" + "fmt" + "regexp" + "strings" + "time" "os" + "github.com/aliyun/aliyun-oss-go-sdk/oss" "github.com/cloudfoundry/storage-cli/alioss/config" "github.com/cloudfoundry/storage-cli/alioss/integration" @@ -151,6 +156,99 @@ var _ = Describe("General testing for all Ali regions", func() { }) }) + Describe("Invoking `delete-recursive`", func() { + It("deletes all objects with a given prefix", func() { + prefix := integration.GenerateRandomString() + blob1 := prefix + "/a" + blob2 := prefix + "/b" + otherBlob := integration.GenerateRandomString() + + contentFile1 := integration.MakeContentFile("content-1") + contentFile2 := integration.MakeContentFile("content-2") + contentFileOther := integration.MakeContentFile("other-content") + defer func() { + _ = os.Remove(contentFile1) //nolint:errcheck + _ = os.Remove(contentFile2) //nolint:errcheck + _ = os.Remove(contentFileOther) //nolint:errcheck + + for _, b := range []string{blob1, blob2, otherBlob} { + cliSession, err := integration.RunCli(cliPath, configPath, storageType, "delete", b) + if err == nil && (cliSession.ExitCode() == 0 || cliSession.ExitCode() == 3) { + continue + } + } + }() + + cliSession, err := integration.RunCli(cliPath, configPath, storageType, "put", contentFile1, blob1) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(BeZero()) + + cliSession, err = integration.RunCli(cliPath, configPath, storageType, "put", contentFile2, blob2) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(BeZero()) + + cliSession, err = integration.RunCli(cliPath, configPath, storageType, "put", contentFileOther, otherBlob) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(BeZero()) + + cliSession, err = integration.RunCli(cliPath, configPath, storageType, "delete-recursive", prefix) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(BeZero()) + + cliSession, err = integration.RunCli(cliPath, configPath, storageType, "exists", blob1) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(Equal(3)) + + cliSession, err = integration.RunCli(cliPath, configPath, storageType, "exists", blob2) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(Equal(3)) + + cliSession, err = integration.RunCli(cliPath, configPath, storageType, "exists", otherBlob) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(Equal(0)) + }) + }) + + Describe("Invoking `copy`", func() { + It("copies the contents from one object to another", func() { + srcBlob := blobName + "-src" + destBlob := blobName + "-dest" + + defer func() { + for _, b := range []string{srcBlob, destBlob} { + cliSession, err := integration.RunCli(cliPath, configPath, storageType, "delete", b) + if err != nil { + GinkgoWriter.Printf("cleanup: error deleting %s: %v\n", b, err) + continue + } + if cliSession.ExitCode() != 0 && cliSession.ExitCode() != 3 { + GinkgoWriter.Printf("cleanup: delete %s exited with code %d\n", b, cliSession.ExitCode()) + } + } + }() + + contentFile = integration.MakeContentFile("copied content") + cliSession, err := integration.RunCli(cliPath, configPath, storageType, "put", contentFile, srcBlob) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(BeZero()) + + cliSession, err = integration.RunCli(cliPath, configPath, storageType, "copy", srcBlob, destBlob) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(BeZero()) + + tmpLocalFile, _ := os.CreateTemp("", "ali-storage-cli-copy") //nolint:errcheck + tmpLocalFile.Close() //nolint:errcheck + defer func() { _ = os.Remove(tmpLocalFile.Name()) }() //nolint:errcheck + + cliSession, err = integration.RunCli(cliPath, configPath, storageType, "get", destBlob, tmpLocalFile.Name()) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(BeZero()) + + gottenBytes, _ := os.ReadFile(tmpLocalFile.Name()) //nolint:errcheck + Expect(string(gottenBytes)).To(Equal("copied content")) + }) + }) + Describe("Invoking `exists`", func() { It("returns 0 for an existing blob", func() { defer func() { @@ -211,4 +309,162 @@ var _ = Describe("General testing for all Ali regions", func() { Expect(consoleOutput).To(ContainSubstring("version")) }) }) + + Describe("Invoking `list`", func() { + It("lists all blobs with a given prefix", func() { + prefix := integration.GenerateRandomString() + blob1 := prefix + "/a" + blob2 := prefix + "/b" + otherBlob := integration.GenerateRandomString() + + defer func() { + for _, b := range []string{blob1, blob2, otherBlob} { + _, err := integration.RunCli(cliPath, configPath, storageType, "delete", b) + Expect(err).ToNot(HaveOccurred()) + } + }() + + contentFile1 := integration.MakeContentFile("list-1") + contentFile2 := integration.MakeContentFile("list-2") + contentFileOther := integration.MakeContentFile("list-other") + defer func() { + _ = os.Remove(contentFile1) //nolint:errcheck + _ = os.Remove(contentFile2) //nolint:errcheck + _ = os.Remove(contentFileOther) //nolint:errcheck + }() + + cliSession, err := integration.RunCli(cliPath, configPath, storageType, "put", contentFile1, blob1) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(BeZero()) + + cliSession, err = integration.RunCli(cliPath, configPath, storageType, "put", contentFile2, blob2) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(BeZero()) + + cliSession, err = integration.RunCli(cliPath, configPath, storageType, "put", contentFileOther, otherBlob) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(BeZero()) + + cliSession, err = integration.RunCli(cliPath, configPath, storageType, "list", prefix) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(BeZero()) + + output := bytes.NewBuffer(cliSession.Out.Contents()).String() + + Expect(output).To(ContainSubstring(blob1)) + Expect(output).To(ContainSubstring(blob2)) + Expect(output).NotTo(ContainSubstring(otherBlob)) + }) + + It("lists all blobs across multiple pages", func() { + prefix := integration.GenerateRandomString() + const totalObjects = 120 + + var blobNames []string + var contentFiles []string + + for i := 0; i < totalObjects; i++ { + blobName := fmt.Sprintf("%s/%03d", prefix, i) + blobNames = append(blobNames, blobName) + + contentFile := integration.MakeContentFile(fmt.Sprintf("content-%d", i)) + contentFiles = append(contentFiles, contentFile) + + cliSession, err := integration.RunCli(cliPath, configPath, storageType, "put", contentFile, blobName) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(BeZero()) + } + + defer func() { + for _, f := range contentFiles { + _ = os.Remove(f) //nolint:errcheck + } + + for _, b := range blobNames { + cliSession, err := integration.RunCli(cliPath, configPath, storageType, "delete", b) + if err == nil && (cliSession.ExitCode() == 0 || cliSession.ExitCode() == 3) { + continue + } + } + }() + + cliSession, err := integration.RunCli(cliPath, configPath, storageType, "list", prefix) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(BeZero()) + + output := bytes.NewBuffer(cliSession.Out.Contents()).String() + + for _, b := range blobNames { + Expect(output).To(ContainSubstring(b)) + } + }) + }) + + const maxBucketLen = 63 + const suffixLen = 8 + + Describe("Invoking `ensure-bucket-exists`", func() { + It("creates a bucket that can be observed via the OSS API", func() { + + base := bucketName + + maxBaseLen := maxBucketLen - 1 - suffixLen + if maxBaseLen < 1 { + maxBaseLen = maxBucketLen - 1 + } + if len(base) > maxBaseLen { + base = base[:maxBaseLen] + } + + rawSuffix := integration.GenerateRandomString() + safe := strings.ToLower(rawSuffix) + + re := regexp.MustCompile(`[^a-z0-9]`) + safe = re.ReplaceAllString(safe, "") + if len(safe) < suffixLen { + safe = safe + "0123456789abcdef" + } + safe = safe[:suffixLen] + + newBucketName := fmt.Sprintf("%s-%s", base, safe) + + cfg := defaultConfig + cfg.BucketName = newBucketName + + configPath = integration.MakeConfigFile(&cfg) + defer func() { _ = os.Remove(configPath) }() //nolint:errcheck + + ossClient, err := oss.New(cfg.Endpoint, cfg.AccessKeyID, cfg.AccessKeySecret) + Expect(err).ToNot(HaveOccurred()) + + defer func() { + if err := ossClient.DeleteBucket(newBucketName); err != nil { + if svcErr, ok := err.(oss.ServiceError); ok && svcErr.StatusCode == 404 { + return + } + fmt.Fprintf(GinkgoWriter, "cleanup: failed to delete bucket %s: %v\n", newBucketName, err) //nolint:errcheck + } + }() + + s1, err := integration.RunCli(cliPath, configPath, storageType, "ensure-storage-exists") + Expect(err).ToNot(HaveOccurred()) + Expect(s1.ExitCode()).To(BeZero()) + + Eventually(func(g Gomega) bool { + exists, existsErr := ossClient.IsBucketExist(newBucketName) + g.Expect(existsErr).ToNot(HaveOccurred()) + return exists + }, 30*time.Second, 1*time.Second).Should(BeTrue()) + }) + + It("is idempotent", func() { + s1, err := integration.RunCli(cliPath, configPath, storageType, "ensure-storage-exists") + Expect(err).ToNot(HaveOccurred()) + Expect(s1.ExitCode()).To(BeZero()) + + s2, err := integration.RunCli(cliPath, configPath, storageType, "ensure-storage-exists") + Expect(err).ToNot(HaveOccurred()) + Expect(s2.ExitCode()).To(BeZero()) + }) + }) })