From 4f45a21681fd1e6e83216396ba242021044f36e8 Mon Sep 17 00:00:00 2001 From: KOJIMA Kazunori Date: Sat, 27 Feb 2021 17:33:44 +0900 Subject: [PATCH] Add ability to fetch restored glacier object --- .../S3NameOrderPrefixFileExplorer.java | 13 +++++++++++++ .../s3/explorer/S3PrefixFileExplorer.java | 11 ++++++++++- .../S3TimeOrderPrefixFileExplorer.java | 13 +++++++++++++ .../s3/explorer/TestS3PrefixFileExplorer.java | 19 +++++++++++++++++++ 4 files changed, 55 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/embulk/input/s3/explorer/S3NameOrderPrefixFileExplorer.java b/src/main/java/org/embulk/input/s3/explorer/S3NameOrderPrefixFileExplorer.java index 29b9611..69b4065 100644 --- a/src/main/java/org/embulk/input/s3/explorer/S3NameOrderPrefixFileExplorer.java +++ b/src/main/java/org/embulk/input/s3/explorer/S3NameOrderPrefixFileExplorer.java @@ -17,8 +17,10 @@ package org.embulk.input.s3.explorer; import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.model.GetObjectMetadataRequest; import com.amazonaws.services.s3.model.ListObjectsRequest; import com.amazonaws.services.s3.model.ObjectListing; +import com.amazonaws.services.s3.model.ObjectMetadata; import com.amazonaws.services.s3.model.S3ObjectSummary; import org.embulk.input.s3.DefaultRetryable; import org.embulk.util.retryhelper.RetryExecutor; @@ -53,6 +55,17 @@ public ObjectListing call() return ol.getObjectSummaries(); } + @Override + protected ObjectMetadata fetchObjectMetadata(S3ObjectSummary obj) { + final GetObjectMetadataRequest req = new GetObjectMetadataRequest(obj.getBucketName(), obj.getKey()); + + return new DefaultRetryable("Get object metadata") + { + @Override + public ObjectMetadata call() { return s3Client.getObjectMetadata(req); } + }.executeWith(retryExecutor); + } + @Override protected boolean hasNext() { diff --git a/src/main/java/org/embulk/input/s3/explorer/S3PrefixFileExplorer.java b/src/main/java/org/embulk/input/s3/explorer/S3PrefixFileExplorer.java index 0f75fe3..5759d03 100644 --- a/src/main/java/org/embulk/input/s3/explorer/S3PrefixFileExplorer.java +++ b/src/main/java/org/embulk/input/s3/explorer/S3PrefixFileExplorer.java @@ -17,6 +17,7 @@ package org.embulk.input.s3.explorer; import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.model.ObjectMetadata; import com.amazonaws.services.s3.model.S3ObjectSummary; import com.amazonaws.services.s3.model.StorageClass; import org.embulk.config.ConfigException; @@ -54,7 +55,13 @@ public void addToBuilder(final FileList.Builder builder) LOGGER.warn("Skipped \"s3://{}/{}\" that stored at Glacier.", bucketName, s.getKey()); continue; } - throw new ConfigException("Detected an object stored at Glacier. Set \"skip_glacier_objects\" option to \"true\" to skip this."); + + ObjectMetadata objectMetadata = fetchObjectMetadata(s); + if (objectMetadata != null && objectMetadata.getRestoreExpirationTime() != null) { + LOGGER.info("Restored Glacier object \"s3://{}/{}\" found", bucketName, s.getKey()); + } else { + throw new ConfigException("Detected an object stored at Glacier. Set \"skip_glacier_objects\" option to \"true\" to skip this."); + } } if (s.getSize() > 0) { builder.add(s.getKey(), s.getSize()); @@ -69,5 +76,7 @@ public void addToBuilder(final FileList.Builder builder) protected abstract List fetch(); + protected abstract ObjectMetadata fetchObjectMetadata(final S3ObjectSummary obj); + protected abstract boolean hasNext(); } diff --git a/src/main/java/org/embulk/input/s3/explorer/S3TimeOrderPrefixFileExplorer.java b/src/main/java/org/embulk/input/s3/explorer/S3TimeOrderPrefixFileExplorer.java index 3f8eede..207d42e 100644 --- a/src/main/java/org/embulk/input/s3/explorer/S3TimeOrderPrefixFileExplorer.java +++ b/src/main/java/org/embulk/input/s3/explorer/S3TimeOrderPrefixFileExplorer.java @@ -17,8 +17,10 @@ package org.embulk.input.s3.explorer; import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.model.GetObjectMetadataRequest; import com.amazonaws.services.s3.model.ListObjectsRequest; import com.amazonaws.services.s3.model.ObjectListing; +import com.amazonaws.services.s3.model.ObjectMetadata; import com.amazonaws.services.s3.model.S3ObjectSummary; import org.embulk.input.s3.DefaultRetryable; import org.embulk.util.retryhelper.RetryExecutor; @@ -72,6 +74,17 @@ public ObjectListing call() .collect(Collectors.toList()); } + @Override + protected ObjectMetadata fetchObjectMetadata(S3ObjectSummary obj) { + final GetObjectMetadataRequest req = new GetObjectMetadataRequest(obj.getBucketName(), obj.getKey()); + + return new DefaultRetryable("Get object metadata") + { + @Override + public ObjectMetadata call() { return s3Client.getObjectMetadata(req); } + }.executeWith(retryExecutor); + } + @Override public boolean hasNext() { diff --git a/src/test/java/org/embulk/input/s3/explorer/TestS3PrefixFileExplorer.java b/src/test/java/org/embulk/input/s3/explorer/TestS3PrefixFileExplorer.java index c5bc6b2..97d498f 100644 --- a/src/test/java/org/embulk/input/s3/explorer/TestS3PrefixFileExplorer.java +++ b/src/test/java/org/embulk/input/s3/explorer/TestS3PrefixFileExplorer.java @@ -17,6 +17,7 @@ package org.embulk.input.s3.explorer; import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.model.ObjectMetadata; import com.amazonaws.services.s3.model.S3ObjectSummary; import com.amazonaws.services.s3.model.StorageClass; import org.embulk.EmbulkTestRuntime; @@ -31,6 +32,7 @@ import org.mockito.runners.MockitoJUnitRunner; import java.util.Collections; +import java.util.Date; import java.util.List; import static org.mockito.Mockito.doReturn; @@ -75,6 +77,20 @@ public void addToBuilder_should_throw_exception_if_notskipped_glacier_storage() s3PrefixFileExplorer.addToBuilder(builder); } + @Test + public void addToBuilder_should_not_throw_exception_if_restored_glacier_object() + { + when(s3ObjectSummary.getStorageClass()).thenReturn(StorageClass.Glacier.toString()); + when(s3ObjectSummary.getKey()).thenReturn(PATH_PREFIX + OBJECT_KEY); + when(s3ObjectSummary.getSize()).thenReturn(1L); + ObjectMetadata objectMetadata = new ObjectMetadata(); + objectMetadata.setRestoreExpirationTime(new Date()); + doReturn(objectMetadata).when(s3PrefixFileExplorer).fetchObjectMetadata(s3ObjectSummary); + doReturn(true).when(s3PrefixFileExplorer).hasNext(); + s3PrefixFileExplorer.addToBuilder(builder); + verify(builder).add(PATH_PREFIX + OBJECT_KEY, 1); + } + @Test public void addToBuilder_should_skip_glacier_storage_if_allowed() { @@ -132,6 +148,9 @@ protected List fetch() return null; } + @Override + protected ObjectMetadata fetchObjectMetadata(S3ObjectSummary obj) { return null; } + @Override protected boolean hasNext() {