From 769c15862067733f92201c04dc1fc835a631f613 Mon Sep 17 00:00:00 2001 From: Gregory <32060161+ggmaleva@users.noreply.github.com> Date: Wed, 24 Jan 2024 14:52:17 +0300 Subject: [PATCH] add generateMultipartDownloadUrl method (#22) Co-authored-by: ggmaleva --- pom.xml | 2 +- .../storage/handler/FileStorageHandler.java | 16 +++++++ .../file/storage/service/S3Service.java | 5 +++ .../file/storage/service/S3V2Service.java | 44 ++++++++++++++----- .../file/storage/service/StorageService.java | 2 + .../vality/file/storage/FileStorageTest.java | 27 ++++++++++++ 6 files changed, 83 insertions(+), 13 deletions(-) diff --git a/pom.xml b/pom.xml index ca8a22a..dea9a71 100644 --- a/pom.xml +++ b/pom.xml @@ -30,7 +30,7 @@ dev.vality file-storage-proto - 1.47-0f31e02 + 1.48-4569cea dev.vality diff --git a/src/main/java/dev/vality/file/storage/handler/FileStorageHandler.java b/src/main/java/dev/vality/file/storage/handler/FileStorageHandler.java index 3564dc0..7a971fc 100644 --- a/src/main/java/dev/vality/file/storage/handler/FileStorageHandler.java +++ b/src/main/java/dev/vality/file/storage/handler/FileStorageHandler.java @@ -92,6 +92,22 @@ public class FileStorageHandler implements FileStorageSrv.Iface { return result; } + @Override + public String generateMultipartDownloadUrl(String fileDataId, String expiresAt) throws TException { + try { + log.info("Receive request for generate download url with fileDataId={}", fileDataId); + CheckerUtil.checkString(fileDataId, "Bad request parameter, fileDataId required and not empty arg"); + CheckerUtil.checkString(expiresAt, "Bad request parameter, expiresAt required and not empty arg"); + Instant instant = TypeUtil.stringToInstant(expiresAt); + URL url = storageService.generateMultipartDownloadUrl(fileDataId, instant); + log.info("Successfully generate download url with fileDataId={}", fileDataId); + log.debug("Generated download url={}", url); + return url.toString(); + } catch (FileNotFoundException e) { + throw fileNotFound(e); + } + } + private FileNotFound fileNotFound(FileNotFoundException e) { log.warn("File not found", e); return new FileNotFound(); diff --git a/src/main/java/dev/vality/file/storage/service/S3Service.java b/src/main/java/dev/vality/file/storage/service/S3Service.java index b0032d6..2818406 100644 --- a/src/main/java/dev/vality/file/storage/service/S3Service.java +++ b/src/main/java/dev/vality/file/storage/service/S3Service.java @@ -134,6 +134,11 @@ public class S3Service implements StorageService { throw new UnsupportedOperationException(METHOD_NOT_SUPPORTED); } + @Override + public URL generateMultipartDownloadUrl(String fileDataId, Instant expirationTime) { + throw new UnsupportedOperationException(METHOD_NOT_SUPPORTED); + } + @PreDestroy public void terminate() { transferManager.shutdownNow(true); diff --git a/src/main/java/dev/vality/file/storage/service/S3V2Service.java b/src/main/java/dev/vality/file/storage/service/S3V2Service.java index 89ed0d6..3a1e80c 100644 --- a/src/main/java/dev/vality/file/storage/service/S3V2Service.java +++ b/src/main/java/dev/vality/file/storage/service/S3V2Service.java @@ -20,6 +20,7 @@ import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.*; import software.amazon.awssdk.services.s3.presigner.S3Presigner; import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest; +import software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequest; import software.amazon.awssdk.services.s3.presigner.model.PutObjectPresignRequest; import javax.annotation.PostConstruct; @@ -66,18 +67,7 @@ public class S3V2Service implements StorageService { var versions = getObjectVersions(fileId); checkFileExist(fileId, versions); var fileVersionId = getFileVersionId(fileId, versions); - var presignRequest = GetObjectPresignRequest.builder() - .signatureDuration(Duration.between(Instant.now(), expirationTime)) - .getObjectRequest(GetObjectRequest.builder() - .bucket(s3SdkV2Properties.getBucketName()) - .key(fileId) - .versionId(fileVersionId) - .build()) - .build(); - var presignedRequest = s3Presigner.presignGetObject(presignRequest); - log.info("Download url was presigned, fileId={}, bucketName={}, isBrowserExecutable={}", - fileId, s3SdkV2Properties.getBucketName(), presignedRequest.isBrowserExecutable()); - log.debug("Presigned http request={}", presignedRequest.httpRequest().toString()); + PresignedGetObjectRequest presignedRequest = getPresignedRequest(fileId, expirationTime, fileVersionId); return presignedRequest.url(); } @@ -510,6 +500,36 @@ public class S3V2Service implements StorageService { } } + @Override + public URL generateMultipartDownloadUrl(String fileId, Instant expirationTime) { + var versions = getObjectVersions(fileId); + if (CollectionUtils.isEmpty(versions)) { + throw new FileNotFoundException(String.format( + "Failed to check object version with file on exist, fileId=%s, bucketName=%s ", + fileId, + s3SdkV2Properties.getBucketName())); + } + var fileVersionId = getFileVersionId(fileId, versions); + PresignedGetObjectRequest presignedRequest = getPresignedRequest(fileId, expirationTime, fileVersionId); + return presignedRequest.url(); + } + + private PresignedGetObjectRequest getPresignedRequest(String fileId, Instant expirationTime, String fileVersionId) { + var presignRequest = GetObjectPresignRequest.builder() + .signatureDuration(Duration.between(Instant.now(), expirationTime)) + .getObjectRequest(GetObjectRequest.builder() + .bucket(s3SdkV2Properties.getBucketName()) + .key(fileId) + .versionId(fileVersionId) + .build()) + .build(); + var presignedRequest = s3Presigner.presignGetObject(presignRequest); + log.info("Download url was presigned, fileId={}, bucketName={}, isBrowserExecutable={}", + fileId, s3SdkV2Properties.getBucketName(), presignedRequest.isBrowserExecutable()); + log.debug("Presigned http request={}", presignedRequest.httpRequest().toString()); + return presignedRequest; + } + private software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest buildRequest( CompleteMultipartUploadRequest request, String fileId, diff --git a/src/main/java/dev/vality/file/storage/service/StorageService.java b/src/main/java/dev/vality/file/storage/service/StorageService.java index 91891f7..70c7dee 100644 --- a/src/main/java/dev/vality/file/storage/service/StorageService.java +++ b/src/main/java/dev/vality/file/storage/service/StorageService.java @@ -23,4 +23,6 @@ public interface StorageService { CompleteMultipartUploadResult completeMultipartUpload(CompleteMultipartUploadRequest request); + URL generateMultipartDownloadUrl(String fileDataId, Instant expirationTime); + } diff --git a/src/test/java/dev/vality/file/storage/FileStorageTest.java b/src/test/java/dev/vality/file/storage/FileStorageTest.java index a0e7c50..b7c4278 100644 --- a/src/test/java/dev/vality/file/storage/FileStorageTest.java +++ b/src/test/java/dev/vality/file/storage/FileStorageTest.java @@ -27,6 +27,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardCopyOption; +import java.time.Instant; import java.util.*; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; @@ -425,4 +426,30 @@ public abstract class FileStorageTest { assertNotNull(multipartFileData.getCreatedAt()); assertNotNull(multipartFileData.getMetadata()); } + + @Test + void generateMultipartDownloadUrl() throws Exception { + dev.vality.msgpack.Value value = new dev.vality.msgpack.Value(); + String fileName = "test_registry.csv"; + value.setStr(fileName); + Map metadata = Map.of("filename", value); + CreateMultipartUploadResult createResult = fileStorageClient.createMultipartUpload(metadata); + assertNotNull(createResult.getFileDataId()); + assertNotNull(createResult.getMultipartUploadId()); + + List completedParts = new ArrayList<>(); + processMultipartUpload(createResult, completedParts); + + var completeRequest = new CompleteMultipartUploadRequest() + .setMultipartUploadId(createResult.getMultipartUploadId()) + .setFileDataId(createResult.getFileDataId()) + .setCompletedParts(completedParts); + + fileStorageClient.completeMultipartUpload(completeRequest); + + String expiredTime = Instant.now().toString(); + String url = fileStorageClient.generateMultipartDownloadUrl(createResult.getFileDataId(), expiredTime); + + assertNotNull(url); + } }