From 376a7036057dc898b0378c90e0a37d706d5be404 Mon Sep 17 00:00:00 2001 From: "a.karlov" Date: Mon, 3 Dec 2018 11:24:59 +0300 Subject: [PATCH] BJ-314: test branch --- pom.xml | 42 +- .../file/storage/FileStorageApplication.java | 4 +- ....java => AmazonS3ClientConfiguration.java} | 8 +- .../storage/config/HandlerConfiguration.java | 18 + .../storage/config/SwaggerConfiguration.java | 35 -- .../storage/contorller/FileController.java | 34 -- .../contorller/UploadFileController.java | 42 ++ .../storage/handler/FileStorageHandler.java | 78 ++++ .../storage/resource/FileStorageServlet.java | 30 ++ .../service/AmazonS3StorageService.java | 387 ++++++++++++++++-- .../file/storage/service/StorageService.java | 21 +- .../file/storage/util/CheckerUtil.java | 13 + .../file/storage/util/DamselUtil.java | 35 ++ src/main/resources/application.properties | 5 + .../file/storage/AbstractIntegrationTest.java | 81 ++++ .../rbkmoney/file/storage/service/Aasd.java | 167 ++++++++ .../service/MetadataS3ObjectsTest.java | 20 + .../service/PresignedUrlAccessRightsTest.java | 111 +++++ src/test/resources/logback-test.xml | 30 ++ 19 files changed, 1035 insertions(+), 126 deletions(-) rename src/main/java/com/rbkmoney/file/storage/config/{StorageConfig.java => AmazonS3ClientConfiguration.java} (94%) create mode 100644 src/main/java/com/rbkmoney/file/storage/config/HandlerConfiguration.java delete mode 100644 src/main/java/com/rbkmoney/file/storage/config/SwaggerConfiguration.java delete mode 100644 src/main/java/com/rbkmoney/file/storage/contorller/FileController.java create mode 100644 src/main/java/com/rbkmoney/file/storage/contorller/UploadFileController.java create mode 100644 src/main/java/com/rbkmoney/file/storage/handler/FileStorageHandler.java create mode 100644 src/main/java/com/rbkmoney/file/storage/resource/FileStorageServlet.java create mode 100644 src/main/java/com/rbkmoney/file/storage/util/CheckerUtil.java create mode 100644 src/main/java/com/rbkmoney/file/storage/util/DamselUtil.java create mode 100644 src/test/java/com/rbkmoney/file/storage/AbstractIntegrationTest.java create mode 100644 src/test/java/com/rbkmoney/file/storage/service/Aasd.java create mode 100644 src/test/java/com/rbkmoney/file/storage/service/MetadataS3ObjectsTest.java create mode 100644 src/test/java/com/rbkmoney/file/storage/service/PresignedUrlAccessRightsTest.java create mode 100644 src/test/resources/logback-test.xml diff --git a/pom.xml b/pom.xml index 023a970..d10628b 100644 --- a/pom.xml +++ b/pom.xml @@ -23,10 +23,34 @@ UTF-8 1.8 - 2.8.0 + 1.1.15 + 1.5-81b8f4a + 0.6.8 + + + com.rbkmoney + file-storage-proto + ${file.storage.proto.version} + + + com.rbkmoney.woody + woody-thrift + ${woody.thrift.version} + + + com.rbkmoney.geck + common + ${geck.version} + + + com.rbkmoney.geck + serializer + ${geck.version} + + org.springframework.boot @@ -44,16 +68,6 @@ aws-java-sdk-s3 1.11.160 - - io.springfox - springfox-swagger2 - ${springfox-swagger2.version} - - - io.springfox - springfox-swagger-ui - ${springfox-swagger2.version} - org.projectlombok lombok @@ -67,6 +81,12 @@ spring-boot-starter-test test + + org.testcontainers + testcontainers + 1.8.3 + compile + diff --git a/src/main/java/com/rbkmoney/file/storage/FileStorageApplication.java b/src/main/java/com/rbkmoney/file/storage/FileStorageApplication.java index 1fb510d..cc184ae 100644 --- a/src/main/java/com/rbkmoney/file/storage/FileStorageApplication.java +++ b/src/main/java/com/rbkmoney/file/storage/FileStorageApplication.java @@ -2,8 +2,10 @@ package com.rbkmoney.file.storage; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.web.servlet.ServletComponentScan; -@SpringBootApplication +@ServletComponentScan +@SpringBootApplication(scanBasePackages = {"com.rbkmoney.file.storage"}) public class FileStorageApplication { public static void main(String[] args) { diff --git a/src/main/java/com/rbkmoney/file/storage/config/StorageConfig.java b/src/main/java/com/rbkmoney/file/storage/config/AmazonS3ClientConfiguration.java similarity index 94% rename from src/main/java/com/rbkmoney/file/storage/config/StorageConfig.java rename to src/main/java/com/rbkmoney/file/storage/config/AmazonS3ClientConfiguration.java index cfb22e7..94840f2 100644 --- a/src/main/java/com/rbkmoney/file/storage/config/StorageConfig.java +++ b/src/main/java/com/rbkmoney/file/storage/config/AmazonS3ClientConfiguration.java @@ -16,7 +16,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @Configuration -public class StorageConfig { +public class AmazonS3ClientConfiguration { @Value("${storage.endpoint}") private String endpoint; @@ -24,13 +24,13 @@ public class StorageConfig { @Value("${storage.signingRegion}") private String signingRegion; - @Value("${storage.accessKey:}") + @Value("${storage.accessKey}") private String accessKey; - @Value("${storage.secretKey:}") + @Value("${storage.secretKey}") private String secretKey; - @Value("${storage.client.protocol:HTTP}") + @Value("${storage.client.protocol}") private Protocol protocol; @Value("${storage.client.maxErrorRetry}") diff --git a/src/main/java/com/rbkmoney/file/storage/config/HandlerConfiguration.java b/src/main/java/com/rbkmoney/file/storage/config/HandlerConfiguration.java new file mode 100644 index 0000000..066f1f1 --- /dev/null +++ b/src/main/java/com/rbkmoney/file/storage/config/HandlerConfiguration.java @@ -0,0 +1,18 @@ +package com.rbkmoney.file.storage.config; + +import com.rbkmoney.file.storage.FileStorageSrv; +import com.rbkmoney.file.storage.handler.FileStorageHandler; +import com.rbkmoney.file.storage.service.StorageService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class HandlerConfiguration { + + @Bean + @Autowired + public FileStorageSrv.Iface fileStorageHandler(StorageService storageService) { + return new FileStorageHandler(storageService); + } +} diff --git a/src/main/java/com/rbkmoney/file/storage/config/SwaggerConfiguration.java b/src/main/java/com/rbkmoney/file/storage/config/SwaggerConfiguration.java deleted file mode 100644 index 6c7031b..0000000 --- a/src/main/java/com/rbkmoney/file/storage/config/SwaggerConfiguration.java +++ /dev/null @@ -1,35 +0,0 @@ -package com.rbkmoney.file.storage.config; - -import com.google.common.base.Predicates; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import springfox.documentation.builders.ApiInfoBuilder; -import springfox.documentation.builders.PathSelectors; -import springfox.documentation.service.ApiInfo; -import springfox.documentation.service.Contact; -import springfox.documentation.spi.DocumentationType; -import springfox.documentation.spring.web.plugins.Docket; -import springfox.documentation.swagger2.annotations.EnableSwagger2; - -@Configuration -@EnableSwagger2 -public class SwaggerConfiguration { - - @Bean - public Docket productApi() { - return new Docket(DocumentationType.SWAGGER_2) - .apiInfo(metaData()) - .select() - .paths(Predicates.not(PathSelectors.regex("/error"))) - .build(); - } - - private ApiInfo metaData() { - return new ApiInfoBuilder() - .title("REST endpoint") - .description("\"Endpoint для выгрузки документов на сервер\"") - .version("0.0.1-SNAPSHOT") - .contact(new Contact("RBK.money", "https://github.com/rbkmoney", "support@rbkmoney.com")) - .build(); - } -} diff --git a/src/main/java/com/rbkmoney/file/storage/contorller/FileController.java b/src/main/java/com/rbkmoney/file/storage/contorller/FileController.java deleted file mode 100644 index cc62da3..0000000 --- a/src/main/java/com/rbkmoney/file/storage/contorller/FileController.java +++ /dev/null @@ -1,34 +0,0 @@ -package com.rbkmoney.file.storage.contorller; - -import com.rbkmoney.file.storage.service.StorageService; -import io.swagger.annotations.*; -import lombok.RequiredArgsConstructor; -import org.springframework.http.HttpStatus; -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RestController; -import org.springframework.web.multipart.MultipartFile; - -@RestController -@Api(description = "Api для операций с файлами") -@RequiredArgsConstructor -public class FileController { - - private final StorageService storageService; - - @PostMapping("/upload") - @ApiOperation(value = "Выгрузить файл на сервер") - @ApiResponses(value = { - @ApiResponse(code = 200, message = "Файл выгружен на сервер") - }) - public ResponseEntity handleFileUpload(@ApiParam(value = "выгружаемый файл", required = true) - @RequestParam(value = "file") - MultipartFile file, - @ApiParam(value = "id файла", required = true) - @RequestParam(value = "file_id") - String fileId) { - storageService.store(fileId, file); - return new ResponseEntity(HttpStatus.OK); - } -} diff --git a/src/main/java/com/rbkmoney/file/storage/contorller/UploadFileController.java b/src/main/java/com/rbkmoney/file/storage/contorller/UploadFileController.java new file mode 100644 index 0000000..edc2c26 --- /dev/null +++ b/src/main/java/com/rbkmoney/file/storage/contorller/UploadFileController.java @@ -0,0 +1,42 @@ +package com.rbkmoney.file.storage.contorller; + +import com.rbkmoney.file.storage.service.StorageService; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.multipart.MultipartFile; + +import java.io.FileNotFoundException; + +import static com.rbkmoney.file.storage.util.CheckerUtil.checkString; + +@RestController +@RequiredArgsConstructor +@Slf4j +public class UploadFileController { + + private final StorageService storageService; + + @PostMapping("/file_storage/upload") + public ResponseEntity handleFileUpload(@RequestParam(value = "file_id") String fileId, + @RequestParam(value = "file") MultipartFile file) { + try { + log.info("Request handleFileUpload fileId: {}", fileId); + checkString(fileId, "Bad request parameter, fileId required and not empty arg"); + storageService.uploadFile(fileId, file.getInputStream()); + ResponseEntity responseEntity = ResponseEntity.ok().build(); + log.info("Response: ResponseEntity: {}", responseEntity); + return responseEntity; + } catch (FileNotFoundException e) { + log.error("Error when handleFileUpload e: ", e); + return ResponseEntity.notFound().build(); + } catch (Exception e) { + log.error("Error when handleFileUpload e: ", e); + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).build(); + } + } +} diff --git a/src/main/java/com/rbkmoney/file/storage/handler/FileStorageHandler.java b/src/main/java/com/rbkmoney/file/storage/handler/FileStorageHandler.java new file mode 100644 index 0000000..9dfdaa8 --- /dev/null +++ b/src/main/java/com/rbkmoney/file/storage/handler/FileStorageHandler.java @@ -0,0 +1,78 @@ +package com.rbkmoney.file.storage.handler; + +import com.rbkmoney.damsel.msgpack.Value; +import com.rbkmoney.file.storage.FileData; +import com.rbkmoney.file.storage.FileNotFound; +import com.rbkmoney.file.storage.FileStorageSrv; +import com.rbkmoney.file.storage.NewFileResult; +import com.rbkmoney.file.storage.service.StorageService; +import com.rbkmoney.geck.common.util.TypeUtil; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.thrift.TException; + +import java.io.FileNotFoundException; +import java.net.URL; +import java.time.Instant; +import java.util.Map; + +import static com.rbkmoney.file.storage.util.CheckerUtil.checkString; + +@RequiredArgsConstructor +@Slf4j +public class FileStorageHandler implements FileStorageSrv.Iface { + + private final StorageService storageService; + + @Override + public FileData getFileData(String fileId) throws TException { + try { + log.info("Request getFileData fileId: {}", fileId); + checkString(fileId, "Bad request parameter, fileId required and not empty arg"); + FileData fileData = storageService.getFileData(fileId); + log.info("Response: fileData: {}", fileData); + return fileData; + } catch (FileNotFoundException e) { + log.error("Error when getFileData e: ", e); + throw new FileNotFound(); + } catch (Exception e) { + log.error("Error when getFileData e: ", e); + throw new TException(e); + } + } + + @Override + public NewFileResult createNewFile(String fileName, Map metadata, String expiresAt) throws TException { + try { + log.info("Request createNewFile fileName: {}, metadata: {}, expiresAt: {}", fileName, metadata, expiresAt); + checkString(fileName, "Bad request parameter, fileName required and not empty arg"); + checkString(expiresAt, "Bad request parameter, expiresAt required and not empty arg"); + Instant instant = TypeUtil.stringToInstant(expiresAt); + NewFileResult newFile = storageService.createNewFile(fileName, metadata, instant); + log.info("Response: newFileResult: {}", newFile); + return newFile; + } catch (Exception e) { + log.error("Error when createNewFile e: ", e); + throw new TException(e); + } + } + + @Override + public String generateDownloadUrl(String fileId, String expiresAt) throws TException { + try { + log.info("Request generateDownloadUrl fileId: {}, expiresAt: {}", fileId, expiresAt); + checkString(fileId, "Bad request parameter, fileId required and not empty arg"); + checkString(expiresAt, "Bad request parameter, expiresAt required and not empty arg"); + Instant instant = TypeUtil.stringToInstant(expiresAt); + URL url = storageService.generateDownloadUrl(fileId, instant); + log.info("Response: url: {}", url); + return url.toString(); + } catch (FileNotFoundException e) { + log.error("Error when generateDownloadUrl e: ", e); + throw new FileNotFound(); + } catch (Exception e) { + log.error("Error when generateDownloadUrl e: ", e); + throw new TException(e); + } + } +} diff --git a/src/main/java/com/rbkmoney/file/storage/resource/FileStorageServlet.java b/src/main/java/com/rbkmoney/file/storage/resource/FileStorageServlet.java new file mode 100644 index 0000000..03bd908 --- /dev/null +++ b/src/main/java/com/rbkmoney/file/storage/resource/FileStorageServlet.java @@ -0,0 +1,30 @@ +package com.rbkmoney.file.storage.resource; + +import com.rbkmoney.file.storage.FileStorageSrv; +import com.rbkmoney.woody.thrift.impl.http.THServiceBuilder; +import lombok.RequiredArgsConstructor; + +import javax.servlet.*; +import javax.servlet.annotation.WebServlet; +import java.io.IOException; + +@WebServlet("/file_storage") +@RequiredArgsConstructor +public class FileStorageServlet extends GenericServlet { + + private Servlet thriftServlet; + + private final FileStorageSrv.Iface requestHandler; + + @Override + public void init(ServletConfig config) throws ServletException { + super.init(config); + thriftServlet = new THServiceBuilder() + .build(FileStorageSrv.Iface.class, requestHandler); + } + + @Override + public void service(ServletRequest req, ServletResponse res) throws ServletException, IOException { + thriftServlet.service(req, res); + } +} diff --git a/src/main/java/com/rbkmoney/file/storage/service/AmazonS3StorageService.java b/src/main/java/com/rbkmoney/file/storage/service/AmazonS3StorageService.java index 083862e..4278ea5 100644 --- a/src/main/java/com/rbkmoney/file/storage/service/AmazonS3StorageService.java +++ b/src/main/java/com/rbkmoney/file/storage/service/AmazonS3StorageService.java @@ -1,99 +1,408 @@ package com.rbkmoney.file.storage.service; import com.amazonaws.AmazonClientException; +import com.amazonaws.HttpMethod; import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.model.ObjectMetadata; -import com.amazonaws.services.s3.model.PutObjectRequest; +import com.amazonaws.services.s3.model.*; import com.amazonaws.services.s3.transfer.TransferManager; -import com.amazonaws.services.s3.transfer.Upload; +import com.rbkmoney.file.storage.FileData; +import com.rbkmoney.file.storage.NewFileResult; +import com.rbkmoney.file.storage.contorller.UploadFileController; import com.rbkmoney.file.storage.service.exception.StorageException; +import com.rbkmoney.file.storage.util.DamselUtil; +import com.rbkmoney.geck.common.util.TypeUtil; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.codec.digest.DigestUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; -import org.springframework.web.multipart.MultipartFile; +import org.springframework.web.servlet.mvc.method.annotation.MvcUriComponentsBuilder; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; -import java.io.FileOutputStream; +import java.io.ByteArrayInputStream; +import java.io.FileNotFoundException; import java.io.IOException; -import java.io.OutputStream; +import java.io.InputStream; +import java.net.MalformedURLException; +import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; +import java.time.Instant; +import java.util.*; +import java.util.stream.Collectors; @Service @Slf4j public class AmazonS3StorageService implements StorageService { + private static final String EXPIRATION_TIME = "x-rbkmoney-file-expiration-time"; + private static final String FILE_UPLOADED = "x-rbkmoney-file-uploaded"; + private static final String FILEDATA_FILE_ID = "x-rbkmoney-filedata-file-id"; + private static final String FILEDATA_FILE_NAME = "x-rbkmoney-filedata-file-name"; + private static final String FILEDATA_CREATED_AT = "x-rbkmoney-filedata-created-at"; + private static final String FILEDATA_MD_5 = "x-rbkmoney-filedata-md5"; + private static final String FILEDATA_METADATA = "x-rbkmoney-filedata-metadata-"; + private final TransferManager transferManager; - private final AmazonS3 storageClient; + private final AmazonS3 s3Client; private final String bucketName; @Autowired public AmazonS3StorageService(TransferManager transferManager, @Value("${storage.bucketName}") String bucketName) { this.transferManager = transferManager; - this.storageClient = transferManager.getAmazonS3Client(); + this.s3Client = transferManager.getAmazonS3Client(); this.bucketName = bucketName; } @PostConstruct public void init() { - if (!storageClient.doesBucketExist(bucketName)) { + if (!s3Client.doesBucketExist(bucketName)) { log.info("Create bucket in file storage, bucketId='{}'", bucketName); - storageClient.createBucket(bucketName); + s3Client.createBucket(bucketName); } } @Override - public void store(String fileId, MultipartFile file) { - String filename = file.getOriginalFilename(); - log.info("Trying to upload file to storage, filename='{}', bucketId='{}'", filename, bucketName); + public FileData getFileData(String fileId) throws StorageException, FileNotFoundException { + S3Object s3Object = getS3Object(fileId); + checkFileStatus(s3Object); + return extractFileData(s3Object.getObjectMetadata()); + } + + @Override + public NewFileResult createNewFile(String fileName, Map metadata, Instant expirationTime) throws StorageException { + log.info("Trying to create new file to storage, filename='{}', bucketId='{}'", fileName, bucketName); try { - Path tempFile = createTempFile(file, filename); + // в хранилище сохраняется пустой файл + InputStream emptyContent = new ByteArrayInputStream(new byte[0]); - ObjectMetadata objectMetadata = new ObjectMetadata(); - objectMetadata.setContentDisposition("attachment;filename=" + filename); + String fileId = getFileId(); + String createdAt = Instant.now().toString(); - PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, fileId, tempFile.toFile()); - putObjectRequest.setMetadata(objectMetadata); - Upload upload = transferManager.upload(putObjectRequest); - try { - upload.waitForUploadResult(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } + FileData fileData = new FileData( + fileId, + fileName, + createdAt, + //todo + "", + metadata + ); - Files.deleteIfExists(tempFile); + writeFileToStorage(fileData, emptyContent, expirationTime); + + URL uploadUrl = createUploadUrl(fileId); log.info( - "File have been successfully uploaded, fileId='{}', bucketId='{}', filename='{}', md5='{}'", + "File have been successfully created, fileId='{}', bucketId='{}', filename='{}', md5='{}'", fileId, bucketName, - filename, - DigestUtils.md5Hex(Files.newInputStream(tempFile)) + fileName, + //todo + "" + ); + + return new NewFileResult(uploadUrl.toString(), fileData); + } catch (AmazonClientException ex) { + throw new StorageException( + String.format( + "Failed to create new file to storage, filename='%s', bucketId='%s'", + fileName, + bucketName + ), + ex ); - } catch (IOException | AmazonClientException ex) { - throw new StorageException(String.format("Failed to upload file to storage, filename='%s', bucketId='%s'", filename, bucketName), ex); } } - private Path createTempFile(MultipartFile file, String filename) throws IOException { - Path tempFile = Files.createTempFile(filename, ""); - OutputStream outputStream = new FileOutputStream(tempFile.toFile()); + @Override + public URL generateDownloadUrl(String fileId, Instant expirationTime) throws StorageException, FileNotFoundException { + checkFileStatus(getS3Object(fileId)); + return generatePresignedUrl(fileId, expirationTime, HttpMethod.GET); + } - int read; - byte[] bytes = new byte[1024]; + @Override + public void uploadFile(String fileId, InputStream inputStream) throws StorageException, IOException { + log.info("Trying to upload file to storage, filename='{}', bucketId='{}'", fileId, bucketName); - while ((read = file.getInputStream().read(bytes)) != -1) { - outputStream.write(bytes, 0, read); + try { + + //todo + Path testFile = Files.createTempFile("", "test_file"); + Files.write(testFile, "Test".getBytes()); + + S3Object object = getS3Object(fileId); + + checkFileStatus(object); + + object.getObjectMetadata().addUserMetadata(FILE_UPLOADED, "true"); +/* + //todo DEBUG + try { + // InputStream inputStream = file.getInputStream(); + Path testActualFile = Files.createTempFile("", "test_actual_file"); + Files.copy(inputStream, testActualFile, StandardCopyOption.REPLACE_EXISTING); + System.out.println("!!!!!!!"); + for (String readAllLine : Files.readAllLines(testActualFile)) { + System.out.println(readAllLine); + } + testActualFile = Files.createTempFile("", "test_actual_file"); + Files.copy(inputStream, testActualFile, StandardCopyOption.REPLACE_EXISTING); + System.out.println("!!!!!!!"); + for (String readAllLine : Files.readAllLines(testActualFile)) { + System.out.println(readAllLine); + } + } catch (Exception e) { + } + //todo +*/ + + PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, fileId, testFile.toFile()); + putObjectRequest.setMetadata(object.getObjectMetadata()); + s3Client.putObject(putObjectRequest); + +/* + PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, fileId, new S3ObjectInputStream( + new FileInputStream(testFile.toFile()),null + ),object.getObjectMetadata()); + s3Client.putObject(putObjectRequest); +*/ +/* + PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, fileId, new FileInputStream(testFile.toFile()),object.getObjectMetadata()); + s3Client.putObject(putObjectRequest); +*/ + +/* todo + PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, fileId, inputStream, object.getObjectMetadata()); + s3Client.putObject(putObjectRequest); +*/ + log.info( + "File have been successfully uploaded, fileId='{}', bucketId='{}'", + fileId, + bucketName + ); + + //todo + /* try { + S3Object object1 = s3Client.getObject(bucketName, fileId); + S3ObjectInputStream objectContent = object1.getObjectContent(); + Path testActualFile = Files.createTempFile("", "test_actual_file"); + Files.copy(objectContent, testActualFile, StandardCopyOption.REPLACE_EXISTING); + System.out.println("!!!!!!!"); + for (String readAllLine : Files.readAllLines(testActualFile)) { + System.out.println(readAllLine); + } + } catch (Exception e) { + } +*/ + //todo + } catch (AmazonClientException ex) { + throw new StorageException( + String.format( + "Failed to to upload file to storage, filename='%s', bucketId='%s'", + fileId, + bucketName + ), + ex + ); } - return tempFile; } @PreDestroy public void terminate() { transferManager.shutdownNow(true); } + + private S3Object getS3Object(String fileId) throws StorageException, FileNotFoundException { + try { + log.info( + "Trying to get file from storage, fileId='{}', bucketId='{}'", + fileId, + bucketName + ); + GetObjectRequest getObjectRequest = new GetObjectRequest(bucketName, fileId); + S3Object object = s3Client.getObject(getObjectRequest); + checkNullable(object, fileId, "File"); + log.info( + "File have been successfully got from storage, fileId='{}', bucketId='{}'", + fileId, + bucketName + ); + return object; + } catch (AmazonClientException ex) { + throw new StorageException( + String.format( + "Failed to get file from storage, fileId='%s', bucketId='%s'", + fileId, + bucketName + ), + ex + ); + } + } + + private void checkFileStatus(S3Object s3Object) throws StorageException { + log.info("Check file expiration and uploaded status: ETag='{}'", s3Object.getObjectMetadata().getETag()); + ObjectMetadata objectMetadata = s3Object.getObjectMetadata(); + + Boolean isUploaded = getBooleanFromObjectMetadata(objectMetadata); + if (isUploaded) { + log.info("File was uploaded: ETag='{}'", s3Object.getObjectMetadata().getETag()); + return; + } + + Date expirationTime = getDateFromObjectMetadata(objectMetadata); + Date time = new Date(); + if (time.getTime() < expirationTime.getTime()) { + log.info("File was uploaded: ETag='{}'", s3Object.getObjectMetadata().getETag()); + return; + } + + // если файл не соотвествует условиям, блокируем доступ к нему + throw new StorageException(String.format("File access error: fileId='%s', bucketId='%s', create a new file", s3Object.getKey(), bucketName)); + } + + private FileData extractFileData(ObjectMetadata objectMetadata) { + log.info("Trying to extract metadata from storage: ETag='{}'", objectMetadata.getETag()); + String fileId = getUserMetadataParameter(objectMetadata, FILEDATA_FILE_ID); + String fileName = getUserMetadataParameter(objectMetadata, FILEDATA_FILE_NAME); + String createdAt = getUserMetadataParameter(objectMetadata, FILEDATA_CREATED_AT); + String md5 = getUserMetadataParameter(objectMetadata, FILEDATA_MD_5); + + Map metadata = objectMetadata.getUserMetadata().entrySet().stream() + .filter(entry -> entry.getKey().startsWith(FILEDATA_METADATA) && entry.getValue() != null) + .collect( + Collectors.toMap( + o -> o.getKey().substring(FILEDATA_METADATA.length()), + o -> DamselUtil.fromJson(o.getValue(), com.rbkmoney.damsel.msgpack.Value.class) + ) + ); + log.info( + "Metadata have been successfully extracted from storage, fileId='{}', bucketId='{}'", + fileId, + bucketName + ); + return new FileData(fileId, fileName, createdAt, md5, metadata); + } + + private URL generatePresignedUrl(String fileId, Instant expirationTime, HttpMethod httpMethod) throws StorageException, FileNotFoundException { + try { + log.info( + "Trying to generate presigned url, fileId='{}', bucketId='{}', expirationTime='{}', httpMethod='{}'", + fileId, + bucketName, + expirationTime, + httpMethod + ); + + GeneratePresignedUrlRequest request = new GeneratePresignedUrlRequest(bucketName, fileId) + .withMethod(httpMethod) + .withExpiration(Date.from(expirationTime)); + URL url = s3Client.generatePresignedUrl(request); + checkNullable(url, fileId, "Presigned url"); + log.info( + "Presigned url have been successfully generated, url='{}', fileId='{}', bucketId='{}', expirationTime='{}', httpMethod='{}'", + url, + fileId, + bucketName, + expirationTime, + httpMethod + ); + return url; + } catch (AmazonClientException ex) { + throw new StorageException( + String.format( + "Failed to generate presigned url, fileId='%s', bucketId='%s', expirationTime='%s', httpMethod='%s'", + fileId, + bucketName, + expirationTime, + httpMethod + ), + ex + ); + } + } + + private void writeFileToStorage(FileData fileData, InputStream inputStream, Instant expirationTime) throws AmazonClientException { + PutObjectRequest request = createS3Request(fileData, inputStream, expirationTime); + s3Client.putObject(request); + } + + private PutObjectRequest createS3Request(FileData fileData, InputStream inputStream, Instant expirationTime) { + return new PutObjectRequest( + bucketName, + fileData.getFileId(), + inputStream, + createObjectMetadata(fileData, expirationTime) + ); + } + + private ObjectMetadata createObjectMetadata(FileData fileData, Instant expirationTime) { + ObjectMetadata objectMetadata = new ObjectMetadata(); + objectMetadata.setContentDisposition("attachment;filename=" + fileData.getFileName()); + // file parameters + objectMetadata.addUserMetadata(EXPIRATION_TIME, expirationTime.toString()); + objectMetadata.addUserMetadata(FILE_UPLOADED, "false"); + // filedata parameters + objectMetadata.addUserMetadata(FILEDATA_FILE_ID, fileData.getFileId()); + objectMetadata.addUserMetadata(FILEDATA_FILE_NAME, fileData.getFileName()); + objectMetadata.addUserMetadata(FILEDATA_CREATED_AT, fileData.getCreatedAt()); + objectMetadata.addUserMetadata(FILEDATA_MD_5, fileData.getMd5()); + fileData.getMetadata().forEach( + (key, value) -> objectMetadata.addUserMetadata(FILEDATA_METADATA + key, DamselUtil.toJsonString(value)) + ); + return objectMetadata; + } + + private URL createUploadUrl(String fileId) { + try { + return MvcUriComponentsBuilder.fromMethodName( + UploadFileController.class, + "handleFileUpload", + fileId, + null + ) + .buildAndExpand() + .encode() + .toUri() + .toURL(); + } catch (MalformedURLException e) { + throw new StorageException( + String.format( + "Exception createUploadUrl: fileId='%s', bucketId='%s', create a new file", + fileId, + bucketName + ), + e); + } + } + + private Boolean getBooleanFromObjectMetadata(ObjectMetadata objectMetadata) { + String isUploadedString = getUserMetadataParameter(objectMetadata, FILE_UPLOADED); + return Boolean.valueOf(isUploadedString); + } + + private Date getDateFromObjectMetadata(ObjectMetadata objectMetadata) throws StorageException { + String expirationTime = getUserMetadataParameter(objectMetadata, EXPIRATION_TIME); + return Date.from(TypeUtil.stringToInstant(expirationTime)); + } + + private String getUserMetadataParameter(ObjectMetadata objectMetadata, String key) throws StorageException { + return Optional.ofNullable(objectMetadata.getUserMetaDataOf(key)) + .orElseThrow(() -> new StorageException("Failed to extract user metadata parameter, " + key + " is null")); + } + + private String getFileId() { + String fileId; + do { + fileId = UUID.randomUUID().toString(); + } while (s3Client.doesObjectExist(bucketName, fileId)); + return fileId; + } + + private void checkNullable(Object object, String fileId, String objectType) throws FileNotFoundException { + if (Objects.isNull(object)) { + throw new FileNotFoundException(String.format(objectType + " is null, fileId='%s', bucketId='%s'", fileId, bucketName)); + } + } } \ No newline at end of file diff --git a/src/main/java/com/rbkmoney/file/storage/service/StorageService.java b/src/main/java/com/rbkmoney/file/storage/service/StorageService.java index 81189e5..41f22c3 100644 --- a/src/main/java/com/rbkmoney/file/storage/service/StorageService.java +++ b/src/main/java/com/rbkmoney/file/storage/service/StorageService.java @@ -1,8 +1,25 @@ package com.rbkmoney.file.storage.service; -import org.springframework.web.multipart.MultipartFile; +import com.rbkmoney.damsel.msgpack.Value; +import com.rbkmoney.file.storage.FileData; +import com.rbkmoney.file.storage.NewFileResult; +import com.rbkmoney.file.storage.service.exception.StorageException; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.time.Instant; +import java.util.Map; public interface StorageService { - void store(String fileId, MultipartFile file); + FileData getFileData(String fileId) throws StorageException, FileNotFoundException; + + NewFileResult createNewFile(String fileName, Map metadata, Instant expirationTime) throws StorageException; + + URL generateDownloadUrl(String fileId, Instant expirationTime) throws StorageException, FileNotFoundException; + + void uploadFile(String fileId, InputStream inputStream) throws StorageException, IOException; + } diff --git a/src/main/java/com/rbkmoney/file/storage/util/CheckerUtil.java b/src/main/java/com/rbkmoney/file/storage/util/CheckerUtil.java new file mode 100644 index 0000000..665be35 --- /dev/null +++ b/src/main/java/com/rbkmoney/file/storage/util/CheckerUtil.java @@ -0,0 +1,13 @@ +package com.rbkmoney.file.storage.util; + +import com.google.common.base.Strings; +import org.apache.thrift.TException; + +public class CheckerUtil { + + public static void checkString(String string, String exMessage) throws TException { + if (Strings.isNullOrEmpty(string)) { + throw new TException(exMessage); + } + } +} diff --git a/src/main/java/com/rbkmoney/file/storage/util/DamselUtil.java b/src/main/java/com/rbkmoney/file/storage/util/DamselUtil.java new file mode 100644 index 0000000..7ca2f89 --- /dev/null +++ b/src/main/java/com/rbkmoney/file/storage/util/DamselUtil.java @@ -0,0 +1,35 @@ +package com.rbkmoney.file.storage.util; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.rbkmoney.geck.serializer.kit.json.JsonHandler; +import com.rbkmoney.geck.serializer.kit.json.JsonProcessor; +import com.rbkmoney.geck.serializer.kit.tbase.TBaseHandler; +import com.rbkmoney.geck.serializer.kit.tbase.TBaseProcessor; +import org.apache.thrift.TBase; + +import java.io.IOException; + +public class DamselUtil { + + public static String toJsonString(TBase tBase) { + return toJson(tBase).toString(); + } + + public static JsonNode toJson(TBase tBase) { + try { + return new TBaseProcessor().process(tBase, new JsonHandler()); + } catch (IOException ex) { + throw new IllegalArgumentException(ex); + } + } + + public static T fromJson(String jsonString, Class type) { + try { + return new JsonProcessor().process(new ObjectMapper().readTree(jsonString), new TBaseHandler<>(type)); + } catch (IOException ex) { + throw new IllegalArgumentException(ex); + } + } + +} diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 9965c02..a2d41bd 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -5,3 +5,8 @@ info.stage=dev spring.servlet.multipart.max-file-size=128KB spring.servlet.multipart.max-request-size=128KB spring.servlet.multipart.enabled=true +storage.endpoint=localhost +storage.signingRegion=RU +storage.client.protocol=HTTP +storage.client.maxErrorRetry=10 +storage.bucketName="files" diff --git a/src/test/java/com/rbkmoney/file/storage/AbstractIntegrationTest.java b/src/test/java/com/rbkmoney/file/storage/AbstractIntegrationTest.java new file mode 100644 index 0000000..2ec0795 --- /dev/null +++ b/src/test/java/com/rbkmoney/file/storage/AbstractIntegrationTest.java @@ -0,0 +1,81 @@ +package com.rbkmoney.file.storage; + +import org.junit.runner.RunWith; +import org.springframework.boot.context.embedded.LocalServerPort; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.util.EnvironmentTestUtils; +import org.springframework.context.ApplicationContextInitializer; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringRunner; + +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; + +import static org.springframework.boot.test.context.SpringBootTest.WebEnvironment.RANDOM_PORT; + +@RunWith(SpringRunner.class) +@SpringBootTest(webEnvironment = RANDOM_PORT) +@ContextConfiguration(classes = FileStorageApplication.class, initializers = AbstractIntegrationTest.Initializer.class) +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) +public class AbstractIntegrationTest { + + private static final String SIGNING_REGION = "RU"; + private static final String AWS_ACCESS_KEY = "test"; + private static final String AWS_SECRET_KEY = "test"; + private static final String PROTOCOL = "HTTP"; + private static final String MAX_ERROR_RETRY = "10"; + private static final String BUCKET_NAME = "TEST"; + + @LocalServerPort + protected int port; + +// @ClassRule +// public static GenericContainer cephContainer = new GenericContainer("dr.rbkmoney.com/ceph-demo:latest") +// .withEnv("RGW_NAME", "localhost") +// .withEnv("NETWORK_AUTO_DETECT", "4") +// .withEnv("CEPH_DEMO_UID", "ceph-test") +// .withEnv("CEPH_DEMO_ACCESS_KEY", AWS_ACCESS_KEY) +// .withEnv("CEPH_DEMO_SECRET_KEY", AWS_SECRET_KEY) +// .withEnv("CEPH_DEMO_BUCKET", BUCKET_NAME) +// .withExposedPorts(5000, 80) +// .waitingFor( +// new HttpWaitStrategy() +// .forPath("/api/v0.1/health") +// .forStatusCode(200) +// .withStartupTimeout(Duration.ofMinutes(10)) +// ); + + public static class Initializer implements ApplicationContextInitializer { + @Override + public void initialize(ConfigurableApplicationContext configurableApplicationContext) { + EnvironmentTestUtils.addEnvironment( + "testcontainers", + configurableApplicationContext.getEnvironment(), +// "storage.endpoint=" + cephContainer.getContainerIpAddress() + ":" + cephContainer.getMappedPort(80), + "storage.endpoint=localhost:32827", + "storage.signingRegion=" + SIGNING_REGION, + "storage.accessKey=" + AWS_ACCESS_KEY, + "storage.secretKey=" + AWS_SECRET_KEY, + "storage.client.protocol=" + PROTOCOL, + "storage.client.maxErrorRetry=" + MAX_ERROR_RETRY, + "storage.bucketName=" + BUCKET_NAME + ); + } + } + + protected Instant getDayInstant() { + return LocalDateTime.now().plusDays(1).toInstant(getZoneOffset()); + } + + protected Instant getSecondInstant() { + return LocalDateTime.now().plusSeconds(1).toInstant(getZoneOffset()); + } + + private ZoneOffset getZoneOffset() { + return ZoneOffset.systemDefault().getRules().getOffset(LocalDateTime.now()); + } + +} diff --git a/src/test/java/com/rbkmoney/file/storage/service/Aasd.java b/src/test/java/com/rbkmoney/file/storage/service/Aasd.java new file mode 100644 index 0000000..e2bef4d --- /dev/null +++ b/src/test/java/com/rbkmoney/file/storage/service/Aasd.java @@ -0,0 +1,167 @@ +package com.rbkmoney.file.storage.service; + +import com.rbkmoney.file.storage.AbstractIntegrationTest; +import com.rbkmoney.file.storage.FileData; +import com.rbkmoney.file.storage.FileStorageSrv; +import com.rbkmoney.file.storage.NewFileResult; +import com.rbkmoney.woody.api.flow.error.WRuntimeException; +import com.rbkmoney.woody.thrift.impl.http.THSpawnClientBuilder; +import org.apache.thrift.TException; +import org.junit.Before; +import org.junit.Test; +import org.springframework.core.io.FileSystemResource; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; +import org.springframework.web.client.RestTemplate; + +import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.Collections; + +import static org.junit.Assert.assertEquals; + +public class Aasd extends AbstractIntegrationTest { + + private static final int TIMEOUT = 555000; + + private FileStorageSrv.Iface client; + + @Before + public void before() throws URISyntaxException { + client = new THSpawnClientBuilder() + .withAddress(new URI("http://localhost:" + port + "/file_storage")) + .withNetworkTimeout(TIMEOUT) + .build(FileStorageSrv.Iface.class); + } + + @Test + public void extractMetadataTest() throws TException { + NewFileResult testFile = client.createNewFile("test_file", Collections.emptyMap(), getDayInstant().toString()); + FileData fileData = client.getFileData(testFile.getFileData().getFileId()); + + assertEquals(fileData, testFile.getFileData()); + } + + @Test(expected = WRuntimeException.class) + public void expiredTimeForFileDataInMetadataTest() throws TException, InterruptedException { + NewFileResult testFile = client.createNewFile("test_file", Collections.emptyMap(), getSecondInstant().toString()); + Thread.sleep(1000); + client.getFileData(testFile.getFileData().getFileId()); + } + + @Test(expected = WRuntimeException.class) + public void expiredTimeForGenerateUrlInMetadataTest() throws TException, InterruptedException { + NewFileResult testFile = client.createNewFile("test_file", Collections.emptyMap(), getSecondInstant().toString()); + Thread.sleep(1000); + client.generateDownloadUrl(testFile.getFileData().getFileId(), getSecondInstant().toString()); + } + + @Test + public void downloadUrlTest() throws TException, IOException { + Path testFile = Files.createTempFile("", "test_file"); + Files.write(testFile, new byte[0]); + + Path testActualFile = Files.createTempFile("", "test_actual_file"); + + NewFileResult fileResult = client.createNewFile("test_file", Collections.emptyMap(), getDayInstant().toString()); + String s = client.generateDownloadUrl(fileResult.getFileData().getFileId(), getDayInstant().toString()); + + URL url = new URL(s); + + assertEquals(HttpStatus.FORBIDDEN.value(), getHttpURLConnection(url, true, "PUT").getResponseCode()); + + assertEquals(HttpStatus.OK.value(), getHttpURLConnection(url, false, "GET").getResponseCode()); + + HttpURLConnection urlConnection = getHttpURLConnection(url, false, "GET"); + InputStream inputStream = urlConnection.getInputStream(); + + Files.copy(inputStream, testActualFile, StandardCopyOption.REPLACE_EXISTING); + + assertEquals(Files.readAllLines(testFile), Files.readAllLines(testActualFile)); + } + + @Test + public void expiredTimeForGenerateUrlConnectionInCephTest() throws TException, IOException, InterruptedException { + NewFileResult fileResult = client.createNewFile("test_file", Collections.emptyMap(), getDayInstant().toString()); + String s = client.generateDownloadUrl(fileResult.getFileData().getFileId(), getSecondInstant().toString()); + + URL url = new URL(s); + + assertEquals(HttpStatus.OK.value(), getHttpURLConnection(url, false, "GET").getResponseCode()); + + Thread.sleep(2000); + + assertEquals(HttpStatus.FORBIDDEN.value(), getHttpURLConnection(url, false, "GET").getResponseCode()); + } + + @Test + public void name() throws IOException, TException { + NewFileResult fileResult = client.createNewFile("test_file", Collections.emptyMap(), getDayInstant().toString()); + String s = fileResult.getUploadUrl(); + + Path testFile = Files.createTempFile("", "test_file"); + Files.write(testFile, "Test".getBytes()); + + MultiValueMap body = new LinkedMultiValueMap<>(); + body.add("file", new FileSystemResource(testFile.toFile())); + + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.MULTIPART_FORM_DATA); + + HttpEntity> requestEntity = new HttpEntity<>(body, headers); + + RestTemplate restTemplate = new RestTemplate(); + restTemplate.postForEntity(s, requestEntity, Void.class); + + + Path testActualFile = Files.createTempFile("", "test_actual_file"); + + String urs = client.generateDownloadUrl(fileResult.getFileData().getFileId(), getDayInstant().toString()); + + URL url = new URL(urs); + + HttpURLConnection urlConnection = getHttpURLConnection(url, false, "GET"); + InputStream inputStream = urlConnection.getInputStream(); + + Files.copy(inputStream, testActualFile, StandardCopyOption.REPLACE_EXISTING); + + assertEquals(Files.readAllLines(testFile), Files.readAllLines(testActualFile)); + } + + private void asd() { +/* + SimpleClientHttpRequestFactory requestFactory = new SimpleClientHttpRequestFactory(); + requestFactory.setBufferRequestBody(false); + + InputStream fis = new FileInputStream(testFile.toFile()); + + RequestCallback requestCallback = request -> { + request.getHeaders().add("Content-type", "application/octet-stream"); + IOUtils.copy(fis, request.getBody()); + }; + HttpMessageConverterExtractor responseExtractor = + new HttpMessageConverterExtractor<>(String.class, restTemplate.getMessageConverters()); + restTemplate.setRequestFactory(requestFactory); +*/ + + + } + + private HttpURLConnection getHttpURLConnection(URL url, boolean doOutput, String method) throws IOException { + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + connection.setDoOutput(doOutput); + connection.setRequestMethod(method); + return connection; + } +} diff --git a/src/test/java/com/rbkmoney/file/storage/service/MetadataS3ObjectsTest.java b/src/test/java/com/rbkmoney/file/storage/service/MetadataS3ObjectsTest.java new file mode 100644 index 0000000..e9d29d9 --- /dev/null +++ b/src/test/java/com/rbkmoney/file/storage/service/MetadataS3ObjectsTest.java @@ -0,0 +1,20 @@ +package com.rbkmoney.file.storage.service; + +import com.rbkmoney.file.storage.AbstractIntegrationTest; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; + +import java.io.FileNotFoundException; + +public class MetadataS3ObjectsTest extends AbstractIntegrationTest { + + @Autowired + private StorageService storageService; + + @Test + public void name() throws FileNotFoundException { +// FileData fileData = storageService.createNewFile("test_file", Collections.emptyMap(), getInstant()); +// URL url = storageService.generateDownloadUrl(fileData.getFileId(), getInstant()); + + } +} diff --git a/src/test/java/com/rbkmoney/file/storage/service/PresignedUrlAccessRightsTest.java b/src/test/java/com/rbkmoney/file/storage/service/PresignedUrlAccessRightsTest.java new file mode 100644 index 0000000..50babed --- /dev/null +++ b/src/test/java/com/rbkmoney/file/storage/service/PresignedUrlAccessRightsTest.java @@ -0,0 +1,111 @@ +package com.rbkmoney.file.storage.service; + +import com.rbkmoney.file.storage.AbstractIntegrationTest; +import com.rbkmoney.file.storage.FileData; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; + +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URL; +import java.nio.file.Path; + +public class PresignedUrlAccessRightsTest extends AbstractIntegrationTest { + + @Autowired + private StorageService storageService; + + @Test + public void urlForUploadTest() throws IOException { + /* String stringForWrite = "This text uploaded as an object via presigned URL."; + + Path testFile = Files.createTempFile("", "test_file"); + + Path testActualFile = Files.createTempFile("", "test_actual_file"); + + try { + FileData fileData = storageService.createNewFile("test_file", Collections.emptyMap(), getInstant()); + + // генерация url с доступом только для выгрузки + URL url = storageService.generateUploadUrl(fileData.getFileId(), getInstant()); + + // ошибка при запросе по url методом get + assertEquals(HttpStatus.FORBIDDEN.value(), getHttpURLConnection(url, false, "GET").getResponseCode()); + + // Length Required при запросе по url методом put + assertEquals(HttpStatus.LENGTH_REQUIRED.value(), getHttpURLConnection(url, true, "PUT").getResponseCode()); + + // запись данных методом put + HttpURLConnection urlConnection = getHttpURLConnection(url, true, "PUT"); + + OutputStreamWriter out = new OutputStreamWriter(urlConnection.getOutputStream()); + out.write(stringForWrite); + out.close(); + + // чтобы завершить загрузку вызываем getResponseCode + assertEquals(HttpStatus.OK.value(), urlConnection.getResponseCode()); + + // файл перезаписывается и затирает метаданные. + // запись метаданных +// storageService.rewriteFileData(fileData, getInstant()); + + copyFromStorageToFile(fileData, testActualFile); + + // testFile пустой + assertNotEquals(Files.readAllLines(testFile), Files.readAllLines(testActualFile)); + + Files.write(testFile, stringForWrite.getBytes()); + assertEquals(Files.readAllLines(testFile), Files.readAllLines(testActualFile)); + } finally { + Files.deleteIfExists(testFile); + Files.deleteIfExists(testActualFile); + }*/ + } + + @Test + public void urlForDownloadTest() throws IOException { + /*Path testFile = Files.createTempFile("", "test_file"); + Files.write(testFile, "4815162342".getBytes()); + + Path testActualFile = Files.createTempFile("", "test_actual_file"); + + try { + FileData fileData = storageService.createNewFile("test_file", Collections.emptyMap(), getInstant()); + + // запись тестового файла в цеф + storageService.uploadFile(fileData.getFileId(), testFile); + + // генерация url с доступом только для загрузки + URL url = storageService.generateDownloadUrl(fileData.getFileId(), getInstant()); + + // ошибка при запросе по url методом put + assertEquals(HttpStatus.FORBIDDEN.value(), getHttpURLConnection(url, true, "PUT").getResponseCode()); + + // ок при запросе по url методом get + assertEquals(HttpStatus.OK.value(), getHttpURLConnection(url, false, "GET").getResponseCode()); + + // получение содержимого методом get + HttpURLConnection urlConnection = getHttpURLConnection(url, false, "GET"); + InputStream inputStream = urlConnection.getInputStream(); + + Files.copy(inputStream, testActualFile, StandardCopyOption.REPLACE_EXISTING); + + assertEquals(Files.readAllLines(testFile), Files.readAllLines(testActualFile)); + } finally { + Files.deleteIfExists(testFile); + Files.deleteIfExists(testActualFile); + }*/ + } + + private void copyFromStorageToFile(FileData fileData, Path file) throws IOException { + /*InputStream inputStream = storageService.getFile(fileData.getFileId()); + Files.copy(inputStream, file, StandardCopyOption.REPLACE_EXISTING);*/ + } + + private HttpURLConnection getHttpURLConnection(URL url, boolean doOutput, String method) throws IOException { + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + connection.setDoOutput(doOutput); + connection.setRequestMethod(method); + return connection; + } +} diff --git a/src/test/resources/logback-test.xml b/src/test/resources/logback-test.xml new file mode 100644 index 0000000..31c18bc --- /dev/null +++ b/src/test/resources/logback-test.xml @@ -0,0 +1,30 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +