From 8bd10b427be7cf2278761631b198acf1419a8e9c Mon Sep 17 00:00:00 2001 From: "a.karlov" Date: Fri, 23 Nov 2018 15:23:28 +0300 Subject: [PATCH] BJ-314: init --- .gitignore | 73 ++++++++++++++ .gitmodules | 4 + Jenkinsfile | 34 +++++++ Makefile | 80 +++++++++++++++ build_utils | 1 + pom.xml | 81 +++++++++++++++ .../file/storage/FileStorageApplication.java | 12 +++ .../file/storage/config/StorageConfig.java | 78 +++++++++++++++ .../storage/config/SwaggerConfiguration.java | 35 +++++++ .../storage/contorller/FileController.java | 34 +++++++ .../service/AmazonS3StorageService.java | 99 +++++++++++++++++++ .../file/storage/service/StorageService.java | 8 ++ .../service/exception/StorageException.java | 12 +++ .../StorageFileNotFoundException.java | 12 +++ src/main/resources/application.properties | 7 ++ .../storage/FileStorageApplicationTests.java | 16 +++ 16 files changed, 586 insertions(+) create mode 100644 .gitignore create mode 100644 .gitmodules create mode 100644 Jenkinsfile create mode 100644 Makefile create mode 160000 build_utils create mode 100644 pom.xml create mode 100644 src/main/java/com/rbkmoney/file/storage/FileStorageApplication.java create mode 100644 src/main/java/com/rbkmoney/file/storage/config/StorageConfig.java create mode 100644 src/main/java/com/rbkmoney/file/storage/config/SwaggerConfiguration.java create mode 100644 src/main/java/com/rbkmoney/file/storage/contorller/FileController.java create mode 100644 src/main/java/com/rbkmoney/file/storage/service/AmazonS3StorageService.java create mode 100644 src/main/java/com/rbkmoney/file/storage/service/StorageService.java create mode 100644 src/main/java/com/rbkmoney/file/storage/service/exception/StorageException.java create mode 100644 src/main/java/com/rbkmoney/file/storage/service/exception/StorageFileNotFoundException.java create mode 100644 src/main/resources/application.properties create mode 100644 src/test/java/com/rbkmoney/file/storage/FileStorageApplicationTests.java diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ddbf952 --- /dev/null +++ b/.gitignore @@ -0,0 +1,73 @@ +# Created by .ignore support plugin (hsz.mobi) +### Maven template +target/ +pom.xml.tag +pom.xml.releaseBackup +pom.xml.versionsBackup +pom.xml.next +release.properties +dependency-reduced-pom.xml +buildNumber.properties +.mvn/timing.properties +### JetBrains template +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff: +.idea/ +.idea/workspace.xml +.idea/tasks.xml +.idea/dictionaries +.idea/vcs.xml +.idea/jsLibraryMappings.xml + +# Sensitive or high-churn files: +.idea/dataSources.ids +.idea/dataSources.xml +.idea/dataSources.local.xml +.idea/sqlDataSources.xml +.idea/dynamic.xml +.idea/uiDesigner.xml + +# Gradle: +.idea/gradle.xml +.idea/libraries + +# Mongo Explorer plugin: +.idea/mongoSettings.xml + +## File-based project format: +*.iws +*.ipr +*.iml + +## Plugin-specific files: + +# IntelliJ +/out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties +### Java template +*.class + +# Mobile Tools for Java (J2ME) +.mtj.tmp/ + +# Package Files # +*.jar +*.war +*.ear + +# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml +hs_err_pid* +env.list diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..ca5a761 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,4 @@ +[submodule "build_utils"] + path = build_utils + url = git@github.com:rbkmoney/build_utils.git + branch = master diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 0000000..b2dc3a7 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,34 @@ +#!groovy +// -*- mode: groovy -*- + +build('file-storage', 'docker-host') { + checkoutRepo() + loadBuildUtils() + + def pipeDefault + def gitUtils + runStage('load pipeline') { + env.JENKINS_LIB = "build_utils/jenkins_lib" + pipeDefault = load("${env.JENKINS_LIB}/pipeDefault.groovy") + gitUtils = load("${env.JENKINS_LIB}/gitUtils.groovy") + } + + pipeDefault() { + + runStage('compile') { + sh "make wc_compile" + } + + // Java + runStage('Execute build container') { + withCredentials([[$class: 'FileBinding', credentialsId: 'java-maven-settings.xml', variable: 'SETTINGS_XML']]) { + if (env.BRANCH_NAME == 'master' || env.BRANCH_NAME.startsWith('epic/')) { + sh 'make SETTINGS_XML=${SETTINGS_XML} BRANCH_NAME=${BRANCH_NAME} wc_java.deploy' + } else { + sh 'make SETTINGS_XML=${SETTINGS_XML} wc_java.compile' + } + } + } + + } +} diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..3e3e327 --- /dev/null +++ b/Makefile @@ -0,0 +1,80 @@ +THRIFT = $(or $(shell which thrift), $(error "`thrift' executable missing")) +REBAR = $(shell which rebar3 2>/dev/null || which ./rebar3) +SUBMODULES = build_utils +SUBTARGETS = $(patsubst %,%/.git,$(SUBMODULES)) + +UTILS_PATH := build_utils +TEMPLATES_PATH := . + +# Name of the service +SERVICE_NAME := file_storage + +# Build image tag to be used +BUILD_IMAGE_TAG := 55e987e74e9457191a5b4a7c5dc9e3838ae82d2b +CALL_ANYWHERE := \ + all submodules compile clean distclean \ + java.compile java.deploy + +CALL_W_CONTAINER := $(CALL_ANYWHERE) + +all: compile + +-include $(UTILS_PATH)/make_lib/utils_container.mk + +.PHONY: $(CALL_W_CONTAINER) + +# CALL_ANYWHERE +$(SUBTARGETS): %/.git: % + git submodule update --init $< + touch $@ + +submodules: $(SUBTARGETS) + +compile: + $(REBAR) compile + +clean: + $(REBAR) clean + +distclean: + $(REBAR) clean -a + rm -rfv _build + +# Java + +ifdef SETTINGS_XML +DOCKER_RUN_OPTS = -v $(SETTINGS_XML):$(SETTINGS_XML) +DOCKER_RUN_OPTS += -e SETTINGS_XML=$(SETTINGS_XML) +endif + +ifdef LOCAL_BUILD +DOCKER_RUN_OPTS += -v $$HOME/.m2:/home/$(UNAME)/.m2:rw +endif + +COMMIT_HASH := $(shell git --no-pager log -1 --pretty=format:"%h") +NUMBER_COMMITS := $(shell git rev-list --count HEAD) + +JAVA_PKG_VERSION := 1.$(NUMBER_COMMITS)-$(COMMIT_HASH) + +ifdef BRANCH_NAME +ifeq "$(findstring epic,$(BRANCH_NAME))" "epic" +JAVA_PKG_VERSION := $(JAVA_PKG_VERSION)-epic +endif +endif + +MVN = mvn -s $(SETTINGS_XML) -Dpath_to_thrift="$(THRIFT)" -Dcommit.number="$(NUMBER_COMMITS)" + +java.compile: java.settings + $(MVN) compile + +java.deploy: java.settings + $(MVN) versions:set versions:commit -DnewVersion="$(JAVA_PKG_VERSION)" && \ + $(MVN) deploy + +java.install: java.settings + $(MVN) clean && \ + $(MVN) versions:set versions:commit -DnewVersion="$(JAVA_PKG_VERSION)" && \ + $(MVN) install + +java.settings: + $(if $(SETTINGS_XML),, echo "SETTINGS_XML not defined"; exit 1) \ No newline at end of file diff --git a/build_utils b/build_utils new file mode 160000 index 0000000..9b66408 --- /dev/null +++ b/build_utils @@ -0,0 +1 @@ +Subproject commit 9b664082ddc8ec8cdfbe7513d54e433d24198cc2 diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..023a970 --- /dev/null +++ b/pom.xml @@ -0,0 +1,81 @@ + + + 4.0.0 + + + com.rbkmoney + spring-boot-starter-parent + 1.5.2.RELEASE + + + file-storage + 0.0.1-SNAPSHOT + jar + + file-storage + Service for uploading & downloading files + + + rbk.money + 8022 + UTF-8 + UTF-8 + 1.8 + + 2.8.0 + + + + + + org.springframework.boot + spring-boot-starter-web + + + org.springframework.boot + spring-boot-configuration-processor + true + + + + + com.amazonaws + aws-java-sdk-s3 + 1.11.160 + + + io.springfox + springfox-swagger2 + ${springfox-swagger2.version} + + + io.springfox + springfox-swagger-ui + ${springfox-swagger2.version} + + + org.projectlombok + lombok + 1.18.4 + provided + + + + + org.springframework.boot + spring-boot-starter-test + test + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + + diff --git a/src/main/java/com/rbkmoney/file/storage/FileStorageApplication.java b/src/main/java/com/rbkmoney/file/storage/FileStorageApplication.java new file mode 100644 index 0000000..1fb510d --- /dev/null +++ b/src/main/java/com/rbkmoney/file/storage/FileStorageApplication.java @@ -0,0 +1,12 @@ +package com.rbkmoney.file.storage; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class FileStorageApplication { + + public static void main(String[] args) { + SpringApplication.run(FileStorageApplication.class, args); + } +} diff --git a/src/main/java/com/rbkmoney/file/storage/config/StorageConfig.java b/src/main/java/com/rbkmoney/file/storage/config/StorageConfig.java new file mode 100644 index 0000000..cfb22e7 --- /dev/null +++ b/src/main/java/com/rbkmoney/file/storage/config/StorageConfig.java @@ -0,0 +1,78 @@ +package com.rbkmoney.file.storage.config; + +import com.amazonaws.ClientConfiguration; +import com.amazonaws.Protocol; +import com.amazonaws.auth.AWSCredentialsProviderChain; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.auth.EnvironmentVariableCredentialsProvider; +import com.amazonaws.client.builder.AwsClientBuilder; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.amazonaws.services.s3.transfer.TransferManager; +import com.amazonaws.services.s3.transfer.TransferManagerBuilder; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class StorageConfig { + + @Value("${storage.endpoint}") + private String endpoint; + + @Value("${storage.signingRegion}") + private String signingRegion; + + @Value("${storage.accessKey:}") + private String accessKey; + + @Value("${storage.secretKey:}") + private String secretKey; + + @Value("${storage.client.protocol:HTTP}") + private Protocol protocol; + + @Value("${storage.client.maxErrorRetry}") + private int maxErrorRetry; + + @Bean + public AmazonS3 storageClient(AWSCredentialsProviderChain credentialsProviderChain, ClientConfiguration clientConfiguration) { + return AmazonS3ClientBuilder.standard() + .withCredentials(credentialsProviderChain) + .withPathStyleAccessEnabled(true) + .withEndpointConfiguration( + new AwsClientBuilder.EndpointConfiguration(endpoint, signingRegion) + ) + .withClientConfiguration(clientConfiguration) + .build(); + } + + @Bean + public AWSCredentialsProviderChain credentialsProviderChain() { + return new AWSCredentialsProviderChain( + new EnvironmentVariableCredentialsProvider(), + new AWSStaticCredentialsProvider( + new BasicAWSCredentials( + accessKey, + secretKey + ) + ) + ); + } + + @Bean + public ClientConfiguration clientConfiguration() { + return new ClientConfiguration() + .withProtocol(protocol) + .withSignerOverride("S3SignerType") + .withMaxErrorRetry(maxErrorRetry); + } + + @Bean + public TransferManager transferManager(AmazonS3 s3Client) { + return TransferManagerBuilder.standard() + .withS3Client(s3Client) + .build(); + } +} diff --git a/src/main/java/com/rbkmoney/file/storage/config/SwaggerConfiguration.java b/src/main/java/com/rbkmoney/file/storage/config/SwaggerConfiguration.java new file mode 100644 index 0000000..6c7031b --- /dev/null +++ b/src/main/java/com/rbkmoney/file/storage/config/SwaggerConfiguration.java @@ -0,0 +1,35 @@ +package com.rbkmoney.file.storage.config; + +import com.google.common.base.Predicates; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import springfox.documentation.builders.ApiInfoBuilder; +import springfox.documentation.builders.PathSelectors; +import springfox.documentation.service.ApiInfo; +import springfox.documentation.service.Contact; +import springfox.documentation.spi.DocumentationType; +import springfox.documentation.spring.web.plugins.Docket; +import springfox.documentation.swagger2.annotations.EnableSwagger2; + +@Configuration +@EnableSwagger2 +public class SwaggerConfiguration { + + @Bean + public Docket productApi() { + return new Docket(DocumentationType.SWAGGER_2) + .apiInfo(metaData()) + .select() + .paths(Predicates.not(PathSelectors.regex("/error"))) + .build(); + } + + private ApiInfo metaData() { + return new ApiInfoBuilder() + .title("REST endpoint") + .description("\"Endpoint для выгрузки документов на сервер\"") + .version("0.0.1-SNAPSHOT") + .contact(new Contact("RBK.money", "https://github.com/rbkmoney", "support@rbkmoney.com")) + .build(); + } +} diff --git a/src/main/java/com/rbkmoney/file/storage/contorller/FileController.java b/src/main/java/com/rbkmoney/file/storage/contorller/FileController.java new file mode 100644 index 0000000..cc62da3 --- /dev/null +++ b/src/main/java/com/rbkmoney/file/storage/contorller/FileController.java @@ -0,0 +1,34 @@ +package com.rbkmoney.file.storage.contorller; + +import com.rbkmoney.file.storage.service.StorageService; +import io.swagger.annotations.*; +import lombok.RequiredArgsConstructor; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.multipart.MultipartFile; + +@RestController +@Api(description = "Api для операций с файлами") +@RequiredArgsConstructor +public class FileController { + + private final StorageService storageService; + + @PostMapping("/upload") + @ApiOperation(value = "Выгрузить файл на сервер") + @ApiResponses(value = { + @ApiResponse(code = 200, message = "Файл выгружен на сервер") + }) + public ResponseEntity handleFileUpload(@ApiParam(value = "выгружаемый файл", required = true) + @RequestParam(value = "file") + MultipartFile file, + @ApiParam(value = "id файла", required = true) + @RequestParam(value = "file_id") + String fileId) { + storageService.store(fileId, file); + return new ResponseEntity(HttpStatus.OK); + } +} diff --git a/src/main/java/com/rbkmoney/file/storage/service/AmazonS3StorageService.java b/src/main/java/com/rbkmoney/file/storage/service/AmazonS3StorageService.java new file mode 100644 index 0000000..083862e --- /dev/null +++ b/src/main/java/com/rbkmoney/file/storage/service/AmazonS3StorageService.java @@ -0,0 +1,99 @@ +package com.rbkmoney.file.storage.service; + +import com.amazonaws.AmazonClientException; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.model.ObjectMetadata; +import com.amazonaws.services.s3.model.PutObjectRequest; +import com.amazonaws.services.s3.transfer.TransferManager; +import com.amazonaws.services.s3.transfer.Upload; +import com.rbkmoney.file.storage.service.exception.StorageException; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.codec.digest.DigestUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.springframework.web.multipart.MultipartFile; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; + +@Service +@Slf4j +public class AmazonS3StorageService implements StorageService { + + private final TransferManager transferManager; + private final AmazonS3 storageClient; + private final String bucketName; + + @Autowired + public AmazonS3StorageService(TransferManager transferManager, @Value("${storage.bucketName}") String bucketName) { + this.transferManager = transferManager; + this.storageClient = transferManager.getAmazonS3Client(); + this.bucketName = bucketName; + } + + @PostConstruct + public void init() { + if (!storageClient.doesBucketExist(bucketName)) { + log.info("Create bucket in file storage, bucketId='{}'", bucketName); + storageClient.createBucket(bucketName); + } + } + + @Override + public void store(String fileId, MultipartFile file) { + String filename = file.getOriginalFilename(); + log.info("Trying to upload file to storage, filename='{}', bucketId='{}'", filename, bucketName); + + try { + Path tempFile = createTempFile(file, filename); + + ObjectMetadata objectMetadata = new ObjectMetadata(); + objectMetadata.setContentDisposition("attachment;filename=" + filename); + + PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, fileId, tempFile.toFile()); + putObjectRequest.setMetadata(objectMetadata); + Upload upload = transferManager.upload(putObjectRequest); + try { + upload.waitForUploadResult(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + + Files.deleteIfExists(tempFile); + + log.info( + "File have been successfully uploaded, fileId='{}', bucketId='{}', filename='{}', md5='{}'", + fileId, + bucketName, + filename, + DigestUtils.md5Hex(Files.newInputStream(tempFile)) + ); + } catch (IOException | AmazonClientException ex) { + throw new StorageException(String.format("Failed to upload file to storage, filename='%s', bucketId='%s'", filename, bucketName), ex); + } + } + + private Path createTempFile(MultipartFile file, String filename) throws IOException { + Path tempFile = Files.createTempFile(filename, ""); + OutputStream outputStream = new FileOutputStream(tempFile.toFile()); + + int read; + byte[] bytes = new byte[1024]; + + while ((read = file.getInputStream().read(bytes)) != -1) { + outputStream.write(bytes, 0, read); + } + return tempFile; + } + + @PreDestroy + public void terminate() { + transferManager.shutdownNow(true); + } +} \ No newline at end of file diff --git a/src/main/java/com/rbkmoney/file/storage/service/StorageService.java b/src/main/java/com/rbkmoney/file/storage/service/StorageService.java new file mode 100644 index 0000000..81189e5 --- /dev/null +++ b/src/main/java/com/rbkmoney/file/storage/service/StorageService.java @@ -0,0 +1,8 @@ +package com.rbkmoney.file.storage.service; + +import org.springframework.web.multipart.MultipartFile; + +public interface StorageService { + + void store(String fileId, MultipartFile file); +} diff --git a/src/main/java/com/rbkmoney/file/storage/service/exception/StorageException.java b/src/main/java/com/rbkmoney/file/storage/service/exception/StorageException.java new file mode 100644 index 0000000..80dd654 --- /dev/null +++ b/src/main/java/com/rbkmoney/file/storage/service/exception/StorageException.java @@ -0,0 +1,12 @@ +package com.rbkmoney.file.storage.service.exception; + +public class StorageException extends RuntimeException { + + public StorageException(String message) { + super(message); + } + + public StorageException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/src/main/java/com/rbkmoney/file/storage/service/exception/StorageFileNotFoundException.java b/src/main/java/com/rbkmoney/file/storage/service/exception/StorageFileNotFoundException.java new file mode 100644 index 0000000..4d9e371 --- /dev/null +++ b/src/main/java/com/rbkmoney/file/storage/service/exception/StorageFileNotFoundException.java @@ -0,0 +1,12 @@ +package com.rbkmoney.file.storage.service.exception; + +public class StorageFileNotFoundException extends StorageException { + + public StorageFileNotFoundException(String message) { + super(message); + } + + public StorageFileNotFoundException(String message, Throwable cause) { + super(message, cause); + } +} \ No newline at end of file diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties new file mode 100644 index 0000000..9965c02 --- /dev/null +++ b/src/main/resources/application.properties @@ -0,0 +1,7 @@ +server.port=@server.port@ +spring.application.name=@project.name@ +info.version=@project.version@ +info.stage=dev +spring.servlet.multipart.max-file-size=128KB +spring.servlet.multipart.max-request-size=128KB +spring.servlet.multipart.enabled=true diff --git a/src/test/java/com/rbkmoney/file/storage/FileStorageApplicationTests.java b/src/test/java/com/rbkmoney/file/storage/FileStorageApplicationTests.java new file mode 100644 index 0000000..f3847c5 --- /dev/null +++ b/src/test/java/com/rbkmoney/file/storage/FileStorageApplicationTests.java @@ -0,0 +1,16 @@ +package com.rbkmoney.file.storage; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.junit4.SpringRunner; + +@RunWith(SpringRunner.class) +@SpringBootTest +public class FileStorageApplicationTests { + + @Test + public void contextLoads() { + } + +}