BJ-314: init

.gitignore

@@ -0,0 +1,73 @@
+# Created by .ignore support plugin (hsz.mobi)
+### Maven template
+target/
+pom.xml.tag
+pom.xml.releaseBackup
+pom.xml.versionsBackup
+pom.xml.next
+release.properties
+dependency-reduced-pom.xml
+buildNumber.properties
+.mvn/timing.properties
+### JetBrains template
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm
+# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
+
+# User-specific stuff:
+.idea/
+.idea/workspace.xml
+.idea/tasks.xml
+.idea/dictionaries
+.idea/vcs.xml
+.idea/jsLibraryMappings.xml
+
+# Sensitive or high-churn files:
+.idea/dataSources.ids
+.idea/dataSources.xml
+.idea/dataSources.local.xml
+.idea/sqlDataSources.xml
+.idea/dynamic.xml
+.idea/uiDesigner.xml
+
+# Gradle:
+.idea/gradle.xml
+.idea/libraries
+
+# Mongo Explorer plugin:
+.idea/mongoSettings.xml
+
+## File-based project format:
+*.iws
+*.ipr
+*.iml
+
+## Plugin-specific files:
+
+# IntelliJ
+/out/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+fabric.properties
+### Java template
+*.class
+
+# Mobile Tools for Java (J2ME)
+.mtj.tmp/
+
+# Package Files #
+*.jar
+*.war
+*.ear
+
+# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
+hs_err_pid*
+env.list

.gitmodules

@@ -0,0 +1,4 @@
+[submodule "build_utils"]
+	path = build_utils
+	url = git@github.com:rbkmoney/build_utils.git
+	branch = master

Jenkinsfile

@@ -0,0 +1,34 @@
+#!groovy
+// -*- mode: groovy -*-
+
+build('file-storage', 'docker-host') {
+    checkoutRepo()
+    loadBuildUtils()
+
+    def pipeDefault
+    def gitUtils
+    runStage('load pipeline') {
+        env.JENKINS_LIB = "build_utils/jenkins_lib"
+        pipeDefault = load("${env.JENKINS_LIB}/pipeDefault.groovy")
+        gitUtils = load("${env.JENKINS_LIB}/gitUtils.groovy")
+    }
+
+    pipeDefault() {
+
+        runStage('compile') {
+            sh "make wc_compile"
+        }
+
+        // Java
+        runStage('Execute build container') {
+            withCredentials([[$class: 'FileBinding', credentialsId: 'java-maven-settings.xml', variable: 'SETTINGS_XML']]) {
+                if (env.BRANCH_NAME == 'master' || env.BRANCH_NAME.startsWith('epic/')) {
+                    sh 'make SETTINGS_XML=${SETTINGS_XML} BRANCH_NAME=${BRANCH_NAME} wc_java.deploy'
+                } else {
+                    sh 'make SETTINGS_XML=${SETTINGS_XML} wc_java.compile'
+                }
+            }
+        }
+
+    }
+}

Makefile

@@ -0,0 +1,80 @@
+THRIFT = $(or $(shell which thrift), $(error "`thrift' executable missing"))
+REBAR = $(shell which rebar3 2>/dev/null || which ./rebar3)
+SUBMODULES = build_utils
+SUBTARGETS = $(patsubst %,%/.git,$(SUBMODULES))
+
+UTILS_PATH := build_utils
+TEMPLATES_PATH := .
+
+# Name of the service
+SERVICE_NAME := file_storage
+
+# Build image tag to be used
+BUILD_IMAGE_TAG := 55e987e74e9457191a5b4a7c5dc9e3838ae82d2b
+CALL_ANYWHERE := \
+	all submodules compile clean distclean \
+	java.compile java.deploy
+
+CALL_W_CONTAINER := $(CALL_ANYWHERE)
+
+all: compile
+
+-include $(UTILS_PATH)/make_lib/utils_container.mk
+
+.PHONY: $(CALL_W_CONTAINER)
+
+# CALL_ANYWHERE
+$(SUBTARGETS): %/.git: %
+	git submodule update --init $<
+	touch $@
+
+submodules: $(SUBTARGETS)
+
+compile:
+	$(REBAR) compile
+
+clean:
+	$(REBAR) clean
+
+distclean:
+	$(REBAR) clean -a
+	rm -rfv _build
+
+# Java
+
+ifdef SETTINGS_XML
+DOCKER_RUN_OPTS = -v $(SETTINGS_XML):$(SETTINGS_XML)
+DOCKER_RUN_OPTS += -e SETTINGS_XML=$(SETTINGS_XML)
+endif
+
+ifdef LOCAL_BUILD
+DOCKER_RUN_OPTS += -v $$HOME/.m2:/home/$(UNAME)/.m2:rw
+endif
+
+COMMIT_HASH := $(shell git --no-pager log -1 --pretty=format:"%h")
+NUMBER_COMMITS := $(shell git rev-list --count HEAD)
+
+JAVA_PKG_VERSION := 1.$(NUMBER_COMMITS)-$(COMMIT_HASH)
+
+ifdef BRANCH_NAME
+ifeq "$(findstring epic,$(BRANCH_NAME))" "epic"
+JAVA_PKG_VERSION := $(JAVA_PKG_VERSION)-epic
+endif
+endif
+
+MVN = mvn -s $(SETTINGS_XML) -Dpath_to_thrift="$(THRIFT)" -Dcommit.number="$(NUMBER_COMMITS)"
+
+java.compile: java.settings
+	$(MVN) compile
+
+java.deploy: java.settings
+	$(MVN) versions:set versions:commit -DnewVersion="$(JAVA_PKG_VERSION)" && \
+	$(MVN) deploy
+
+java.install: java.settings
+	$(MVN) clean && \
+	$(MVN) versions:set versions:commit -DnewVersion="$(JAVA_PKG_VERSION)" && \
+	$(MVN) install
+
+java.settings:
+	$(if $(SETTINGS_XML),, echo "SETTINGS_XML not defined"; exit 1)

build_utils

@@ -0,0 +1 @@
+Subproject commit 9b664082ddc8ec8cdfbe7513d54e433d24198cc2

pom.xml

@@ -0,0 +1,81 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>com.rbkmoney</groupId>
+        <artifactId>spring-boot-starter-parent</artifactId>
+        <version>1.5.2.RELEASE</version>
+    </parent>
+
+    <artifactId>file-storage</artifactId>
+    <version>0.0.1-SNAPSHOT</version>
+    <packaging>jar</packaging>
+
+    <name>file-storage</name>
+    <description>Service for uploading &amp; downloading files</description>
+
+    <properties>
+        <project.maintainer>rbk.money</project.maintainer>
+        <server.port>8022</server.port>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+        <java.version>1.8</java.version>
+
+        <springfox-swagger2.version>2.8.0</springfox-swagger2.version>
+    </properties>
+
+    <dependencies>
+        <!-- Spring libs -->
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-web</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-configuration-processor</artifactId>
+            <optional>true</optional>
+        </dependency>
+
+        <!-- Third party libs -->
+        <dependency>
+            <groupId>com.amazonaws</groupId>
+            <artifactId>aws-java-sdk-s3</artifactId>
+            <version>1.11.160</version>
+        </dependency>
+        <dependency>
+            <groupId>io.springfox</groupId>
+            <artifactId>springfox-swagger2</artifactId>
+            <version>${springfox-swagger2.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>io.springfox</groupId>
+            <artifactId>springfox-swagger-ui</artifactId>
+            <version>${springfox-swagger2.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.projectlombok</groupId>
+            <artifactId>lombok</artifactId>
+            <version>1.18.4</version>
+            <scope>provided</scope>
+        </dependency>
+
+        <!-- Test libs -->
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-test</artifactId>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.springframework.boot</groupId>
+                <artifactId>spring-boot-maven-plugin</artifactId>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>

src/main/java/com/rbkmoney/file/storage/FileStorageApplication.java

@@ -0,0 +1,12 @@
+package com.rbkmoney.file.storage;
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+
+@SpringBootApplication
+public class FileStorageApplication {
+
+    public static void main(String[] args) {
+        SpringApplication.run(FileStorageApplication.class, args);
+    }
+}

src/main/java/com/rbkmoney/file/storage/config/StorageConfig.java

@@ -0,0 +1,78 @@
+package com.rbkmoney.file.storage.config;
+
+import com.amazonaws.ClientConfiguration;
+import com.amazonaws.Protocol;
+import com.amazonaws.auth.AWSCredentialsProviderChain;
+import com.amazonaws.auth.AWSStaticCredentialsProvider;
+import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.auth.EnvironmentVariableCredentialsProvider;
+import com.amazonaws.client.builder.AwsClientBuilder;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3ClientBuilder;
+import com.amazonaws.services.s3.transfer.TransferManager;
+import com.amazonaws.services.s3.transfer.TransferManagerBuilder;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+public class StorageConfig {
+
+    @Value("${storage.endpoint}")
+    private String endpoint;
+
+    @Value("${storage.signingRegion}")
+    private String signingRegion;
+
+    @Value("${storage.accessKey:}")
+    private String accessKey;
+
+    @Value("${storage.secretKey:}")
+    private String secretKey;
+
+    @Value("${storage.client.protocol:HTTP}")
+    private Protocol protocol;
+
+    @Value("${storage.client.maxErrorRetry}")
+    private int maxErrorRetry;
+
+    @Bean
+    public AmazonS3 storageClient(AWSCredentialsProviderChain credentialsProviderChain, ClientConfiguration clientConfiguration) {
+        return AmazonS3ClientBuilder.standard()
+                .withCredentials(credentialsProviderChain)
+                .withPathStyleAccessEnabled(true)
+                .withEndpointConfiguration(
+                        new AwsClientBuilder.EndpointConfiguration(endpoint, signingRegion)
+                )
+                .withClientConfiguration(clientConfiguration)
+                .build();
+    }
+
+    @Bean
+    public AWSCredentialsProviderChain credentialsProviderChain() {
+        return new AWSCredentialsProviderChain(
+                new EnvironmentVariableCredentialsProvider(),
+                new AWSStaticCredentialsProvider(
+                        new BasicAWSCredentials(
+                                accessKey,
+                                secretKey
+                        )
+                )
+        );
+    }
+
+    @Bean
+    public ClientConfiguration clientConfiguration() {
+        return new ClientConfiguration()
+                .withProtocol(protocol)
+                .withSignerOverride("S3SignerType")
+                .withMaxErrorRetry(maxErrorRetry);
+    }
+
+    @Bean
+    public TransferManager transferManager(AmazonS3 s3Client) {
+        return TransferManagerBuilder.standard()
+                .withS3Client(s3Client)
+                .build();
+    }
+}

src/main/java/com/rbkmoney/file/storage/config/SwaggerConfiguration.java

@@ -0,0 +1,35 @@
+package com.rbkmoney.file.storage.config;
+
+import com.google.common.base.Predicates;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import springfox.documentation.builders.ApiInfoBuilder;
+import springfox.documentation.builders.PathSelectors;
+import springfox.documentation.service.ApiInfo;
+import springfox.documentation.service.Contact;
+import springfox.documentation.spi.DocumentationType;
+import springfox.documentation.spring.web.plugins.Docket;
+import springfox.documentation.swagger2.annotations.EnableSwagger2;
+
+@Configuration
+@EnableSwagger2
+public class SwaggerConfiguration {
+
+    @Bean
+    public Docket productApi() {
+        return new Docket(DocumentationType.SWAGGER_2)
+                .apiInfo(metaData())
+                .select()
+                .paths(Predicates.not(PathSelectors.regex("/error")))
+                .build();
+    }
+
+    private ApiInfo metaData() {
+        return new ApiInfoBuilder()
+                .title("REST endpoint")
+                .description("\"Endpoint для выгрузки документов на сервер\"")
+                .version("0.0.1-SNAPSHOT")
+                .contact(new Contact("RBK.money", "https://github.com/rbkmoney", "support@rbkmoney.com"))
+                .build();
+    }
+}

src/main/java/com/rbkmoney/file/storage/contorller/FileController.java

@@ -0,0 +1,34 @@
+package com.rbkmoney.file.storage.contorller;
+
+import com.rbkmoney.file.storage.service.StorageService;
+import io.swagger.annotations.*;
+import lombok.RequiredArgsConstructor;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+import org.springframework.web.multipart.MultipartFile;
+
+@RestController
+@Api(description = "Api для операций с файлами")
+@RequiredArgsConstructor
+public class FileController {
+
+    private final StorageService storageService;
+
+    @PostMapping("/upload")
+    @ApiOperation(value = "Выгрузить файл на сервер")
+    @ApiResponses(value = {
+            @ApiResponse(code = 200, message = "Файл выгружен на сервер")
+    })
+    public ResponseEntity handleFileUpload(@ApiParam(value = "выгружаемый файл", required = true)
+                                           @RequestParam(value = "file")
+                                                   MultipartFile file,
+                                           @ApiParam(value = "id файла", required = true)
+                                           @RequestParam(value = "file_id")
+                                                   String fileId) {
+        storageService.store(fileId, file);
+        return new ResponseEntity(HttpStatus.OK);
+    }
+}

src/main/java/com/rbkmoney/file/storage/service/AmazonS3StorageService.java

@@ -0,0 +1,99 @@
+package com.rbkmoney.file.storage.service;
+
+import com.amazonaws.AmazonClientException;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.model.ObjectMetadata;
+import com.amazonaws.services.s3.model.PutObjectRequest;
+import com.amazonaws.services.s3.transfer.TransferManager;
+import com.amazonaws.services.s3.transfer.Upload;
+import com.rbkmoney.file.storage.service.exception.StorageException;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.codec.digest.DigestUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Service;
+import org.springframework.web.multipart.MultipartFile;
+
+import javax.annotation.PostConstruct;
+import javax.annotation.PreDestroy;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+@Service
+@Slf4j
+public class AmazonS3StorageService implements StorageService {
+
+    private final TransferManager transferManager;
+    private final AmazonS3 storageClient;
+    private final String bucketName;
+
+    @Autowired
+    public AmazonS3StorageService(TransferManager transferManager, @Value("${storage.bucketName}") String bucketName) {
+        this.transferManager = transferManager;
+        this.storageClient = transferManager.getAmazonS3Client();
+        this.bucketName = bucketName;
+    }
+
+    @PostConstruct
+    public void init() {
+        if (!storageClient.doesBucketExist(bucketName)) {
+            log.info("Create bucket in file storage, bucketId='{}'", bucketName);
+            storageClient.createBucket(bucketName);
+        }
+    }
+
+    @Override
+    public void store(String fileId, MultipartFile file) {
+        String filename = file.getOriginalFilename();
+        log.info("Trying to upload file to storage, filename='{}', bucketId='{}'", filename, bucketName);
+
+        try {
+            Path tempFile = createTempFile(file, filename);
+
+            ObjectMetadata objectMetadata = new ObjectMetadata();
+            objectMetadata.setContentDisposition("attachment;filename=" + filename);
+
+            PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, fileId, tempFile.toFile());
+            putObjectRequest.setMetadata(objectMetadata);
+            Upload upload = transferManager.upload(putObjectRequest);
+            try {
+                upload.waitForUploadResult();
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+            }
+
+            Files.deleteIfExists(tempFile);
+
+            log.info(
+                    "File have been successfully uploaded, fileId='{}', bucketId='{}', filename='{}', md5='{}'",
+                    fileId,
+                    bucketName,
+                    filename,
+                    DigestUtils.md5Hex(Files.newInputStream(tempFile))
+            );
+        } catch (IOException | AmazonClientException ex) {
+            throw new StorageException(String.format("Failed to upload file to storage, filename='%s', bucketId='%s'", filename, bucketName), ex);
+        }
+    }
+
+    private Path createTempFile(MultipartFile file, String filename) throws IOException {
+        Path tempFile = Files.createTempFile(filename, "");
+        OutputStream outputStream = new FileOutputStream(tempFile.toFile());
+
+        int read;
+        byte[] bytes = new byte[1024];
+
+        while ((read = file.getInputStream().read(bytes)) != -1) {
+            outputStream.write(bytes, 0, read);
+        }
+        return tempFile;
+    }
+
+    @PreDestroy
+    public void terminate() {
+        transferManager.shutdownNow(true);
+    }
+}

src/main/java/com/rbkmoney/file/storage/service/StorageService.java

@@ -0,0 +1,8 @@
+package com.rbkmoney.file.storage.service;
+
+import org.springframework.web.multipart.MultipartFile;
+
+public interface StorageService {
+
+    void store(String fileId, MultipartFile file);
+}

src/main/java/com/rbkmoney/file/storage/service/exception/StorageException.java

@@ -0,0 +1,12 @@
+package com.rbkmoney.file.storage.service.exception;
+
+public class StorageException extends RuntimeException {
+
+    public StorageException(String message) {
+        super(message);
+    }
+
+    public StorageException(String message, Throwable cause) {
+        super(message, cause);
+    }
+}

src/main/java/com/rbkmoney/file/storage/service/exception/StorageFileNotFoundException.java

@@ -0,0 +1,12 @@
+package com.rbkmoney.file.storage.service.exception;
+
+public class StorageFileNotFoundException extends StorageException {
+
+    public StorageFileNotFoundException(String message) {
+        super(message);
+    }
+
+    public StorageFileNotFoundException(String message, Throwable cause) {
+        super(message, cause);
+    }
+}

src/main/resources/application.properties

@@ -0,0 +1,7 @@
+server.port=@server.port@
+spring.application.name=@project.name@
+info.version=@project.version@
+info.stage=dev
+spring.servlet.multipart.max-file-size=128KB
+spring.servlet.multipart.max-request-size=128KB
+spring.servlet.multipart.enabled=true

src/test/java/com/rbkmoney/file/storage/FileStorageApplicationTests.java

@@ -0,0 +1,16 @@
+package com.rbkmoney.file.storage;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.test.context.junit4.SpringRunner;
+
+@RunWith(SpringRunner.class)
+@SpringBootTest
+public class FileStorageApplicationTests {
+
+    @Test
+    public void contextLoads() {
+    }
+
+}