mirror of
https://github.com/valitydev/file-storage.git
synced 2024-11-06 00:35:22 +00:00
BJ-314: test branch
This commit is contained in:
a.karlov
parent
8bd10b427b
commit
376a703605
42
pom.xml
42
pom.xml
@ -23,10 +23,34 @@
|
||||
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
|
||||
<java.version>1.8</java.version>
|
||||
|
||||
<springfox-swagger2.version>2.8.0</springfox-swagger2.version>
|
||||
<woody.thrift.version>1.1.15</woody.thrift.version>
|
||||
<file.storage.proto.version>1.5-81b8f4a</file.storage.proto.version>
|
||||
<geck.version>0.6.8</geck.version>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
<!-- rbkmoney -->
|
||||
<dependency>
|
||||
<groupId>com.rbkmoney</groupId>
|
||||
<artifactId>file-storage-proto</artifactId>
|
||||
<version>${file.storage.proto.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.rbkmoney.woody</groupId>
|
||||
<artifactId>woody-thrift</artifactId>
|
||||
<version>${woody.thrift.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.rbkmoney.geck</groupId>
|
||||
<artifactId>common</artifactId>
|
||||
<version>${geck.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.rbkmoney.geck</groupId>
|
||||
<artifactId>serializer</artifactId>
|
||||
<version>${geck.version}</version>
|
||||
</dependency>
|
||||
|
||||
<!-- Spring libs -->
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
@ -44,16 +68,6 @@
|
||||
<artifactId>aws-java-sdk-s3</artifactId>
|
||||
<version>1.11.160</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>io.springfox</groupId>
|
||||
<artifactId>springfox-swagger2</artifactId>
|
||||
<version>${springfox-swagger2.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>io.springfox</groupId>
|
||||
<artifactId>springfox-swagger-ui</artifactId>
|
||||
<version>${springfox-swagger2.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.projectlombok</groupId>
|
||||
<artifactId>lombok</artifactId>
|
||||
@ -67,6 +81,12 @@
|
||||
<artifactId>spring-boot-starter-test</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.testcontainers</groupId>
|
||||
<artifactId>testcontainers</artifactId>
|
||||
<version>1.8.3</version>
|
||||
<scope>compile</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
|
||||
@ -2,8 +2,10 @@ package com.rbkmoney.file.storage;
|
||||
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
import org.springframework.boot.web.servlet.ServletComponentScan;
|
||||
|
||||
@SpringBootApplication
|
||||
@ServletComponentScan
|
||||
@SpringBootApplication(scanBasePackages = {"com.rbkmoney.file.storage"})
|
||||
public class FileStorageApplication {
|
||||
|
||||
public static void main(String[] args) {
|
||||
|
||||
@ -16,7 +16,7 @@ import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
@Configuration
|
||||
public class StorageConfig {
|
||||
public class AmazonS3ClientConfiguration {
|
||||
|
||||
@Value("${storage.endpoint}")
|
||||
private String endpoint;
|
||||
@ -24,13 +24,13 @@ public class StorageConfig {
|
||||
@Value("${storage.signingRegion}")
|
||||
private String signingRegion;
|
||||
|
||||
@Value("${storage.accessKey:}")
|
||||
@Value("${storage.accessKey}")
|
||||
private String accessKey;
|
||||
|
||||
@Value("${storage.secretKey:}")
|
||||
@Value("${storage.secretKey}")
|
||||
private String secretKey;
|
||||
|
||||
@Value("${storage.client.protocol:HTTP}")
|
||||
@Value("${storage.client.protocol}")
|
||||
private Protocol protocol;
|
||||
|
||||
@Value("${storage.client.maxErrorRetry}")
|
||||
@ -0,0 +1,18 @@
|
||||
package com.rbkmoney.file.storage.config;
|
||||
|
||||
import com.rbkmoney.file.storage.FileStorageSrv;
|
||||
import com.rbkmoney.file.storage.handler.FileStorageHandler;
|
||||
import com.rbkmoney.file.storage.service.StorageService;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
@Configuration
|
||||
public class HandlerConfiguration {
|
||||
|
||||
@Bean
|
||||
@Autowired
|
||||
public FileStorageSrv.Iface fileStorageHandler(StorageService storageService) {
|
||||
return new FileStorageHandler(storageService);
|
||||
}
|
||||
}
|
||||
@ -1,35 +0,0 @@
|
||||
package com.rbkmoney.file.storage.config;
|
||||
|
||||
import com.google.common.base.Predicates;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import springfox.documentation.builders.ApiInfoBuilder;
|
||||
import springfox.documentation.builders.PathSelectors;
|
||||
import springfox.documentation.service.ApiInfo;
|
||||
import springfox.documentation.service.Contact;
|
||||
import springfox.documentation.spi.DocumentationType;
|
||||
import springfox.documentation.spring.web.plugins.Docket;
|
||||
import springfox.documentation.swagger2.annotations.EnableSwagger2;
|
||||
|
||||
@Configuration
|
||||
@EnableSwagger2
|
||||
public class SwaggerConfiguration {
|
||||
|
||||
@Bean
|
||||
public Docket productApi() {
|
||||
return new Docket(DocumentationType.SWAGGER_2)
|
||||
.apiInfo(metaData())
|
||||
.select()
|
||||
.paths(Predicates.not(PathSelectors.regex("/error")))
|
||||
.build();
|
||||
}
|
||||
|
||||
private ApiInfo metaData() {
|
||||
return new ApiInfoBuilder()
|
||||
.title("REST endpoint")
|
||||
.description("\"Endpoint для выгрузки документов на сервер\"")
|
||||
.version("0.0.1-SNAPSHOT")
|
||||
.contact(new Contact("RBK.money", "https://github.com/rbkmoney", "support@rbkmoney.com"))
|
||||
.build();
|
||||
}
|
||||
}
|
||||
@ -1,34 +0,0 @@
|
||||
package com.rbkmoney.file.storage.contorller;
|
||||
|
||||
import com.rbkmoney.file.storage.service.StorageService;
|
||||
import io.swagger.annotations.*;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
@RestController
|
||||
@Api(description = "Api для операций с файлами")
|
||||
@RequiredArgsConstructor
|
||||
public class FileController {
|
||||
|
||||
private final StorageService storageService;
|
||||
|
||||
@PostMapping("/upload")
|
||||
@ApiOperation(value = "Выгрузить файл на сервер")
|
||||
@ApiResponses(value = {
|
||||
@ApiResponse(code = 200, message = "Файл выгружен на сервер")
|
||||
})
|
||||
public ResponseEntity handleFileUpload(@ApiParam(value = "выгружаемый файл", required = true)
|
||||
@RequestParam(value = "file")
|
||||
MultipartFile file,
|
||||
@ApiParam(value = "id файла", required = true)
|
||||
@RequestParam(value = "file_id")
|
||||
String fileId) {
|
||||
storageService.store(fileId, file);
|
||||
return new ResponseEntity(HttpStatus.OK);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,42 @@
|
||||
package com.rbkmoney.file.storage.contorller;
|
||||
|
||||
import com.rbkmoney.file.storage.service.StorageService;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
import java.io.FileNotFoundException;
|
||||
|
||||
import static com.rbkmoney.file.storage.util.CheckerUtil.checkString;
|
||||
|
||||
@RestController
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class UploadFileController {
|
||||
|
||||
private final StorageService storageService;
|
||||
|
||||
@PostMapping("/file_storage/upload")
|
||||
public ResponseEntity handleFileUpload(@RequestParam(value = "file_id") String fileId,
|
||||
@RequestParam(value = "file") MultipartFile file) {
|
||||
try {
|
||||
log.info("Request handleFileUpload fileId: {}", fileId);
|
||||
checkString(fileId, "Bad request parameter, fileId required and not empty arg");
|
||||
storageService.uploadFile(fileId, file.getInputStream());
|
||||
ResponseEntity<Object> responseEntity = ResponseEntity.ok().build();
|
||||
log.info("Response: ResponseEntity: {}", responseEntity);
|
||||
return responseEntity;
|
||||
} catch (FileNotFoundException e) {
|
||||
log.error("Error when handleFileUpload e: ", e);
|
||||
return ResponseEntity.notFound().build();
|
||||
} catch (Exception e) {
|
||||
log.error("Error when handleFileUpload e: ", e);
|
||||
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).build();
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,78 @@
|
||||
package com.rbkmoney.file.storage.handler;
|
||||
|
||||
import com.rbkmoney.damsel.msgpack.Value;
|
||||
import com.rbkmoney.file.storage.FileData;
|
||||
import com.rbkmoney.file.storage.FileNotFound;
|
||||
import com.rbkmoney.file.storage.FileStorageSrv;
|
||||
import com.rbkmoney.file.storage.NewFileResult;
|
||||
import com.rbkmoney.file.storage.service.StorageService;
|
||||
import com.rbkmoney.geck.common.util.TypeUtil;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.thrift.TException;
|
||||
|
||||
import java.io.FileNotFoundException;
|
||||
import java.net.URL;
|
||||
import java.time.Instant;
|
||||
import java.util.Map;
|
||||
|
||||
import static com.rbkmoney.file.storage.util.CheckerUtil.checkString;
|
||||
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class FileStorageHandler implements FileStorageSrv.Iface {
|
||||
|
||||
private final StorageService storageService;
|
||||
|
||||
@Override
|
||||
public FileData getFileData(String fileId) throws TException {
|
||||
try {
|
||||
log.info("Request getFileData fileId: {}", fileId);
|
||||
checkString(fileId, "Bad request parameter, fileId required and not empty arg");
|
||||
FileData fileData = storageService.getFileData(fileId);
|
||||
log.info("Response: fileData: {}", fileData);
|
||||
return fileData;
|
||||
} catch (FileNotFoundException e) {
|
||||
log.error("Error when getFileData e: ", e);
|
||||
throw new FileNotFound();
|
||||
} catch (Exception e) {
|
||||
log.error("Error when getFileData e: ", e);
|
||||
throw new TException(e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public NewFileResult createNewFile(String fileName, Map<String, Value> metadata, String expiresAt) throws TException {
|
||||
try {
|
||||
log.info("Request createNewFile fileName: {}, metadata: {}, expiresAt: {}", fileName, metadata, expiresAt);
|
||||
checkString(fileName, "Bad request parameter, fileName required and not empty arg");
|
||||
checkString(expiresAt, "Bad request parameter, expiresAt required and not empty arg");
|
||||
Instant instant = TypeUtil.stringToInstant(expiresAt);
|
||||
NewFileResult newFile = storageService.createNewFile(fileName, metadata, instant);
|
||||
log.info("Response: newFileResult: {}", newFile);
|
||||
return newFile;
|
||||
} catch (Exception e) {
|
||||
log.error("Error when createNewFile e: ", e);
|
||||
throw new TException(e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String generateDownloadUrl(String fileId, String expiresAt) throws TException {
|
||||
try {
|
||||
log.info("Request generateDownloadUrl fileId: {}, expiresAt: {}", fileId, expiresAt);
|
||||
checkString(fileId, "Bad request parameter, fileId required and not empty arg");
|
||||
checkString(expiresAt, "Bad request parameter, expiresAt required and not empty arg");
|
||||
Instant instant = TypeUtil.stringToInstant(expiresAt);
|
||||
URL url = storageService.generateDownloadUrl(fileId, instant);
|
||||
log.info("Response: url: {}", url);
|
||||
return url.toString();
|
||||
} catch (FileNotFoundException e) {
|
||||
log.error("Error when generateDownloadUrl e: ", e);
|
||||
throw new FileNotFound();
|
||||
} catch (Exception e) {
|
||||
log.error("Error when generateDownloadUrl e: ", e);
|
||||
throw new TException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,30 @@
|
||||
package com.rbkmoney.file.storage.resource;
|
||||
|
||||
import com.rbkmoney.file.storage.FileStorageSrv;
|
||||
import com.rbkmoney.woody.thrift.impl.http.THServiceBuilder;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
import javax.servlet.*;
|
||||
import javax.servlet.annotation.WebServlet;
|
||||
import java.io.IOException;
|
||||
|
||||
@WebServlet("/file_storage")
|
||||
@RequiredArgsConstructor
|
||||
public class FileStorageServlet extends GenericServlet {
|
||||
|
||||
private Servlet thriftServlet;
|
||||
|
||||
private final FileStorageSrv.Iface requestHandler;
|
||||
|
||||
@Override
|
||||
public void init(ServletConfig config) throws ServletException {
|
||||
super.init(config);
|
||||
thriftServlet = new THServiceBuilder()
|
||||
.build(FileStorageSrv.Iface.class, requestHandler);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void service(ServletRequest req, ServletResponse res) throws ServletException, IOException {
|
||||
thriftServlet.service(req, res);
|
||||
}
|
||||
}
|
||||
@ -1,99 +1,408 @@
|
||||
package com.rbkmoney.file.storage.service;
|
||||
|
||||
import com.amazonaws.AmazonClientException;
|
||||
import com.amazonaws.HttpMethod;
|
||||
import com.amazonaws.services.s3.AmazonS3;
|
||||
import com.amazonaws.services.s3.model.ObjectMetadata;
|
||||
import com.amazonaws.services.s3.model.PutObjectRequest;
|
||||
import com.amazonaws.services.s3.model.*;
|
||||
import com.amazonaws.services.s3.transfer.TransferManager;
|
||||
import com.amazonaws.services.s3.transfer.Upload;
|
||||
import com.rbkmoney.file.storage.FileData;
|
||||
import com.rbkmoney.file.storage.NewFileResult;
|
||||
import com.rbkmoney.file.storage.contorller.UploadFileController;
|
||||
import com.rbkmoney.file.storage.service.exception.StorageException;
|
||||
import com.rbkmoney.file.storage.util.DamselUtil;
|
||||
import com.rbkmoney.geck.common.util.TypeUtil;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.codec.digest.DigestUtils;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
import org.springframework.web.servlet.mvc.method.annotation.MvcUriComponentsBuilder;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
import javax.annotation.PreDestroy;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
import java.io.InputStream;
|
||||
import java.net.MalformedURLException;
|
||||
import java.net.URL;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.time.Instant;
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Service
|
||||
@Slf4j
|
||||
public class AmazonS3StorageService implements StorageService {
|
||||
|
||||
private static final String EXPIRATION_TIME = "x-rbkmoney-file-expiration-time";
|
||||
private static final String FILE_UPLOADED = "x-rbkmoney-file-uploaded";
|
||||
private static final String FILEDATA_FILE_ID = "x-rbkmoney-filedata-file-id";
|
||||
private static final String FILEDATA_FILE_NAME = "x-rbkmoney-filedata-file-name";
|
||||
private static final String FILEDATA_CREATED_AT = "x-rbkmoney-filedata-created-at";
|
||||
private static final String FILEDATA_MD_5 = "x-rbkmoney-filedata-md5";
|
||||
private static final String FILEDATA_METADATA = "x-rbkmoney-filedata-metadata-";
|
||||
|
||||
private final TransferManager transferManager;
|
||||
private final AmazonS3 storageClient;
|
||||
private final AmazonS3 s3Client;
|
||||
private final String bucketName;
|
||||
|
||||
@Autowired
|
||||
public AmazonS3StorageService(TransferManager transferManager, @Value("${storage.bucketName}") String bucketName) {
|
||||
this.transferManager = transferManager;
|
||||
this.storageClient = transferManager.getAmazonS3Client();
|
||||
this.s3Client = transferManager.getAmazonS3Client();
|
||||
this.bucketName = bucketName;
|
||||
}
|
||||
|
||||
@PostConstruct
|
||||
public void init() {
|
||||
if (!storageClient.doesBucketExist(bucketName)) {
|
||||
if (!s3Client.doesBucketExist(bucketName)) {
|
||||
log.info("Create bucket in file storage, bucketId='{}'", bucketName);
|
||||
storageClient.createBucket(bucketName);
|
||||
s3Client.createBucket(bucketName);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void store(String fileId, MultipartFile file) {
|
||||
String filename = file.getOriginalFilename();
|
||||
log.info("Trying to upload file to storage, filename='{}', bucketId='{}'", filename, bucketName);
|
||||
public FileData getFileData(String fileId) throws StorageException, FileNotFoundException {
|
||||
S3Object s3Object = getS3Object(fileId);
|
||||
checkFileStatus(s3Object);
|
||||
return extractFileData(s3Object.getObjectMetadata());
|
||||
}
|
||||
|
||||
@Override
|
||||
public NewFileResult createNewFile(String fileName, Map<String, com.rbkmoney.damsel.msgpack.Value> metadata, Instant expirationTime) throws StorageException {
|
||||
log.info("Trying to create new file to storage, filename='{}', bucketId='{}'", fileName, bucketName);
|
||||
|
||||
try {
|
||||
Path tempFile = createTempFile(file, filename);
|
||||
// в хранилище сохраняется пустой файл
|
||||
InputStream emptyContent = new ByteArrayInputStream(new byte[0]);
|
||||
|
||||
ObjectMetadata objectMetadata = new ObjectMetadata();
|
||||
objectMetadata.setContentDisposition("attachment;filename=" + filename);
|
||||
String fileId = getFileId();
|
||||
String createdAt = Instant.now().toString();
|
||||
|
||||
PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, fileId, tempFile.toFile());
|
||||
putObjectRequest.setMetadata(objectMetadata);
|
||||
Upload upload = transferManager.upload(putObjectRequest);
|
||||
try {
|
||||
upload.waitForUploadResult();
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
FileData fileData = new FileData(
|
||||
fileId,
|
||||
fileName,
|
||||
createdAt,
|
||||
//todo
|
||||
"",
|
||||
metadata
|
||||
);
|
||||
|
||||
Files.deleteIfExists(tempFile);
|
||||
writeFileToStorage(fileData, emptyContent, expirationTime);
|
||||
|
||||
URL uploadUrl = createUploadUrl(fileId);
|
||||
|
||||
log.info(
|
||||
"File have been successfully uploaded, fileId='{}', bucketId='{}', filename='{}', md5='{}'",
|
||||
"File have been successfully created, fileId='{}', bucketId='{}', filename='{}', md5='{}'",
|
||||
fileId,
|
||||
bucketName,
|
||||
filename,
|
||||
DigestUtils.md5Hex(Files.newInputStream(tempFile))
|
||||
fileName,
|
||||
//todo
|
||||
""
|
||||
);
|
||||
|
||||
return new NewFileResult(uploadUrl.toString(), fileData);
|
||||
} catch (AmazonClientException ex) {
|
||||
throw new StorageException(
|
||||
String.format(
|
||||
"Failed to create new file to storage, filename='%s', bucketId='%s'",
|
||||
fileName,
|
||||
bucketName
|
||||
),
|
||||
ex
|
||||
);
|
||||
} catch (IOException | AmazonClientException ex) {
|
||||
throw new StorageException(String.format("Failed to upload file to storage, filename='%s', bucketId='%s'", filename, bucketName), ex);
|
||||
}
|
||||
}
|
||||
|
||||
private Path createTempFile(MultipartFile file, String filename) throws IOException {
|
||||
Path tempFile = Files.createTempFile(filename, "");
|
||||
OutputStream outputStream = new FileOutputStream(tempFile.toFile());
|
||||
@Override
|
||||
public URL generateDownloadUrl(String fileId, Instant expirationTime) throws StorageException, FileNotFoundException {
|
||||
checkFileStatus(getS3Object(fileId));
|
||||
return generatePresignedUrl(fileId, expirationTime, HttpMethod.GET);
|
||||
}
|
||||
|
||||
int read;
|
||||
byte[] bytes = new byte[1024];
|
||||
@Override
|
||||
public void uploadFile(String fileId, InputStream inputStream) throws StorageException, IOException {
|
||||
log.info("Trying to upload file to storage, filename='{}', bucketId='{}'", fileId, bucketName);
|
||||
|
||||
while ((read = file.getInputStream().read(bytes)) != -1) {
|
||||
outputStream.write(bytes, 0, read);
|
||||
try {
|
||||
|
||||
//todo
|
||||
Path testFile = Files.createTempFile("", "test_file");
|
||||
Files.write(testFile, "Test".getBytes());
|
||||
|
||||
S3Object object = getS3Object(fileId);
|
||||
|
||||
checkFileStatus(object);
|
||||
|
||||
object.getObjectMetadata().addUserMetadata(FILE_UPLOADED, "true");
|
||||
/*
|
||||
//todo DEBUG
|
||||
try {
|
||||
// InputStream inputStream = file.getInputStream();
|
||||
Path testActualFile = Files.createTempFile("", "test_actual_file");
|
||||
Files.copy(inputStream, testActualFile, StandardCopyOption.REPLACE_EXISTING);
|
||||
System.out.println("!!!!!!!");
|
||||
for (String readAllLine : Files.readAllLines(testActualFile)) {
|
||||
System.out.println(readAllLine);
|
||||
}
|
||||
testActualFile = Files.createTempFile("", "test_actual_file");
|
||||
Files.copy(inputStream, testActualFile, StandardCopyOption.REPLACE_EXISTING);
|
||||
System.out.println("!!!!!!!");
|
||||
for (String readAllLine : Files.readAllLines(testActualFile)) {
|
||||
System.out.println(readAllLine);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
}
|
||||
//todo
|
||||
*/
|
||||
|
||||
PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, fileId, testFile.toFile());
|
||||
putObjectRequest.setMetadata(object.getObjectMetadata());
|
||||
s3Client.putObject(putObjectRequest);
|
||||
|
||||
/*
|
||||
PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, fileId, new S3ObjectInputStream(
|
||||
new FileInputStream(testFile.toFile()),null
|
||||
),object.getObjectMetadata());
|
||||
s3Client.putObject(putObjectRequest);
|
||||
*/
|
||||
/*
|
||||
PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, fileId, new FileInputStream(testFile.toFile()),object.getObjectMetadata());
|
||||
s3Client.putObject(putObjectRequest);
|
||||
*/
|
||||
|
||||
/* todo
|
||||
PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, fileId, inputStream, object.getObjectMetadata());
|
||||
s3Client.putObject(putObjectRequest);
|
||||
*/
|
||||
log.info(
|
||||
"File have been successfully uploaded, fileId='{}', bucketId='{}'",
|
||||
fileId,
|
||||
bucketName
|
||||
);
|
||||
|
||||
//todo
|
||||
/* try {
|
||||
S3Object object1 = s3Client.getObject(bucketName, fileId);
|
||||
S3ObjectInputStream objectContent = object1.getObjectContent();
|
||||
Path testActualFile = Files.createTempFile("", "test_actual_file");
|
||||
Files.copy(objectContent, testActualFile, StandardCopyOption.REPLACE_EXISTING);
|
||||
System.out.println("!!!!!!!");
|
||||
for (String readAllLine : Files.readAllLines(testActualFile)) {
|
||||
System.out.println(readAllLine);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
}
|
||||
*/
|
||||
//todo
|
||||
} catch (AmazonClientException ex) {
|
||||
throw new StorageException(
|
||||
String.format(
|
||||
"Failed to to upload file to storage, filename='%s', bucketId='%s'",
|
||||
fileId,
|
||||
bucketName
|
||||
),
|
||||
ex
|
||||
);
|
||||
}
|
||||
return tempFile;
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
public void terminate() {
|
||||
transferManager.shutdownNow(true);
|
||||
}
|
||||
|
||||
private S3Object getS3Object(String fileId) throws StorageException, FileNotFoundException {
|
||||
try {
|
||||
log.info(
|
||||
"Trying to get file from storage, fileId='{}', bucketId='{}'",
|
||||
fileId,
|
||||
bucketName
|
||||
);
|
||||
GetObjectRequest getObjectRequest = new GetObjectRequest(bucketName, fileId);
|
||||
S3Object object = s3Client.getObject(getObjectRequest);
|
||||
checkNullable(object, fileId, "File");
|
||||
log.info(
|
||||
"File have been successfully got from storage, fileId='{}', bucketId='{}'",
|
||||
fileId,
|
||||
bucketName
|
||||
);
|
||||
return object;
|
||||
} catch (AmazonClientException ex) {
|
||||
throw new StorageException(
|
||||
String.format(
|
||||
"Failed to get file from storage, fileId='%s', bucketId='%s'",
|
||||
fileId,
|
||||
bucketName
|
||||
),
|
||||
ex
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private void checkFileStatus(S3Object s3Object) throws StorageException {
|
||||
log.info("Check file expiration and uploaded status: ETag='{}'", s3Object.getObjectMetadata().getETag());
|
||||
ObjectMetadata objectMetadata = s3Object.getObjectMetadata();
|
||||
|
||||
Boolean isUploaded = getBooleanFromObjectMetadata(objectMetadata);
|
||||
if (isUploaded) {
|
||||
log.info("File was uploaded: ETag='{}'", s3Object.getObjectMetadata().getETag());
|
||||
return;
|
||||
}
|
||||
|
||||
Date expirationTime = getDateFromObjectMetadata(objectMetadata);
|
||||
Date time = new Date();
|
||||
if (time.getTime() < expirationTime.getTime()) {
|
||||
log.info("File was uploaded: ETag='{}'", s3Object.getObjectMetadata().getETag());
|
||||
return;
|
||||
}
|
||||
|
||||
// если файл не соотвествует условиям, блокируем доступ к нему
|
||||
throw new StorageException(String.format("File access error: fileId='%s', bucketId='%s', create a new file", s3Object.getKey(), bucketName));
|
||||
}
|
||||
|
||||
private FileData extractFileData(ObjectMetadata objectMetadata) {
|
||||
log.info("Trying to extract metadata from storage: ETag='{}'", objectMetadata.getETag());
|
||||
String fileId = getUserMetadataParameter(objectMetadata, FILEDATA_FILE_ID);
|
||||
String fileName = getUserMetadataParameter(objectMetadata, FILEDATA_FILE_NAME);
|
||||
String createdAt = getUserMetadataParameter(objectMetadata, FILEDATA_CREATED_AT);
|
||||
String md5 = getUserMetadataParameter(objectMetadata, FILEDATA_MD_5);
|
||||
|
||||
Map<String, com.rbkmoney.damsel.msgpack.Value> metadata = objectMetadata.getUserMetadata().entrySet().stream()
|
||||
.filter(entry -> entry.getKey().startsWith(FILEDATA_METADATA) && entry.getValue() != null)
|
||||
.collect(
|
||||
Collectors.toMap(
|
||||
o -> o.getKey().substring(FILEDATA_METADATA.length()),
|
||||
o -> DamselUtil.fromJson(o.getValue(), com.rbkmoney.damsel.msgpack.Value.class)
|
||||
)
|
||||
);
|
||||
log.info(
|
||||
"Metadata have been successfully extracted from storage, fileId='{}', bucketId='{}'",
|
||||
fileId,
|
||||
bucketName
|
||||
);
|
||||
return new FileData(fileId, fileName, createdAt, md5, metadata);
|
||||
}
|
||||
|
||||
private URL generatePresignedUrl(String fileId, Instant expirationTime, HttpMethod httpMethod) throws StorageException, FileNotFoundException {
|
||||
try {
|
||||
log.info(
|
||||
"Trying to generate presigned url, fileId='{}', bucketId='{}', expirationTime='{}', httpMethod='{}'",
|
||||
fileId,
|
||||
bucketName,
|
||||
expirationTime,
|
||||
httpMethod
|
||||
);
|
||||
|
||||
GeneratePresignedUrlRequest request = new GeneratePresignedUrlRequest(bucketName, fileId)
|
||||
.withMethod(httpMethod)
|
||||
.withExpiration(Date.from(expirationTime));
|
||||
URL url = s3Client.generatePresignedUrl(request);
|
||||
checkNullable(url, fileId, "Presigned url");
|
||||
log.info(
|
||||
"Presigned url have been successfully generated, url='{}', fileId='{}', bucketId='{}', expirationTime='{}', httpMethod='{}'",
|
||||
url,
|
||||
fileId,
|
||||
bucketName,
|
||||
expirationTime,
|
||||
httpMethod
|
||||
);
|
||||
return url;
|
||||
} catch (AmazonClientException ex) {
|
||||
throw new StorageException(
|
||||
String.format(
|
||||
"Failed to generate presigned url, fileId='%s', bucketId='%s', expirationTime='%s', httpMethod='%s'",
|
||||
fileId,
|
||||
bucketName,
|
||||
expirationTime,
|
||||
httpMethod
|
||||
),
|
||||
ex
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private void writeFileToStorage(FileData fileData, InputStream inputStream, Instant expirationTime) throws AmazonClientException {
|
||||
PutObjectRequest request = createS3Request(fileData, inputStream, expirationTime);
|
||||
s3Client.putObject(request);
|
||||
}
|
||||
|
||||
private PutObjectRequest createS3Request(FileData fileData, InputStream inputStream, Instant expirationTime) {
|
||||
return new PutObjectRequest(
|
||||
bucketName,
|
||||
fileData.getFileId(),
|
||||
inputStream,
|
||||
createObjectMetadata(fileData, expirationTime)
|
||||
);
|
||||
}
|
||||
|
||||
private ObjectMetadata createObjectMetadata(FileData fileData, Instant expirationTime) {
|
||||
ObjectMetadata objectMetadata = new ObjectMetadata();
|
||||
objectMetadata.setContentDisposition("attachment;filename=" + fileData.getFileName());
|
||||
// file parameters
|
||||
objectMetadata.addUserMetadata(EXPIRATION_TIME, expirationTime.toString());
|
||||
objectMetadata.addUserMetadata(FILE_UPLOADED, "false");
|
||||
// filedata parameters
|
||||
objectMetadata.addUserMetadata(FILEDATA_FILE_ID, fileData.getFileId());
|
||||
objectMetadata.addUserMetadata(FILEDATA_FILE_NAME, fileData.getFileName());
|
||||
objectMetadata.addUserMetadata(FILEDATA_CREATED_AT, fileData.getCreatedAt());
|
||||
objectMetadata.addUserMetadata(FILEDATA_MD_5, fileData.getMd5());
|
||||
fileData.getMetadata().forEach(
|
||||
(key, value) -> objectMetadata.addUserMetadata(FILEDATA_METADATA + key, DamselUtil.toJsonString(value))
|
||||
);
|
||||
return objectMetadata;
|
||||
}
|
||||
|
||||
private URL createUploadUrl(String fileId) {
|
||||
try {
|
||||
return MvcUriComponentsBuilder.fromMethodName(
|
||||
UploadFileController.class,
|
||||
"handleFileUpload",
|
||||
fileId,
|
||||
null
|
||||
)
|
||||
.buildAndExpand()
|
||||
.encode()
|
||||
.toUri()
|
||||
.toURL();
|
||||
} catch (MalformedURLException e) {
|
||||
throw new StorageException(
|
||||
String.format(
|
||||
"Exception createUploadUrl: fileId='%s', bucketId='%s', create a new file",
|
||||
fileId,
|
||||
bucketName
|
||||
),
|
||||
e);
|
||||
}
|
||||
}
|
||||
|
||||
private Boolean getBooleanFromObjectMetadata(ObjectMetadata objectMetadata) {
|
||||
String isUploadedString = getUserMetadataParameter(objectMetadata, FILE_UPLOADED);
|
||||
return Boolean.valueOf(isUploadedString);
|
||||
}
|
||||
|
||||
private Date getDateFromObjectMetadata(ObjectMetadata objectMetadata) throws StorageException {
|
||||
String expirationTime = getUserMetadataParameter(objectMetadata, EXPIRATION_TIME);
|
||||
return Date.from(TypeUtil.stringToInstant(expirationTime));
|
||||
}
|
||||
|
||||
private String getUserMetadataParameter(ObjectMetadata objectMetadata, String key) throws StorageException {
|
||||
return Optional.ofNullable(objectMetadata.getUserMetaDataOf(key))
|
||||
.orElseThrow(() -> new StorageException("Failed to extract user metadata parameter, " + key + " is null"));
|
||||
}
|
||||
|
||||
private String getFileId() {
|
||||
String fileId;
|
||||
do {
|
||||
fileId = UUID.randomUUID().toString();
|
||||
} while (s3Client.doesObjectExist(bucketName, fileId));
|
||||
return fileId;
|
||||
}
|
||||
|
||||
private void checkNullable(Object object, String fileId, String objectType) throws FileNotFoundException {
|
||||
if (Objects.isNull(object)) {
|
||||
throw new FileNotFoundException(String.format(objectType + " is null, fileId='%s', bucketId='%s'", fileId, bucketName));
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,8 +1,25 @@
|
||||
package com.rbkmoney.file.storage.service;
|
||||
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
import com.rbkmoney.damsel.msgpack.Value;
|
||||
import com.rbkmoney.file.storage.FileData;
|
||||
import com.rbkmoney.file.storage.NewFileResult;
|
||||
import com.rbkmoney.file.storage.service.exception.StorageException;
|
||||
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.net.URL;
|
||||
import java.time.Instant;
|
||||
import java.util.Map;
|
||||
|
||||
public interface StorageService {
|
||||
|
||||
void store(String fileId, MultipartFile file);
|
||||
FileData getFileData(String fileId) throws StorageException, FileNotFoundException;
|
||||
|
||||
NewFileResult createNewFile(String fileName, Map<String, Value> metadata, Instant expirationTime) throws StorageException;
|
||||
|
||||
URL generateDownloadUrl(String fileId, Instant expirationTime) throws StorageException, FileNotFoundException;
|
||||
|
||||
void uploadFile(String fileId, InputStream inputStream) throws StorageException, IOException;
|
||||
|
||||
}
|
||||
|
||||
@ -0,0 +1,13 @@
|
||||
package com.rbkmoney.file.storage.util;
|
||||
|
||||
import com.google.common.base.Strings;
|
||||
import org.apache.thrift.TException;
|
||||
|
||||
public class CheckerUtil {
|
||||
|
||||
public static void checkString(String string, String exMessage) throws TException {
|
||||
if (Strings.isNullOrEmpty(string)) {
|
||||
throw new TException(exMessage);
|
||||
}
|
||||
}
|
||||
}
|
||||
35
src/main/java/com/rbkmoney/file/storage/util/DamselUtil.java
Normal file
35
src/main/java/com/rbkmoney/file/storage/util/DamselUtil.java
Normal file
@ -0,0 +1,35 @@
|
||||
package com.rbkmoney.file.storage.util;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.rbkmoney.geck.serializer.kit.json.JsonHandler;
|
||||
import com.rbkmoney.geck.serializer.kit.json.JsonProcessor;
|
||||
import com.rbkmoney.geck.serializer.kit.tbase.TBaseHandler;
|
||||
import com.rbkmoney.geck.serializer.kit.tbase.TBaseProcessor;
|
||||
import org.apache.thrift.TBase;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
public class DamselUtil {
|
||||
|
||||
public static String toJsonString(TBase tBase) {
|
||||
return toJson(tBase).toString();
|
||||
}
|
||||
|
||||
public static JsonNode toJson(TBase tBase) {
|
||||
try {
|
||||
return new TBaseProcessor().process(tBase, new JsonHandler());
|
||||
} catch (IOException ex) {
|
||||
throw new IllegalArgumentException(ex);
|
||||
}
|
||||
}
|
||||
|
||||
public static <T extends TBase> T fromJson(String jsonString, Class<T> type) {
|
||||
try {
|
||||
return new JsonProcessor().process(new ObjectMapper().readTree(jsonString), new TBaseHandler<>(type));
|
||||
} catch (IOException ex) {
|
||||
throw new IllegalArgumentException(ex);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -5,3 +5,8 @@ info.stage=dev
|
||||
spring.servlet.multipart.max-file-size=128KB
|
||||
spring.servlet.multipart.max-request-size=128KB
|
||||
spring.servlet.multipart.enabled=true
|
||||
storage.endpoint=localhost
|
||||
storage.signingRegion=RU
|
||||
storage.client.protocol=HTTP
|
||||
storage.client.maxErrorRetry=10
|
||||
storage.bucketName="files"
|
||||
|
||||
@ -0,0 +1,81 @@
|
||||
package com.rbkmoney.file.storage;
|
||||
|
||||
import org.junit.runner.RunWith;
|
||||
import org.springframework.boot.context.embedded.LocalServerPort;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
import org.springframework.boot.test.util.EnvironmentTestUtils;
|
||||
import org.springframework.context.ApplicationContextInitializer;
|
||||
import org.springframework.context.ConfigurableApplicationContext;
|
||||
import org.springframework.test.annotation.DirtiesContext;
|
||||
import org.springframework.test.context.ContextConfiguration;
|
||||
import org.springframework.test.context.junit4.SpringRunner;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.ZoneOffset;
|
||||
|
||||
import static org.springframework.boot.test.context.SpringBootTest.WebEnvironment.RANDOM_PORT;
|
||||
|
||||
@RunWith(SpringRunner.class)
|
||||
@SpringBootTest(webEnvironment = RANDOM_PORT)
|
||||
@ContextConfiguration(classes = FileStorageApplication.class, initializers = AbstractIntegrationTest.Initializer.class)
|
||||
@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS)
|
||||
public class AbstractIntegrationTest {
|
||||
|
||||
private static final String SIGNING_REGION = "RU";
|
||||
private static final String AWS_ACCESS_KEY = "test";
|
||||
private static final String AWS_SECRET_KEY = "test";
|
||||
private static final String PROTOCOL = "HTTP";
|
||||
private static final String MAX_ERROR_RETRY = "10";
|
||||
private static final String BUCKET_NAME = "TEST";
|
||||
|
||||
@LocalServerPort
|
||||
protected int port;
|
||||
|
||||
// @ClassRule
|
||||
// public static GenericContainer cephContainer = new GenericContainer("dr.rbkmoney.com/ceph-demo:latest")
|
||||
// .withEnv("RGW_NAME", "localhost")
|
||||
// .withEnv("NETWORK_AUTO_DETECT", "4")
|
||||
// .withEnv("CEPH_DEMO_UID", "ceph-test")
|
||||
// .withEnv("CEPH_DEMO_ACCESS_KEY", AWS_ACCESS_KEY)
|
||||
// .withEnv("CEPH_DEMO_SECRET_KEY", AWS_SECRET_KEY)
|
||||
// .withEnv("CEPH_DEMO_BUCKET", BUCKET_NAME)
|
||||
// .withExposedPorts(5000, 80)
|
||||
// .waitingFor(
|
||||
// new HttpWaitStrategy()
|
||||
// .forPath("/api/v0.1/health")
|
||||
// .forStatusCode(200)
|
||||
// .withStartupTimeout(Duration.ofMinutes(10))
|
||||
// );
|
||||
|
||||
public static class Initializer implements ApplicationContextInitializer<ConfigurableApplicationContext> {
|
||||
@Override
|
||||
public void initialize(ConfigurableApplicationContext configurableApplicationContext) {
|
||||
EnvironmentTestUtils.addEnvironment(
|
||||
"testcontainers",
|
||||
configurableApplicationContext.getEnvironment(),
|
||||
// "storage.endpoint=" + cephContainer.getContainerIpAddress() + ":" + cephContainer.getMappedPort(80),
|
||||
"storage.endpoint=localhost:32827",
|
||||
"storage.signingRegion=" + SIGNING_REGION,
|
||||
"storage.accessKey=" + AWS_ACCESS_KEY,
|
||||
"storage.secretKey=" + AWS_SECRET_KEY,
|
||||
"storage.client.protocol=" + PROTOCOL,
|
||||
"storage.client.maxErrorRetry=" + MAX_ERROR_RETRY,
|
||||
"storage.bucketName=" + BUCKET_NAME
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
protected Instant getDayInstant() {
|
||||
return LocalDateTime.now().plusDays(1).toInstant(getZoneOffset());
|
||||
}
|
||||
|
||||
protected Instant getSecondInstant() {
|
||||
return LocalDateTime.now().plusSeconds(1).toInstant(getZoneOffset());
|
||||
}
|
||||
|
||||
private ZoneOffset getZoneOffset() {
|
||||
return ZoneOffset.systemDefault().getRules().getOffset(LocalDateTime.now());
|
||||
}
|
||||
|
||||
}
|
||||
167
src/test/java/com/rbkmoney/file/storage/service/Aasd.java
Normal file
167
src/test/java/com/rbkmoney/file/storage/service/Aasd.java
Normal file
@ -0,0 +1,167 @@
|
||||
package com.rbkmoney.file.storage.service;
|
||||
|
||||
import com.rbkmoney.file.storage.AbstractIntegrationTest;
|
||||
import com.rbkmoney.file.storage.FileData;
|
||||
import com.rbkmoney.file.storage.FileStorageSrv;
|
||||
import com.rbkmoney.file.storage.NewFileResult;
|
||||
import com.rbkmoney.woody.api.flow.error.WRuntimeException;
|
||||
import com.rbkmoney.woody.thrift.impl.http.THSpawnClientBuilder;
|
||||
import org.apache.thrift.TException;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
import org.springframework.core.io.FileSystemResource;
|
||||
import org.springframework.http.HttpEntity;
|
||||
import org.springframework.http.HttpHeaders;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.util.LinkedMultiValueMap;
|
||||
import org.springframework.util.MultiValueMap;
|
||||
import org.springframework.web.client.RestTemplate;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.net.HttpURLConnection;
|
||||
import java.net.URI;
|
||||
import java.net.URISyntaxException;
|
||||
import java.net.URL;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardCopyOption;
|
||||
import java.util.Collections;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
|
||||
public class Aasd extends AbstractIntegrationTest {
|
||||
|
||||
private static final int TIMEOUT = 555000;
|
||||
|
||||
private FileStorageSrv.Iface client;
|
||||
|
||||
@Before
|
||||
public void before() throws URISyntaxException {
|
||||
client = new THSpawnClientBuilder()
|
||||
.withAddress(new URI("http://localhost:" + port + "/file_storage"))
|
||||
.withNetworkTimeout(TIMEOUT)
|
||||
.build(FileStorageSrv.Iface.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void extractMetadataTest() throws TException {
|
||||
NewFileResult testFile = client.createNewFile("test_file", Collections.emptyMap(), getDayInstant().toString());
|
||||
FileData fileData = client.getFileData(testFile.getFileData().getFileId());
|
||||
|
||||
assertEquals(fileData, testFile.getFileData());
|
||||
}
|
||||
|
||||
@Test(expected = WRuntimeException.class)
|
||||
public void expiredTimeForFileDataInMetadataTest() throws TException, InterruptedException {
|
||||
NewFileResult testFile = client.createNewFile("test_file", Collections.emptyMap(), getSecondInstant().toString());
|
||||
Thread.sleep(1000);
|
||||
client.getFileData(testFile.getFileData().getFileId());
|
||||
}
|
||||
|
||||
@Test(expected = WRuntimeException.class)
|
||||
public void expiredTimeForGenerateUrlInMetadataTest() throws TException, InterruptedException {
|
||||
NewFileResult testFile = client.createNewFile("test_file", Collections.emptyMap(), getSecondInstant().toString());
|
||||
Thread.sleep(1000);
|
||||
client.generateDownloadUrl(testFile.getFileData().getFileId(), getSecondInstant().toString());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void downloadUrlTest() throws TException, IOException {
|
||||
Path testFile = Files.createTempFile("", "test_file");
|
||||
Files.write(testFile, new byte[0]);
|
||||
|
||||
Path testActualFile = Files.createTempFile("", "test_actual_file");
|
||||
|
||||
NewFileResult fileResult = client.createNewFile("test_file", Collections.emptyMap(), getDayInstant().toString());
|
||||
String s = client.generateDownloadUrl(fileResult.getFileData().getFileId(), getDayInstant().toString());
|
||||
|
||||
URL url = new URL(s);
|
||||
|
||||
assertEquals(HttpStatus.FORBIDDEN.value(), getHttpURLConnection(url, true, "PUT").getResponseCode());
|
||||
|
||||
assertEquals(HttpStatus.OK.value(), getHttpURLConnection(url, false, "GET").getResponseCode());
|
||||
|
||||
HttpURLConnection urlConnection = getHttpURLConnection(url, false, "GET");
|
||||
InputStream inputStream = urlConnection.getInputStream();
|
||||
|
||||
Files.copy(inputStream, testActualFile, StandardCopyOption.REPLACE_EXISTING);
|
||||
|
||||
assertEquals(Files.readAllLines(testFile), Files.readAllLines(testActualFile));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void expiredTimeForGenerateUrlConnectionInCephTest() throws TException, IOException, InterruptedException {
|
||||
NewFileResult fileResult = client.createNewFile("test_file", Collections.emptyMap(), getDayInstant().toString());
|
||||
String s = client.generateDownloadUrl(fileResult.getFileData().getFileId(), getSecondInstant().toString());
|
||||
|
||||
URL url = new URL(s);
|
||||
|
||||
assertEquals(HttpStatus.OK.value(), getHttpURLConnection(url, false, "GET").getResponseCode());
|
||||
|
||||
Thread.sleep(2000);
|
||||
|
||||
assertEquals(HttpStatus.FORBIDDEN.value(), getHttpURLConnection(url, false, "GET").getResponseCode());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void name() throws IOException, TException {
|
||||
NewFileResult fileResult = client.createNewFile("test_file", Collections.emptyMap(), getDayInstant().toString());
|
||||
String s = fileResult.getUploadUrl();
|
||||
|
||||
Path testFile = Files.createTempFile("", "test_file");
|
||||
Files.write(testFile, "Test".getBytes());
|
||||
|
||||
MultiValueMap<String, Object> body = new LinkedMultiValueMap<>();
|
||||
body.add("file", new FileSystemResource(testFile.toFile()));
|
||||
|
||||
HttpHeaders headers = new HttpHeaders();
|
||||
headers.setContentType(MediaType.MULTIPART_FORM_DATA);
|
||||
|
||||
HttpEntity<MultiValueMap<String, Object>> requestEntity = new HttpEntity<>(body, headers);
|
||||
|
||||
RestTemplate restTemplate = new RestTemplate();
|
||||
restTemplate.postForEntity(s, requestEntity, Void.class);
|
||||
|
||||
|
||||
Path testActualFile = Files.createTempFile("", "test_actual_file");
|
||||
|
||||
String urs = client.generateDownloadUrl(fileResult.getFileData().getFileId(), getDayInstant().toString());
|
||||
|
||||
URL url = new URL(urs);
|
||||
|
||||
HttpURLConnection urlConnection = getHttpURLConnection(url, false, "GET");
|
||||
InputStream inputStream = urlConnection.getInputStream();
|
||||
|
||||
Files.copy(inputStream, testActualFile, StandardCopyOption.REPLACE_EXISTING);
|
||||
|
||||
assertEquals(Files.readAllLines(testFile), Files.readAllLines(testActualFile));
|
||||
}
|
||||
|
||||
private void asd() {
|
||||
/*
|
||||
SimpleClientHttpRequestFactory requestFactory = new SimpleClientHttpRequestFactory();
|
||||
requestFactory.setBufferRequestBody(false);
|
||||
|
||||
InputStream fis = new FileInputStream(testFile.toFile());
|
||||
|
||||
RequestCallback requestCallback = request -> {
|
||||
request.getHeaders().add("Content-type", "application/octet-stream");
|
||||
IOUtils.copy(fis, request.getBody());
|
||||
};
|
||||
HttpMessageConverterExtractor<String> responseExtractor =
|
||||
new HttpMessageConverterExtractor<>(String.class, restTemplate.getMessageConverters());
|
||||
restTemplate.setRequestFactory(requestFactory);
|
||||
*/
|
||||
|
||||
|
||||
}
|
||||
|
||||
private HttpURLConnection getHttpURLConnection(URL url, boolean doOutput, String method) throws IOException {
|
||||
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
|
||||
connection.setDoOutput(doOutput);
|
||||
connection.setRequestMethod(method);
|
||||
return connection;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,20 @@
|
||||
package com.rbkmoney.file.storage.service;
|
||||
|
||||
import com.rbkmoney.file.storage.AbstractIntegrationTest;
|
||||
import org.junit.Test;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
|
||||
import java.io.FileNotFoundException;
|
||||
|
||||
public class MetadataS3ObjectsTest extends AbstractIntegrationTest {
|
||||
|
||||
@Autowired
|
||||
private StorageService storageService;
|
||||
|
||||
@Test
|
||||
public void name() throws FileNotFoundException {
|
||||
// FileData fileData = storageService.createNewFile("test_file", Collections.emptyMap(), getInstant());
|
||||
// URL url = storageService.generateDownloadUrl(fileData.getFileId(), getInstant());
|
||||
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,111 @@
|
||||
package com.rbkmoney.file.storage.service;
|
||||
|
||||
import com.rbkmoney.file.storage.AbstractIntegrationTest;
|
||||
import com.rbkmoney.file.storage.FileData;
|
||||
import org.junit.Test;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.HttpURLConnection;
|
||||
import java.net.URL;
|
||||
import java.nio.file.Path;
|
||||
|
||||
public class PresignedUrlAccessRightsTest extends AbstractIntegrationTest {
|
||||
|
||||
@Autowired
|
||||
private StorageService storageService;
|
||||
|
||||
@Test
|
||||
public void urlForUploadTest() throws IOException {
|
||||
/* String stringForWrite = "This text uploaded as an object via presigned URL.";
|
||||
|
||||
Path testFile = Files.createTempFile("", "test_file");
|
||||
|
||||
Path testActualFile = Files.createTempFile("", "test_actual_file");
|
||||
|
||||
try {
|
||||
FileData fileData = storageService.createNewFile("test_file", Collections.emptyMap(), getInstant());
|
||||
|
||||
// генерация url с доступом только для выгрузки
|
||||
URL url = storageService.generateUploadUrl(fileData.getFileId(), getInstant());
|
||||
|
||||
// ошибка при запросе по url методом get
|
||||
assertEquals(HttpStatus.FORBIDDEN.value(), getHttpURLConnection(url, false, "GET").getResponseCode());
|
||||
|
||||
// Length Required при запросе по url методом put
|
||||
assertEquals(HttpStatus.LENGTH_REQUIRED.value(), getHttpURLConnection(url, true, "PUT").getResponseCode());
|
||||
|
||||
// запись данных методом put
|
||||
HttpURLConnection urlConnection = getHttpURLConnection(url, true, "PUT");
|
||||
|
||||
OutputStreamWriter out = new OutputStreamWriter(urlConnection.getOutputStream());
|
||||
out.write(stringForWrite);
|
||||
out.close();
|
||||
|
||||
// чтобы завершить загрузку вызываем getResponseCode
|
||||
assertEquals(HttpStatus.OK.value(), urlConnection.getResponseCode());
|
||||
|
||||
// файл перезаписывается и затирает метаданные.
|
||||
// запись метаданных
|
||||
// storageService.rewriteFileData(fileData, getInstant());
|
||||
|
||||
copyFromStorageToFile(fileData, testActualFile);
|
||||
|
||||
// testFile пустой
|
||||
assertNotEquals(Files.readAllLines(testFile), Files.readAllLines(testActualFile));
|
||||
|
||||
Files.write(testFile, stringForWrite.getBytes());
|
||||
assertEquals(Files.readAllLines(testFile), Files.readAllLines(testActualFile));
|
||||
} finally {
|
||||
Files.deleteIfExists(testFile);
|
||||
Files.deleteIfExists(testActualFile);
|
||||
}*/
|
||||
}
|
||||
|
||||
@Test
|
||||
public void urlForDownloadTest() throws IOException {
|
||||
/*Path testFile = Files.createTempFile("", "test_file");
|
||||
Files.write(testFile, "4815162342".getBytes());
|
||||
|
||||
Path testActualFile = Files.createTempFile("", "test_actual_file");
|
||||
|
||||
try {
|
||||
FileData fileData = storageService.createNewFile("test_file", Collections.emptyMap(), getInstant());
|
||||
|
||||
// запись тестового файла в цеф
|
||||
storageService.uploadFile(fileData.getFileId(), testFile);
|
||||
|
||||
// генерация url с доступом только для загрузки
|
||||
URL url = storageService.generateDownloadUrl(fileData.getFileId(), getInstant());
|
||||
|
||||
// ошибка при запросе по url методом put
|
||||
assertEquals(HttpStatus.FORBIDDEN.value(), getHttpURLConnection(url, true, "PUT").getResponseCode());
|
||||
|
||||
// ок при запросе по url методом get
|
||||
assertEquals(HttpStatus.OK.value(), getHttpURLConnection(url, false, "GET").getResponseCode());
|
||||
|
||||
// получение содержимого методом get
|
||||
HttpURLConnection urlConnection = getHttpURLConnection(url, false, "GET");
|
||||
InputStream inputStream = urlConnection.getInputStream();
|
||||
|
||||
Files.copy(inputStream, testActualFile, StandardCopyOption.REPLACE_EXISTING);
|
||||
|
||||
assertEquals(Files.readAllLines(testFile), Files.readAllLines(testActualFile));
|
||||
} finally {
|
||||
Files.deleteIfExists(testFile);
|
||||
Files.deleteIfExists(testActualFile);
|
||||
}*/
|
||||
}
|
||||
|
||||
private void copyFromStorageToFile(FileData fileData, Path file) throws IOException {
|
||||
/*InputStream inputStream = storageService.getFile(fileData.getFileId());
|
||||
Files.copy(inputStream, file, StandardCopyOption.REPLACE_EXISTING);*/
|
||||
}
|
||||
|
||||
private HttpURLConnection getHttpURLConnection(URL url, boolean doOutput, String method) throws IOException {
|
||||
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
|
||||
connection.setDoOutput(doOutput);
|
||||
connection.setRequestMethod(method);
|
||||
return connection;
|
||||
}
|
||||
}
|
||||
30
src/test/resources/logback-test.xml
Normal file
30
src/test/resources/logback-test.xml
Normal file
@ -0,0 +1,30 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<configuration>
|
||||
<include resource="org/springframework/boot/logging/logback/defaults.xml"/>
|
||||
<include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
|
||||
|
||||
<root level="info">
|
||||
<appender-ref ref="CONSOLE"/>
|
||||
</root>
|
||||
|
||||
<logger name="org.springframework.jdbc.core.JdbcTemplate">
|
||||
<level value="error"/>
|
||||
</logger>
|
||||
|
||||
<logger name="org.springframework.jdbc.core.StatementCreatorUtils">
|
||||
<level value="error"/>
|
||||
</logger>
|
||||
|
||||
<logger name="org.springframework.jdbc.support.JdbcAccessor">
|
||||
<level value="error"/>
|
||||
</logger>
|
||||
|
||||
<logger name="com.amazonaws">
|
||||
<level value="error"/>
|
||||
</logger>
|
||||
|
||||
<logger name="org.apache.http">
|
||||
<level value="error"/>
|
||||
</logger>
|
||||
|
||||
</configuration>
|
||||
Loading…
Reference in New Issue
Block a user