valitydev/file-storage
14
0
Fork 0
mirror of https://github.com/valitydev/file-storage.git synced 2024-11-06 00:35:22 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

BJ-323: replace rest endpoint on ceph upload url

Browse Source
This commit is contained in:
a.karlov 2018-12-05 19:05:27 +03:00
parent e912f22790
commit 57d4876042
7 changed files with 215 additions and 333 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
pom.xml
View File

@ -28,7 +28,6 @@
<woody.thrift.version>1.1.15</woody.thrift.version> <woody.thrift.version>1.1.15</woody.thrift.version>
<file.storage.proto.version>1.10-f44896e</file.storage.proto.version> <file.storage.proto.version>1.10-f44896e</file.storage.proto.version>
<geck.version>0.6.8</geck.version> <geck.version>0.6.8</geck.version>
<swagger.version>2.8.0</swagger.version>
</properties> </properties>
<dependencies> <dependencies>
@ -77,22 +76,30 @@
<version>1.18.4</version> <version>1.18.4</version>
<scope>provided</scope> <scope>provided</scope>
</dependency> </dependency>
<dependency>
<groupId>io.springfox</groupId>
<artifactId>springfox-swagger2</artifactId>
<version>${swagger.version}</version>
</dependency>
<dependency>
<groupId>io.springfox</groupId>
<artifactId>springfox-swagger-ui</artifactId>
<version>${swagger.version}</version>
</dependency>
<!-- Test libs --> <!-- Test libs -->
<dependency> <dependency>
<groupId>org.springframework.boot</groupId> <groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId> <artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope> <scope>test</scope>
<exclusions>
<exclusion>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- Provide JUnit 5 API -->
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-api</artifactId>
<scope>test</scope>
</dependency>
<!-- and the engine for surefire and failsafe -->
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-engine</artifactId>
<scope>test</scope>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.testcontainers</groupId> <groupId>org.testcontainers</groupId>

26
src/main/java/com/rbkmoney/file/storage/configuration/SwaggerConfig.java
View File

@ -1,26 +0,0 @@
package com.rbkmoney.file.storage.configuration;
import com.rbkmoney.file.storage.contorller.UploadFileController;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import springfox.documentation.builders.PathSelectors;
import springfox.documentation.builders.RequestHandlerSelectors;
import springfox.documentation.spi.DocumentationType;
import springfox.documentation.spring.web.plugins.Docket;
import springfox.documentation.swagger2.annotations.EnableSwagger2;
@Configuration
@EnableSwagger2
public class SwaggerConfig {
@Bean
public Docket api() {
Docket docket = new Docket(DocumentationType.SWAGGER_2)
.select()
.apis(RequestHandlerSelectors.basePackage(UploadFileController.class.getPackage().getName()))
.paths(PathSelectors.any())
.build();
docket.forCodeGeneration(true);
return docket;
}
}

60
src/main/java/com/rbkmoney/file/storage/contorller/UploadFileController.java
View File

@ -1,60 +0,0 @@
package com.rbkmoney.file.storage.contorller;
import com.rbkmoney.file.storage.service.StorageService;
import com.rbkmoney.file.storage.service.exception.StorageFileNotFoundException;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import io.swagger.annotations.ApiResponse;
import io.swagger.annotations.ApiResponses;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.multipart.MultipartFile;
import static com.rbkmoney.file.storage.util.CheckerUtil.checkFile;
import static com.rbkmoney.file.storage.util.CheckerUtil.checkString;
@RestController
@RequestMapping("/api/v1")
@Api(description = "File upload API")
@RequiredArgsConstructor
@Slf4j
public class UploadFileController {
private final StorageService storageService;
@ApiOperation(value = "Request upload file")
@ApiResponses(value = {
@ApiResponse(code = 200, message = "File was uploaded"),
@ApiResponse(code = 401, message = "File id not found"),
@ApiResponse(code = 500, message = "Internal service error")
})
@PostMapping(value = "/upload", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
public ResponseEntity handleFileUpload(@RequestParam(value = "file_id") String fileId,
@RequestParam(value = "file") MultipartFile file) {
try {
log.info("Request handleFileUpload fileId: {}", fileId);
checkFile(file, "Bad request parameter, file required and not empty arg");
checkString(fileId, "Bad request parameter, fileId required and not empty arg");
storageService.uploadFile(fileId, file);
ResponseEntity<Object> responseEntity = ResponseEntity.ok().build();
log.info("Response: ResponseEntity: {}", responseEntity);
return responseEntity;
} catch (StorageFileNotFoundException e) {
log.error("Error when handleFileUpload e: ", e);
return ResponseEntity.notFound().build();
} catch (IllegalArgumentException e) {
log.error("Error when handleFileUpload e: ", e);
return ResponseEntity.badRequest().body(e.getMessage());
} catch (Exception e) {
log.error("Error when handleFileUpload e: ", e);
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body("Failed to request upload file");
}
}
}

48
src/main/java/com/rbkmoney/file/storage/handler/FileStorageHandler.java
View File

@ -27,26 +27,6 @@ public class FileStorageHandler implements FileStorageSrv.Iface {
private final StorageService storageService; private final StorageService storageService;
@Override
public FileData getFileData(String fileId) throws TException {
try {
log.info("Request getFileData fileId: {}", fileId);
checkString(fileId, "Bad request parameter, fileId required and not empty arg");
FileData fileData = storageService.getFileData(fileId);
log.info("Response: fileData: {}", fileData);
return fileData;
} catch (StorageFileNotFoundException e) {
log.error("Error when getFileData e: ", e);
throw new FileNotFound();
} catch (StorageException e) {
log.error("Error when getFileData e: ", e);
throw new WUnavailableResultException(e);
} catch (Exception e) {
log.error("Error when getFileData e: ", e);
throw new TException(e);
}
}
@Override @Override
public NewFileResult createNewFile(String fileName, Map<String, Value> metadata, String expiresAt) throws TException { public NewFileResult createNewFile(String fileName, Map<String, Value> metadata, String expiresAt) throws TException {
try { try {
@ -71,14 +51,14 @@ public class FileStorageHandler implements FileStorageSrv.Iface {
} }
@Override @Override
public String generateDownloadUrl(String fileId, String expiresAt) throws TException { public String generateDownloadUrl(String fileDataId, String expiresAt) throws TException {
try { try {
log.info("Request generateDownloadUrl fileId: {}, expiresAt: {}", fileId, expiresAt); log.info("Request generateDownloadUrl fileDataId: {}, expiresAt: {}", fileDataId, expiresAt);
checkString(fileId, "Bad request parameter, fileId required and not empty arg"); checkString(fileDataId, "Bad request parameter, fileDataId required and not empty arg");
checkString(expiresAt, "Bad request parameter, expiresAt required and not empty arg"); checkString(expiresAt, "Bad request parameter, expiresAt required and not empty arg");
// stringToInstant уже содержит проверки аргемента // stringToInstant уже содержит проверки аргемента
Instant instant = TypeUtil.stringToInstant(expiresAt); Instant instant = TypeUtil.stringToInstant(expiresAt);
URL url = storageService.generateDownloadUrl(fileId, instant); URL url = storageService.generateDownloadUrl(fileDataId, instant);
log.info("Response: url: {}", url); log.info("Response: url: {}", url);
return url.toString(); return url.toString();
} catch (StorageFileNotFoundException e) { } catch (StorageFileNotFoundException e) {
@ -92,4 +72,24 @@ public class FileStorageHandler implements FileStorageSrv.Iface {
throw new TException(e); throw new TException(e);
} }
} }
@Override
public FileData getFileData(String fileDataId) throws TException {
try {
log.info("Request getFileData fileDataId: {}", fileDataId);
checkString(fileDataId, "Bad request parameter, fileDataId required and not empty arg");
FileData fileData = storageService.getFileData(fileDataId);
log.info("Response: fileData: {}", fileData);
return fileData;
} catch (StorageFileNotFoundException e) {
log.error("Error when getFileData e: ", e);
throw new FileNotFound();
} catch (StorageException e) {
log.error("Error when getFileData e: ", e);
throw new WUnavailableResultException(e);
} catch (Exception e) {
log.error("Error when getFileData e: ", e);
throw new TException(e);
}
}
} }

176
src/main/java/com/rbkmoney/file/storage/service/AmazonS3StorageService.java
View File

@ -9,23 +9,17 @@ import com.amazonaws.services.s3.transfer.Upload;
import com.rbkmoney.file.storage.FileData; import com.rbkmoney.file.storage.FileData;
import com.rbkmoney.file.storage.NewFileResult; import com.rbkmoney.file.storage.NewFileResult;
import com.rbkmoney.file.storage.configuration.properties.StorageProperties; import com.rbkmoney.file.storage.configuration.properties.StorageProperties;
import com.rbkmoney.file.storage.contorller.UploadFileController;
import com.rbkmoney.file.storage.service.exception.StorageException; import com.rbkmoney.file.storage.service.exception.StorageException;
import com.rbkmoney.file.storage.service.exception.StorageFileNotFoundException; import com.rbkmoney.file.storage.service.exception.StorageFileNotFoundException;
import com.rbkmoney.file.storage.util.DamselUtil; import com.rbkmoney.file.storage.util.DamselUtil;
import com.rbkmoney.geck.common.util.TypeUtil;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;
import org.springframework.web.servlet.mvc.method.annotation.MvcUriComponentsBuilder;
import javax.annotation.PostConstruct; import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy; import javax.annotation.PreDestroy;
import java.io.ByteArrayInputStream; import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL; import java.net.URL;
import java.time.Instant; import java.time.Instant;
import java.util.*; import java.util.*;
@ -36,9 +30,8 @@ import java.util.stream.Collectors;
@RequiredArgsConstructor @RequiredArgsConstructor
public class AmazonS3StorageService implements StorageService { public class AmazonS3StorageService implements StorageService {
private static final String EXPIRATION_TIME = "x-rbkmoney-file-expiration-time";
private static final String FILE_UPLOADED = "x-rbkmoney-file-uploaded";
private static final String FILEDATA_FILE_ID = "x-rbkmoney-filedata-file-id"; private static final String FILEDATA_FILE_ID = "x-rbkmoney-filedata-file-id";
private static final String FILEDATA_FILEDATA_ID = "x-rbkmoney-filedata-filedata-id";
private static final String FILEDATA_FILE_NAME = "x-rbkmoney-filedata-file-name"; private static final String FILEDATA_FILE_NAME = "x-rbkmoney-filedata-file-name";
private static final String FILEDATA_CREATED_AT = "x-rbkmoney-filedata-created-at"; private static final String FILEDATA_CREATED_AT = "x-rbkmoney-filedata-created-at";
private static final String FILEDATA_METADATA = "x-rbkmoney-filedata-metadata-"; private static final String FILEDATA_METADATA = "x-rbkmoney-filedata-metadata-";
@ -57,34 +50,30 @@ public class AmazonS3StorageService implements StorageService {
} }
} }
@Override
public FileData getFileData(String fileId) throws StorageException {
S3Object s3Object = getS3Object(fileId);
checkFileStatus(s3Object);
return extractFileData(s3Object.getObjectMetadata());
}
@Override @Override
public NewFileResult createNewFile(String fileName, Map<String, com.rbkmoney.damsel.msgpack.Value> metadata, Instant expirationTime) throws StorageException { public NewFileResult createNewFile(String fileName, Map<String, com.rbkmoney.damsel.msgpack.Value> metadata, Instant expirationTime) throws StorageException {
log.info("Trying to create new file to storage, filename='{}', bucketId='{}'", fileName, bucketName); log.info("Trying to create new file to storage, filename='{}', bucketId='{}'", fileName, bucketName);
try { try {
// в хранилище сохраняется пустой файл InputStream emptyContent = getEmptyContent();
InputStream emptyContent = new ByteArrayInputStream(new byte[0]);
String fileId = getFileId(); String fileDataId = getId();
String fileId = getId();
String createdAt = Instant.now().toString(); String createdAt = Instant.now().toString();
FileData fileData = new FileData( FileData fileData = new FileData(
fileDataId,
fileId, fileId,
fileName, fileName,
createdAt, createdAt,
metadata metadata
); );
writeFileToStorage(fileData, emptyContent, expirationTime); // записываем в хранилище пустой файл с метаданными по ключу fileDataId
uploadRequest(fileDataId, fileData, emptyContent);
URL uploadUrl = createUploadUrl(fileId); // генерируем ссылку на запись файла в хранилище напрямую в цеф по ключу fileId
URL uploadUrl = generateUploadUrl(fileId, expirationTime);
log.info( log.info(
"File have been successfully created, fileId='{}', bucketId='{}', filename='{}'", "File have been successfully created, fileId='{}', bucketId='{}', filename='{}'",
@ -107,54 +96,14 @@ public class AmazonS3StorageService implements StorageService {
} }
@Override @Override
public URL generateDownloadUrl(String fileId, Instant expirationTime) throws StorageException { public URL generateDownloadUrl(String fileDataId, Instant expirationTime) throws StorageException {
checkFileStatus(getS3Object(fileId)); String fileId = getValidFileData(fileDataId).getFileId();
return generatePresignedUrl(fileId, expirationTime, HttpMethod.GET); return generatePresignedUrl(fileId, expirationTime, HttpMethod.GET);
} }
@Override @Override
public void uploadFile(String fileId, MultipartFile multipartFile) throws StorageException, IOException { public FileData getFileData(String fileDataId) throws StorageException {
log.info("Trying to upload file to storage, filename='{}', bucketId='{}'", fileId, bucketName); return getValidFileData(fileDataId);
try {
S3Object object = getS3Object(fileId);
checkFileStatus(object);
ObjectMetadata objectMetadata = object.getObjectMetadata();
objectMetadata.addUserMetadata(FILE_UPLOADED, "true");
objectMetadata.setContentLength(multipartFile.getSize());
PutObjectRequest putObjectRequest = new PutObjectRequest(
bucketName,
fileId,
multipartFile.getInputStream(),
objectMetadata
);
putObjectRequest.setMetadata(object.getObjectMetadata());
Upload upload = transferManager.upload(putObjectRequest);
try {
upload.waitForUploadResult();
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
log.info(
"File have been successfully uploaded, fileId='{}', bucketId='{}'",
fileId,
bucketName
);
} catch (AmazonClientException ex) {
throw new StorageException(
String.format(
"Failed to to upload file to storage, filename='%s', bucketId='%s'",
fileId,
bucketName
),
ex
);
}
} }
@PreDestroy @PreDestroy
@ -162,27 +111,27 @@ public class AmazonS3StorageService implements StorageService {
transferManager.shutdownNow(true); transferManager.shutdownNow(true);
} }
private S3Object getS3Object(String fileId) throws StorageException { private S3Object getS3Object(String fileDataId) throws StorageException {
try { try {
log.info( log.info(
"Trying to get file from storage, fileId='{}', bucketId='{}'", "Trying to get file from storage, fileDataId='{}', bucketId='{}'",
fileId, fileDataId,
bucketName bucketName
); );
GetObjectRequest getObjectRequest = new GetObjectRequest(bucketName, fileId); GetObjectRequest getObjectRequest = new GetObjectRequest(bucketName, fileDataId);
S3Object object = s3Client.getObject(getObjectRequest); S3Object object = s3Client.getObject(getObjectRequest);
checkNullable(object, fileId, "File"); checkNullable(object, fileDataId, "File");
log.info( log.info(
"File have been successfully got from storage, fileId='{}', bucketId='{}'", "File have been successfully got from storage, fileDataId='{}', bucketId='{}'",
fileId, fileDataId,
bucketName bucketName
); );
return object; return object;
} catch (AmazonClientException ex) { } catch (AmazonClientException ex) {
throw new StorageException( throw new StorageException(
String.format( String.format(
"Failed to get file from storage, fileId='%s', bucketId='%s'", "Failed to get file from storage, fileDataId='%s', bucketId='%s'",
fileId, fileDataId,
bucketName bucketName
), ),
ex ex
@ -194,26 +143,20 @@ public class AmazonS3StorageService implements StorageService {
log.info("Check file expiration and uploaded status: ETag='{}'", s3Object.getObjectMetadata().getETag()); log.info("Check file expiration and uploaded status: ETag='{}'", s3Object.getObjectMetadata().getETag());
ObjectMetadata objectMetadata = s3Object.getObjectMetadata(); ObjectMetadata objectMetadata = s3Object.getObjectMetadata();
Boolean isUploaded = getBooleanFromObjectMetadata(objectMetadata); String fileId = getFileIdFromObjectMetadata(objectMetadata);
if (isUploaded) { if (s3Client.doesObjectExist(bucketName, fileId)) {
log.info("File was uploaded: ETag='{}'", s3Object.getObjectMetadata().getETag()); log.info("File was uploaded: ETag='{}'", s3Object.getObjectMetadata().getETag());
return; return;
} }
Date expirationTime = getDateFromObjectMetadata(objectMetadata);
Date time = new Date();
if (time.getTime() < expirationTime.getTime()) {
log.info("File was not uploaded, but expiration time is valid: ETag='{}'", s3Object.getObjectMetadata().getETag());
return;
}
// если файл не соотвествует условиям, блокируем доступ к нему // если файл не соотвествует условиям, блокируем доступ к нему
throw new StorageFileNotFoundException(String.format("File access error: fileId='%s', bucketId='%s', create a new file", s3Object.getKey(), bucketName)); throw new StorageFileNotFoundException(String.format("File not found: fileId='%s', bucketId='%s', create a new file", s3Object.getKey(), bucketName));
} }
private FileData extractFileData(ObjectMetadata objectMetadata) { private FileData extractFileData(ObjectMetadata objectMetadata) {
log.info("Trying to extract metadata from storage: ETag='{}'", objectMetadata.getETag()); log.info("Trying to extract metadata from storage: ETag='{}'", objectMetadata.getETag());
String fileId = getUserMetadataParameter(objectMetadata, FILEDATA_FILE_ID); String fileId = getUserMetadataParameter(objectMetadata, FILEDATA_FILE_ID);
String fileDataId = getUserMetadataParameter(objectMetadata, FILEDATA_FILEDATA_ID);
String fileName = getUserMetadataParameter(objectMetadata, FILEDATA_FILE_NAME); String fileName = getUserMetadataParameter(objectMetadata, FILEDATA_FILE_NAME);
String createdAt = getUserMetadataParameter(objectMetadata, FILEDATA_CREATED_AT); String createdAt = getUserMetadataParameter(objectMetadata, FILEDATA_CREATED_AT);
@ -230,7 +173,7 @@ public class AmazonS3StorageService implements StorageService {
fileId, fileId,
bucketName bucketName
); );
return new FileData(fileId, fileName, createdAt, metadata); return new FileData(fileDataId, fileId, fileName, createdAt, metadata);
} }
private URL generatePresignedUrl(String fileId, Instant expirationTime, HttpMethod httpMethod) throws StorageException { private URL generatePresignedUrl(String fileId, Instant expirationTime, HttpMethod httpMethod) throws StorageException {
@ -271,27 +214,35 @@ public class AmazonS3StorageService implements StorageService {
} }
} }
private void writeFileToStorage(FileData fileData, InputStream inputStream, Instant expirationTime) throws AmazonClientException { private ByteArrayInputStream getEmptyContent() {
PutObjectRequest request = createS3Request(fileData, inputStream, expirationTime); return new ByteArrayInputStream(new byte[0]);
s3Client.putObject(request);
} }
private PutObjectRequest createS3Request(FileData fileData, InputStream inputStream, Instant expirationTime) { private void uploadRequest(String id, FileData fileData, InputStream inputStream) throws AmazonClientException {
PutObjectRequest putObjectRequest = createS3Request(id, fileData, inputStream);
Upload upload = transferManager.upload(putObjectRequest);
try {
upload.waitForUploadResult();
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
private PutObjectRequest createS3Request(String id, FileData fileData, InputStream inputStream) {
return new PutObjectRequest( return new PutObjectRequest(
bucketName, bucketName,
fileData.getFileId(), id,
inputStream, inputStream,
createObjectMetadata(fileData, expirationTime) createObjectMetadata(fileData)
); );
} }
private ObjectMetadata createObjectMetadata(FileData fileData, Instant expirationTime) { private ObjectMetadata createObjectMetadata(FileData fileData) {
ObjectMetadata objectMetadata = new ObjectMetadata(); ObjectMetadata objectMetadata = new ObjectMetadata();
objectMetadata.setContentDisposition("attachment;filename=" + fileData.getFileName()); objectMetadata.setContentDisposition("attachment;filename=" + fileData.getFileName());
// file parameters
objectMetadata.addUserMetadata(EXPIRATION_TIME, expirationTime.toString());
objectMetadata.addUserMetadata(FILE_UPLOADED, "false");
// filedata parameters // filedata parameters
objectMetadata.addUserMetadata(FILEDATA_FILEDATA_ID, fileData.getFiledataId());
objectMetadata.addUserMetadata(FILEDATA_FILE_ID, fileData.getFileId()); objectMetadata.addUserMetadata(FILEDATA_FILE_ID, fileData.getFileId());
objectMetadata.addUserMetadata(FILEDATA_FILE_NAME, fileData.getFileName()); objectMetadata.addUserMetadata(FILEDATA_FILE_NAME, fileData.getFileName());
objectMetadata.addUserMetadata(FILEDATA_CREATED_AT, fileData.getCreatedAt()); objectMetadata.addUserMetadata(FILEDATA_CREATED_AT, fileData.getCreatedAt());
@ -301,37 +252,18 @@ public class AmazonS3StorageService implements StorageService {
return objectMetadata; return objectMetadata;
} }
private URL createUploadUrl(String fileId) { private FileData getValidFileData(String fileDataId) throws StorageException {
try { S3Object s3Object = getS3Object(fileDataId);
return MvcUriComponentsBuilder.fromMethodName( checkFileStatus(s3Object);
UploadFileController.class, return extractFileData(s3Object.getObjectMetadata());
"handleFileUpload",
fileId,
null
)
.buildAndExpand()
.encode()
.toUri()
.toURL();
} catch (MalformedURLException e) {
throw new StorageException(
String.format(
"Exception createUploadUrl: fileId='%s', bucketId='%s', create a new file",
fileId,
bucketName
),
e);
}
} }
private Boolean getBooleanFromObjectMetadata(ObjectMetadata objectMetadata) { private URL generateUploadUrl(String fileId, Instant expirationTime) throws StorageException {
String isUploadedString = getUserMetadataParameter(objectMetadata, FILE_UPLOADED); return generatePresignedUrl(fileId, expirationTime, HttpMethod.PUT);
return Boolean.valueOf(isUploadedString);
} }
private Date getDateFromObjectMetadata(ObjectMetadata objectMetadata) throws StorageException { private String getFileIdFromObjectMetadata(ObjectMetadata objectMetadata) {
String expirationTime = getUserMetadataParameter(objectMetadata, EXPIRATION_TIME); return getUserMetadataParameter(objectMetadata, FILEDATA_FILE_ID);
return Date.from(TypeUtil.stringToInstant(expirationTime));
} }
private String getUserMetadataParameter(ObjectMetadata objectMetadata, String key) throws StorageException { private String getUserMetadataParameter(ObjectMetadata objectMetadata, String key) throws StorageException {
@ -339,7 +271,7 @@ public class AmazonS3StorageService implements StorageService {
.orElseThrow(() -> new StorageException("Failed to extract user metadata parameter, " + key + " is null")); .orElseThrow(() -> new StorageException("Failed to extract user metadata parameter, " + key + " is null"));
} }
private String getFileId() { private String getId() {
return UUID.randomUUID().toString(); return UUID.randomUUID().toString();
} }

8
src/main/java/com/rbkmoney/file/storage/service/StorageService.java
View File

@ -4,21 +4,17 @@ import com.rbkmoney.damsel.msgpack.Value;
import com.rbkmoney.file.storage.FileData; import com.rbkmoney.file.storage.FileData;
import com.rbkmoney.file.storage.NewFileResult; import com.rbkmoney.file.storage.NewFileResult;
import com.rbkmoney.file.storage.service.exception.StorageException; import com.rbkmoney.file.storage.service.exception.StorageException;
import org.springframework.web.multipart.MultipartFile;
import java.io.IOException;
import java.net.URL; import java.net.URL;
import java.time.Instant; import java.time.Instant;
import java.util.Map; import java.util.Map;
public interface StorageService { public interface StorageService {
FileData getFileData(String fileId) throws StorageException;
NewFileResult createNewFile(String fileName, Map<String, Value> metadata, Instant expirationTime) throws StorageException; NewFileResult createNewFile(String fileName, Map<String, Value> metadata, Instant expirationTime) throws StorageException;
URL generateDownloadUrl(String fileId, Instant expirationTime) throws StorageException; URL generateDownloadUrl(String fileDataId, Instant expirationTime) throws StorageException;
void uploadFile(String fileId, MultipartFile multipartFile) throws StorageException, IOException; FileData getFileData(String fileDataId) throws StorageException;
} }

201
src/test/java/com/rbkmoney/file/storage/FileStorageTest.java
View File

@ -1,34 +1,30 @@
package com.rbkmoney.file.storage; package com.rbkmoney.file.storage;
import com.rbkmoney.file.storage.service.StorageService; import com.rbkmoney.damsel.msgpack.Value;
import org.apache.thrift.TException; import org.apache.thrift.TException;
import org.junit.Test; import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.FileSystemResource;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus; import org.springframework.http.HttpStatus;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap;
import org.springframework.web.client.RestTemplate;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.OutputStreamWriter;
import java.net.HttpURLConnection; import java.net.HttpURLConnection;
import java.net.URL; import java.net.URL;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.nio.file.StandardCopyOption; import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
// все тесты в 1 классе , чтобы сэкономить время на поднятии тест контейнера // все тесты в 1 классе , чтобы сэкономить время на поднятии тест контейнера
public class FileStorageTest extends AbstractIntegrationTest { public class FileStorageTest extends AbstractIntegrationTest {
@Autowired
private StorageService storageService;
@Test @Test
public void uploadAndDownloadFileFromStorageTest() throws IOException, TException { public void uploadAndDownloadFileFromStorageTest() throws IOException, TException {
Path testFile = Files.createTempFile("", "test_file"); Path testFile = Files.createTempFile("", "test_file");
@ -37,35 +33,23 @@ public class FileStorageTest extends AbstractIntegrationTest {
try { try {
// создание нового файла // создание нового файла
NewFileResult fileResult = client.createNewFile("test_file", Collections.emptyMap(), getDayInstant().toString()); String expirationTime = getDayInstant().toString();
String uploadUrl = fileResult.getUploadUrl(); NewFileResult fileResult = client.createNewFile("test_file", Collections.emptyMap(), expirationTime);
uploadTestData(fileResult);
// запись данных в файл
Files.write(testFile, "Test".getBytes());
MultiValueMap<String, Object> body = new LinkedMultiValueMap<>();
// запись файла в тело запроса
body.add("file", new FileSystemResource(testFile.toFile()));
HttpHeaders headers = new HttpHeaders();
HttpEntity<MultiValueMap<String, Object>> requestEntity = new HttpEntity<>(body, headers);
// запись файла в хранилище через ссылку доступа
RestTemplate restTemplate = new RestTemplate();
restTemplate.postForEntity(uploadUrl, requestEntity, Void.class);
// генерация url с доступом только для загрузки // генерация url с доступом только для загрузки
String urs = client.generateDownloadUrl(fileResult.getFileData().getFileId(), getDayInstant().toString()); URL downloadUrl = new URL(client.generateDownloadUrl(fileResult.getFileData().getFiledataId(), expirationTime));
URL url = new URL(urs); HttpURLConnection downloadUrlConnection = getHttpURLConnection(downloadUrl, false, "GET");
InputStream inputStream = downloadUrlConnection.getInputStream();
HttpURLConnection urlConnection = getHttpURLConnection(url, false, "GET");
InputStream inputStream = urlConnection.getInputStream();
// чтение записанного файла из хранилища // чтение записанного файла из хранилища
Files.copy(inputStream, testActualFile, StandardCopyOption.REPLACE_EXISTING); Files.copy(inputStream, testActualFile, StandardCopyOption.REPLACE_EXISTING);
// testFile пустой, testActualFile содержит данные из хранилища
assertNotEquals(Files.readAllLines(testFile), Files.readAllLines(testActualFile));
Files.write(testFile, "test".getBytes());
assertEquals(Files.readAllLines(testFile), Files.readAllLines(testActualFile)); assertEquals(Files.readAllLines(testFile), Files.readAllLines(testActualFile));
} finally { } finally {
Files.deleteIfExists(testFile); Files.deleteIfExists(testFile);
@ -74,80 +58,129 @@ public class FileStorageTest extends AbstractIntegrationTest {
} }
@Test @Test
public void downloadUrlTest() throws TException, IOException { public void uploadUrlConnectionAccessTest() throws IOException, TException {
Path testFile = Files.createTempFile("", "test_file"); // создание файла с доступом к файлу на день
String expirationTime = getDayInstant().toString();
NewFileResult fileResult = client.createNewFile("test_file", Collections.emptyMap(), expirationTime);
Path testActualFile = Files.createTempFile("", "test_actual_file"); String fileDataId = fileResult.getFileData().getFiledataId();
try { // ошибка доступа - файла не существует, тк не было upload
Files.write(testFile, new byte[0]); assertThrows(FileNotFound.class, () -> client.generateDownloadUrl(fileDataId, expirationTime));
assertThrows(FileNotFound.class, () -> client.getFileData(fileDataId));
// создание нового файла URL uploadUrl = new URL(fileResult.getUploadUrl());
NewFileResult fileResult = client.createNewFile("test_file", Collections.emptyMap(), getDayInstant().toString());
// генерация url с доступом только для загрузки // ошибка при запросе по url методом get
URL url = storageService.generateDownloadUrl(fileResult.getFileData().getFileId(), getDayInstant()); assertEquals(HttpStatus.FORBIDDEN.value(), getHttpURLConnection(uploadUrl, false, "GET").getResponseCode());
// с данной ссылкой нельзя записывать // Length Required при запросе по url методом put
assertEquals(HttpStatus.FORBIDDEN.value(), getHttpURLConnection(url, true, "PUT").getResponseCode()); assertEquals(HttpStatus.LENGTH_REQUIRED.value(), getHttpURLConnection(uploadUrl, true, "PUT").getResponseCode());
// можно читать uploadTestData(fileResult);
assertEquals(HttpStatus.OK.value(), getHttpURLConnection(url, false, "GET").getResponseCode());
// чтение данных
HttpURLConnection urlConnection = getHttpURLConnection(url, false, "GET");
InputStream inputStream = urlConnection.getInputStream();
// чтение записанного файла из хранилища
Files.copy(inputStream, testActualFile, StandardCopyOption.REPLACE_EXISTING);
assertEquals(Files.readAllLines(testFile), Files.readAllLines(testActualFile));
} finally {
Files.deleteIfExists(testFile);
Files.deleteIfExists(testActualFile);
}
}
@Test(expected = FileNotFound.class)
public void expiredTimeForFileDataInMetadataTest() throws TException, InterruptedException {
NewFileResult testFile = client.createNewFile("test_file", Collections.emptyMap(), getSecondInstant().toString());
Thread.sleep(1000);
client.getFileData(testFile.getFileData().getFileId());
}
@Test(expected = FileNotFound.class)
public void expiredTimeForGenerateUrlInMetadataTest() throws TException, InterruptedException {
NewFileResult testFile = client.createNewFile("test_file", Collections.emptyMap(), getSecondInstant().toString());
Thread.sleep(1000);
client.generateDownloadUrl(testFile.getFileData().getFileId(), getSecondInstant().toString());
} }
@Test @Test
public void expiredTimeForGenerateUrlConnectionInCephTest() throws TException, IOException, InterruptedException { public void downloadUrlConnectionAccessTest() throws IOException, TException {
NewFileResult fileResult = client.createNewFile("test_file", Collections.emptyMap(), getDayInstant().toString()); // создание файла с доступом к файлу на день
String expirationTime = getDayInstant().toString();
NewFileResult fileResult = client.createNewFile("test_file", Collections.emptyMap(), expirationTime);
URL url = storageService.generateDownloadUrl(fileResult.getFileData().getFileId(), getSecondInstant()); String fileDataId = fileResult.getFileData().getFiledataId();
// ошибка доступа - файла не существует, тк не было upload
assertThrows(FileNotFound.class, () -> client.generateDownloadUrl(fileDataId, expirationTime));
assertThrows(FileNotFound.class, () -> client.getFileData(fileDataId));
// upload тестовых данных в хранилище
uploadTestData(fileResult);
// генерация url с доступом только для загрузки
URL url = new URL(client.generateDownloadUrl(fileDataId, expirationTime));
// с данной ссылкой нельзя записывать
assertEquals(HttpStatus.FORBIDDEN.value(), getHttpURLConnection(url, true, "PUT").getResponseCode());
// можно читать
assertEquals(HttpStatus.OK.value(), getHttpURLConnection(url, false, "GET").getResponseCode()); assertEquals(HttpStatus.OK.value(), getHttpURLConnection(url, false, "GET").getResponseCode());
}
@Test
public void expirationTimeTest() throws TException, InterruptedException, IOException {
// создание файла с доступом к файлу на день
String expirationTime = getDayInstant().toString();
NewFileResult validFileResult = client.createNewFile("test_file", Collections.emptyMap(), expirationTime);
String validFileDataId = validFileResult.getFileData().getFiledataId();
// ошибка доступа - файла не существует, тк не было upload
assertThrows(FileNotFound.class, () -> client.generateDownloadUrl(validFileDataId, expirationTime));
assertThrows(FileNotFound.class, () -> client.getFileData(validFileDataId));
// задержка перед upload для теста expiration
Thread.sleep(1000);
// сохранение тестовых данных в хранилище
uploadTestData(validFileResult);
// доступ есть
client.getFileData(validFileDataId);
client.generateDownloadUrl(validFileDataId, getDayInstant().toString());
// - - - - - сделаем задержку больше expiration
// создание файла с доступом к файлу на секунду
NewFileResult throwingFileResult = client.createNewFile("test_file", Collections.emptyMap(), getSecondInstant().toString());
String throwingFileDataId = throwingFileResult.getFileData().getFiledataId();
// ошибка доступа - файла не существует, тк не было upload
assertThrows(FileNotFound.class, () -> client.generateDownloadUrl(throwingFileDataId, expirationTime));
assertThrows(FileNotFound.class, () -> client.getFileData(throwingFileDataId));
// задержка перед upload для теста expiration
Thread.sleep(2000); Thread.sleep(2000);
assertEquals(HttpStatus.FORBIDDEN.value(), getHttpURLConnection(url, false, "GET").getResponseCode()); // сохранение тестовых данных в хранилище вызывает ошибку доступа
assertThrows(AssertionError.class, () -> uploadTestData(throwingFileResult));
// ошибка доступа
assertThrows(FileNotFound.class, () -> client.getFileData(throwingFileDataId));
assertThrows(FileNotFound.class, () -> client.generateDownloadUrl(throwingFileDataId, expirationTime));
} }
@Test @Test
public void extractMetadataTest() throws TException { public void extractMetadataTest() throws TException, IOException {
String expirationTime = getDayInstant().toString();
String fileName = "test_file"; String fileName = "test_file";
NewFileResult fileResult = client.createNewFile(fileName, Collections.emptyMap(), getDayInstant().toString()); Map<String, Value> metadata = new HashMap<String, Value>() {{
put("key1", Value.b(true));
put("key2", Value.i(1));
put("key3", Value.flt(1));
put("key4", Value.arr(new ArrayList<>()));
put("key5", Value.str("test"));
put("key6", Value.bin(new byte[]{}));
}};
NewFileResult fileResult = client.createNewFile(fileName, metadata, expirationTime);
uploadTestData(fileResult);
assertEquals(fileResult.getFileData().getFileName(), fileName); assertEquals(fileResult.getFileData().getFileName(), fileName);
FileData fileData = storageService.getFileData(fileResult.getFileData().getFileId()); FileData fileData = client.getFileData(fileResult.getFileData().getFiledataId());
assertEquals(fileData, fileResult.getFileData()); assertEquals(fileData, fileResult.getFileData());
} }
private void uploadTestData(final NewFileResult fileResult) throws IOException {
// запись данных методом put
URL uploadUrl = new URL(fileResult.getUploadUrl());
HttpURLConnection uploadUrlConnection = getHttpURLConnection(uploadUrl, true, "PUT");
OutputStreamWriter out = new OutputStreamWriter(uploadUrlConnection.getOutputStream());
out.write("test");
out.close();
// чтобы завершить загрузку вызываем getResponseCode
assertEquals(HttpStatus.OK.value(), uploadUrlConnection.getResponseCode());
}
} }